//! The correlation matrix: chosen traits correlated against each other. //! //! GN2 draws this from a collection (a saved set of traits, which spans //! datasets), and that collection is what `/compute/correlation/matrix` takes: //! a list of `{dataset, trait_name}` references rather than a dataset. //! //! Five decisions worth stating, because each one is a place where the //! reference implementation is inconsistent with itself (ai-memory #102, #103, //! #106): //! //! 1. **One method for the whole matrix.** GN2's `show_corr_matrix.py` flips //! from Pearson to Spearman part-way through a row once a Pearson r passes //! 0.999 -- which the self-correlation always does -- so its matrix is //! asymmetric *by construction*. Here the method is chosen once and every //! cell obeys it, which makes `r[i][j] == r[j][i]` exactly, and a test says //! so. //! 2. **Each unordered pair is computed once and mirrored**: N(N-1)/2 engine //! calls, not N^2. //! 3. **The diagonal is by construction**: r = 1.0, p = null, n = the trait's //! own count. Computing it would return 1.0000000000000002 for half the //! traits and a NaN p (the artefact the row endpoint clamps). //! 4. **Per-cell n is its own matrix**, and a cell whose overlap is below //! `min_overlap` (default 6, the row endpoint's floor) comes back as null //! rather than 0 or a number computed from too few points. GN2 accepts 2, //! where |r| is 1 by definition. //! 5. **The sample axis comes from the first trait's group** (#106), resolved //! once and stated in the reply with the number of individuals it kept. use axum::Json; use serde::Deserialize; use utoipa::ToSchema; use crate::correlation::{self, Method, Reference}; use crate::db; use gnauth_macros::gnauth; use crate::auth::AuthToken; use crate::error::ApiError; use crate::response::{Ctx, Envelope}; use super::correlation::{db_failure, sample_axis, Failure, TraitRef}; /// How many traits a matrix may ask for. /// /// The cost is quadratic in both time (N(N-1)/2 engine calls) and reply size /// (N^2 numbers, twice over for r and n), so the cap is about the reply being /// a reply rather than a download: 1 000 traits is 1 MB of JSON and half a /// million correlations of a few dozen points each. pub const MAX_TRAITS: usize = 1_000; /// The floor is the row endpoint's, unless the caller says otherwise. pub const DEFAULT_MIN_OVERLAP: usize = correlation::MIN_OVERLAP; #[derive(Debug, Clone, Deserialize, ToSchema)] pub struct MatrixRequest { /// The traits to correlate, in the order they should appear in the matrix. pub traits: Vec, /// "pearson" (default) or "spearman" -- one method for the whole matrix. #[serde(default)] pub method: Option, /// GN2's `corr_samples_group`; "samples_primary" (default) resolves the /// sample axis from the **first** trait's group. #[serde(default)] pub corr_samples_group: Option, /// Fewest paired observations a cell may have. Defaults to 6. #[serde(default)] pub min_overlap: Option, /// Also return a clustered ordering (and its tree) for a heatmap: average /// linkage on `1 - r`. The matrix is symmetric, so one order serves both /// axes. #[serde(default)] pub cluster: Option, /// Also return a PCA of the matrix (eigenvalues, explained variance, /// loadings over the traits whose every cell is valid). Off by default: /// the r matrix is worth reading on its own first. #[serde(default)] pub pca: Option, } /// One trait in the matrix, as the reply names it. #[derive(Debug, Clone, serde::Serialize, ToSchema)] pub struct MatrixTrait { pub dataset: String, pub trait_name: String, } /// The matrix itself: `r` and `n` are rows of `r`/`n` in `traits` order. #[derive(Debug, Clone, serde::Serialize, ToSchema)] pub struct Matrix { pub traits: Vec, /// Correlation coefficients. Diagonals are 1.0; cells below `min_overlap` /// are null. pub r: Vec>>, /// Two-sided p-values of those coefficients; null on the diagonal and /// wherever there is no coefficient. pub p: Vec>>, /// Paired observations behind each cell; the diagonal is the trait's own /// count. pub n: Vec>>, } #[utoipa::path(post, path = "/compute/correlation/matrix", summary = "Correlate a list of traits against each other", description = "An N x N correlation matrix over a list of {dataset, trait_name} \ references (a collection, as GN2's matrix form posts it). One method for the whole \ matrix, each unordered pair computed once and mirrored so r is exactly symmetric, \ per-cell n returned as its own matrix, cells with fewer than min_overlap paired \ observations null. PCA is deliberately not part of this: the r matrix has to be \ trusted on its own first.", request_body = MatrixRequest, responses( (status = 200, description = "The matrix", body = Envelope), (status = 401, description = "Missing or invalid token", body = ApiError), (status = 422, description = "Bad request: no traits, too many, unknown method, or a trait with no values", body = ApiError), (status = 500, description = "Internal error", body = ApiError) ) )] #[gnauth(token_only)] pub async fn correlation_matrix( ctx: Ctx, _token: AuthToken, Json(request): Json, ) -> Result>, ApiError> { let rid = ctx.request_info(); let fail = |code: &str, title: &str, detail: String| { ApiError::new( axum::http::StatusCode::UNPROCESSABLE_ENTITY, code, title, detail, "correlation_matrix", ) .with_request_id(rid.clone()) }; if request.traits.is_empty() { return Err(fail( "no_traits", "Nothing to correlate", "give at least one {dataset, trait_name} reference in 'traits'".to_string(), )); } if request.traits.len() > MAX_TRAITS { return Err(fail( "too_many_traits", "Too many traits", format!( "{} traits asked for; a matrix is capped at {MAX_TRAITS}, because the cost is \ quadratic in both the correlations and the reply", request.traits.len() ), )); } let method = Method::parse(request.method.as_deref().unwrap_or("pearson")).map_err(|detail| { fail("invalid_method", "Invalid correlation method", detail) })?; let min_overlap = request.min_overlap.unwrap_or(DEFAULT_MIN_OVERLAP); if min_overlap < 3 { return Err(fail( "invalid_min_overlap", "Invalid min_overlap", format!( "{min_overlap} paired observations per cell is not a correlation; the floor is 3 \ (and the default {DEFAULT_MIN_OVERLAP})" ), )); } let wanted = request.clone(); let matrix = tokio::task::spawn_blocking(move || build(&request, method, min_overlap)) .await .map_err(|error| { ApiError::new( axum::http::StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), "correlation_matrix", ) .with_request_id(rid.clone()) })? .map_err(|failure| failure.into_api(rid.clone()))?; let traits = matrix.traits.len(); let pairs = traits * traits.saturating_sub(1) / 2; let mut measured = 0usize; for (row, cells) in matrix.r.iter().enumerate() { for (column, cell) in cells.iter().enumerate() { if column != row && cell.is_some() { measured += 1; } } } let measured = measured / 2; let pca = if wanted.pca.unwrap_or(false) { Some(crate::pca::analyse(&matrix)) } else { None }; let mut body = serde_json::json!({ "traits": matrix.traits, "r": matrix.r, "p": matrix.p, "n": matrix.n, }); if let Some(pca) = &pca { body["pca"] = serde_json::to_value(pca).unwrap_or(serde_json::Value::Null); } let clustering = if wanted.cluster.unwrap_or(false) { // Refused rather than guessed when a cell is missing: an unknown // distance treated as 1 would invent a structure and a plot of it. crate::heatmap::cluster(&matrix.r, crate::pca::MAX_PCA_TRAITS) } else { None }; if let Some(clustering) = &clustering { body["heatmap"] = serde_json::to_value(clustering).unwrap_or(serde_json::Value::Null); } Ok(Json( ctx.envelope(body) .with_note(&format!( "{} trait(s): {} unordered pair(s), {measured} of them correlated with at least \ {min_overlap} paired observations.", traits, pairs )) .with_note(&format!( "Method {}; each pair is computed once and mirrored, so r[i][j] == r[j][i] \ exactly. The diagonal is r = 1, p = null, n = the trait's own count.", method.as_str() )) .with_note(&format!( "The sample axis is the first trait's group ({}), resolved once: {} paired \ observation(s) at most. Samples were not selected per pair.", wanted .traits .first() .map(|trait_ref| format!("{}::{}", trait_ref.dataset, trait_ref.trait_name)) .unwrap_or_default(), matrix_axis_size(&matrix) )) .with_note( "Cells with fewer than min_overlap observations are null in r and p, and their \ n still says how many there were.", ) .with_note(&match (&clustering, wanted.cluster.unwrap_or(false)) { (Some(clustering), _) => format!( "Clustering: average linkage on 1 - r, {} merge(s), largest distance {:.3}; \ `heatmap.order` is the drawing order for both axes.", clustering.merges.len(), clustering.max_height ), (None, true) => "No clustering: the matrix has a gap (a null cell), so some \ distances are unknown -- drop those traits and ask again." .to_string(), (None, false) => "No clustering: pass \"cluster\": true for a heatmap order." .to_string(), }) .with_note(&match &pca { Some(pca) if pca.components.is_empty() => format!( "PCA: not computed -- {} trait(s) were usable and it needs at least 2, and \ at most {}.", matrix_trait_count(&matrix) - pca.dropped.len(), crate::pca::MAX_PCA_TRAITS ), Some(pca) => format!( "PCA: {} component(s) over {} trait(s); {} dropped for gaps, and the \ eigenvalues sum to {:.6} (it should equal the trait count).", pca.components.len(), matrix_trait_count(&matrix) - pca.dropped.len(), pca.dropped.len(), pca.trace ), None => "No PCA: pass \"pca\": true to fold the matrix into principal \ components." .to_string(), }) .with_next("/compute/correlation", "The row form: one trait against a whole dataset"), )) } /// How many traits the request asked for. fn matrix_trait_count(matrix: &Matrix) -> usize { matrix.traits.len() } /// The largest `n` on the diagonal, which is the axis' own size. fn matrix_axis_size(matrix: &Matrix) -> usize { matrix .n .iter() .enumerate() .filter_map(|(index, row)| row.get(index).copied().flatten()) .max() .unwrap_or(0) } /// Resolve every trait, then correlate the pairs. fn build( request: &MatrixRequest, method: Method, min_overlap: usize, ) -> Result { let axis = matrix_axis(request)?; let references = resolve_traits(request, axis.as_deref())?; let count = references.len(); let mut r = vec![vec![None; count]; count]; let mut p = vec![vec![None; count]; count]; let mut n = vec![vec![None; count]; count]; for (index, reference) in references.iter().enumerate() { // The diagonal: by construction, not by computing 1.0000000000000002. r[index][index] = Some(1.0); n[index][index] = Some(reference.len()); // Only the upper triangle: the lower one is mirrored below. for (other, target) in references.iter().enumerate().skip(index + 1) { let pairs: Vec<(String, Option)> = target .names .iter() .cloned() .zip(target.values.iter().copied().map(Some)) .collect(); let overlap = pairs .iter() .filter(|(name, value)| { value.is_some() && reference.names.iter().any(|mine| mine == name) }) .count(); n[index][other] = Some(overlap); n[other][index] = Some(overlap); if overlap < min_overlap { continue; } if let Some(hit) = correlation::correlate_one(reference, method, &pairs) { if hit.r.is_finite() { r[index][other] = Some(hit.r); r[other][index] = Some(hit.r); let p_value = if hit.p.is_finite() { Some(hit.p) } else { None }; p[index][other] = p_value; p[other][index] = p_value; } } } } Ok(Matrix { traits: request .traits .iter() .map(|trait_ref| MatrixTrait { dataset: trait_ref.dataset.clone(), trait_name: trait_ref.trait_name.clone(), }) .collect(), r, p, n, }) } /// The individuals every trait is measured over: the first trait's group. fn matrix_axis(request: &MatrixRequest) -> Result>, Failure> { let Some(first) = request.traits.first() else { return Ok(None); }; match super::correlation::SampleGroup::parse(request.corr_samples_group.as_deref())? { super::correlation::SampleGroup::Primary => sample_axis(&first.dataset), super::correlation::SampleGroup::All => Ok(None), } } /// One `Reference` per requested trait, fetched one query per dataset. fn resolve_traits( request: &MatrixRequest, axis: Option<&[String]>, ) -> Result, Failure> { // Group the request by dataset, keeping the caller's order in the reply. let mut datasets: Vec<(String, Vec)> = Vec::new(); for (index, trait_ref) in request.traits.iter().enumerate() { match datasets.iter_mut().find(|(name, _)| name == &trait_ref.dataset) { Some((_, indices)) => indices.push(index), None => datasets.push((trait_ref.dataset.clone(), vec![index])), } } let mut values: Vec)>> = vec![Vec::new(); request.traits.len()]; for (dataset, indices) in &datasets { let names: Vec = indices .iter() .map(|index| request.traits[*index].trait_name.clone()) .collect(); let rows = if let Some(probeset) = db::get_dataset_by_name(dataset).map_err(db_failure)? { db::get_dataset_values_for_traits( probeset.id, &names, axis.unwrap_or(&[]), ) .map_err(db_failure)? } else if let Some(publish) = db::get_publish_freeze_by_name(dataset).map_err(db_failure)? { let ids: Result, Failure> = names .iter() .map(|name| { name.trim().parse::().map_err(|_| { Failure::NoValues(format!( "'{dataset}' is a phenotype dataset, so its traits are addressed by \ numeric id, not by name '{name}'" )) }) }) .collect(); db::get_publish_values_for_traits(publish.inbred_set_id, &ids?, axis.unwrap_or(&[])) .map_err(db_failure)? } else { return Err(Failure::NotFound(format!("no dataset named '{dataset}'"))); }; // Rows are grouped by trait: split them back out to the traits that // asked for them. for (index, name) in indices.iter().zip(&names) { let rows: Vec<(String, Option)> = rows .iter() .filter(|(trait_name, _, _)| trait_name == name) .map(|(_, individual, value)| (individual.clone(), *value)) .collect(); values[*index] = rows .into_iter() .map(|(individual, value)| (name.clone(), individual, value)) .collect(); } } let mut references = Vec::with_capacity(request.traits.len()); for (index, rows) in values.into_iter().enumerate() { let reference = Reference::from_pairs( rows.into_iter() .map(|(_, individual, value)| (individual, value)), ); if reference.is_empty() { return Err(Failure::NoValues(format!( "'{}' of '{}' has no values in the requested samples", request.traits[index].trait_name, request.traits[index].dataset ))); } references.push(reference); } Ok(references) } #[cfg(test)] mod tests { use super::*; /// Two traits over the same six individuals, one pair computed: the matrix /// must be exactly symmetric, because it is one number written twice. fn references() -> Vec { let a = Reference::from_pairs( (0..8).map(|index| (format!("s{index}"), Some(index as f64))).collect::>(), ); let b = Reference::from_pairs( (0..8) .map(|index| (format!("s{index}"), Some(8.0 - index as f64))) .collect::>(), ); vec![a, b] } #[test] fn the_matrix_is_symmetric_and_the_diagonal_is_by_construction() { let references = references(); let request = MatrixRequest { traits: vec![ TraitRef { dataset: "d".into(), trait_name: "a".into() }, TraitRef { dataset: "d".into(), trait_name: "b".into() }, ], method: None, corr_samples_group: None, min_overlap: None, cluster: None, pca: None, }; // build() would go to the database; the arithmetic is what is tested // here, so mirror what it does with the references it is given. let count = references.len(); let mut matrix = Matrix { traits: request .traits .iter() .map(|trait_ref| MatrixTrait { dataset: trait_ref.dataset.clone(), trait_name: trait_ref.trait_name.clone(), }) .collect(), r: vec![vec![None; count]; count], p: vec![vec![None; count]; count], n: vec![vec![None; count]; count], }; for (index, reference) in references.iter().enumerate() { matrix.r[index][index] = Some(1.0); matrix.n[index][index] = Some(reference.len()); for (other, target) in references.iter().enumerate().skip(index + 1) { let pairs: Vec<(String, Option)> = target .names .iter() .cloned() .zip(target.values.iter().copied().map(Some)) .collect(); let hit = correlation::correlate_one(reference, Method::Pearson, &pairs) .expect("eight pairs"); matrix.r[index][other] = Some(hit.r); matrix.r[other][index] = Some(hit.r); matrix.n[index][other] = Some(hit.n); matrix.n[other][index] = Some(hit.n); } } assert_eq!(matrix.r[0][1], matrix.r[1][0], "must be the same number, twice"); assert_eq!(matrix.r[0][0], Some(1.0)); assert_eq!(matrix.n[0][0], Some(8)); // x ascending against y descending is a perfect negative correlation. assert!((matrix.r[0][1].expect("a coefficient") + 1.0).abs() < 1e-12); } #[test] fn the_cap_and_floor_are_stated_numbers() { assert_eq!(MAX_TRAITS, 1_000); assert_eq!(DEFAULT_MIN_OVERLAP, 6); } }