//! Sample-axis correlation: one trait against every trait of a target dataset, //! observations being individuals. //! //! The arithmetic is the `correlation_rust` engine's, not ours: `Pearson` and //! `Spearman` (16c2a37 made the latter public) and its p-values, which come //! from GSL's t-distribution. What lives here is the part the engine does not //! offer in memory -- the row alignment into its parser, the minimum-overlap //! gate, and the ordering/top-N policy -- plus the fetch, which is the //! expensive half. //! //! Semantics, as decided in docs/correlations.md: //! //! - **minimum overlap 6**, the gn3/python value rather than the engine's 4, //! so a two- or four-point correlation cannot appear at all; //! - **|r| descending, NaNs last** -- the engine's own `custom_float_sorter`, //! so GN2, GN3 and GN4 sort identically; //! - **top-N after the sort**; //! - pearson and spearman only. Bicor does not exist in the engine and is //! refused by name rather than silently answered with something else. use correlation_rust::correlations::{Correlation, Pearson, Spearman}; use correlation_rust::parser::parse_rows_with_names; use correlation_rust::sorter::custom_float_sorter; /// The fewest paired observations a correlation may be computed from. pub const MIN_OVERLAP: usize = 6; /// How many hits a request may ask for when it does not say. pub const DEFAULT_TOP_N: usize = 500; /// The largest top-N we will run: GN2's menu tops out at 20000. pub const MAX_TOP_N: usize = 20_000; /// Which correlation is computed. /// /// Worth knowing when comparing against the live site: GN2's JSON API /// (`/api/v_pre1/correlation`) silently answers **Spearman** whatever method it /// is asked for. `get_sample_r_and_p_values` takes a parameter named `type` and /// is called with `corr_params['type']` -- "sample" -- instead of the method, so /// `if type == 'pearson'` is never true and the `spearmanr` branch always runs. /// /// Measured, with `method=pearson&type=sample` on the live site: the r it /// returns for HC_M2_0606_P's 1427571_at against 1447043_at is /// -0.8446202819617522, which is the Spearman rho over its own published sample /// values to 1e-16, not the Pearson r (-0.8393085803732605). Asked for /// spearman, our endpoint reproduces the live site's five rows exactly -- same /// traits, same order, r to 2.2e-16, p to 5e-10 relative. /// /// So the live site is the reference for the sample axis, the overlap, the /// ordering and the reply shape, but not for the choice of statistic: it does /// not honour the method it is given, which is a bug rather than a convention. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Method { Pearson, Spearman, } impl Method { /// Parse the request's method name. `Err` carries the names we do accept, /// so the caller can say what the request should have been. pub fn parse(name: &str) -> Result { match name.trim().to_ascii_lowercase().as_str() { "pearson" => Ok(Method::Pearson), "spearman" => Ok(Method::Spearman), "bicor" => Err( "bicor is not available: the correlation engine implements pearson and \ spearman only." .to_string(), ), other => Err(format!( "unknown correlation method '{other}'; use pearson or spearman." )), } } pub fn as_str(&self) -> &'static str { match self { Method::Pearson => "pearson", Method::Spearman => "spearman", } } } /// The engine's coefficient and p-value for two already-aligned vectors. /// /// Used by the partial axis, which needs the zero-order correlation of the /// same pairs the partial one was computed from, on the same method and the /// same engine -- so the two numbers in a reply are comparable. pub fn correlate_named_value( x: &[f64], y: &[f64], n: usize, method: Method, ) -> Option<(f64, f64)> { if x.len() != y.len() || n > x.len() || n < 2 { return None; } let (r, p) = match method { Method::Pearson => Pearson::new(n).correlate(x, y), Method::Spearman => Spearman::new(n).correlate(x, y), }; if !r.is_finite() { return None; } if r.abs() >= 1.0 { return Some((r.signum(), 0.0)); } Some((r, p)) } /// Plain Pearson over two equal-length vectors, no p-value. /// /// This is for *residuals*, where n-2 is not the right degrees of freedom and /// the caller computes the p-value itself (see `partial::partial_one`). pub fn pearson_of(x: &[f64], y: &[f64]) -> f64 { if x.len() != y.len() || x.len() < 2 { return f64::NAN; } Pearson::new(x.len()).correlate(x, y).0 } /// One trait's correlation against the reference trait. #[derive(Debug, Clone, PartialEq)] pub struct Hit { /// The trait's name in the target dataset (ProbeSet.Name). pub name: String, pub r: f64, pub p: f64, /// How many pairs the coefficient was computed from. pub n: usize, } /// The reference trait's values, one per individual, in a fixed order. /// /// `names` is the alignment key: a target trait's value is paired with the /// reference value for the *same individual*, never by position, because two /// traits of the same dataset need not list their individuals in the same /// order -- and a dataset's rows may be sparse. #[derive(Debug, Clone, Default)] pub struct Reference { pub names: Vec, pub values: Vec, } impl Reference { /// Build from (individual, value) pairs, keeping only those with a value. pub fn from_pairs(pairs: impl IntoIterator)>) -> Self { let mut reference = Reference::default(); for (name, value) in pairs { if let Some(value) = value.filter(|value| value.is_finite()) { reference.names.push(name); reference.values.push(value); } } reference } pub fn len(&self) -> usize { self.values.len() } pub fn is_empty(&self) -> bool { self.values.is_empty() } /// The value vector the engine parses against. pub fn x_vals(&self) -> &[f64] { &self.values } /// Keep only the individuals in `allowed`, returning how many remain. /// /// This is GN2's `samples_primary`: a correlation over a group is a /// correlation among that group's animals, even when the reference trait /// carries values for more of them. Measured against the live API, the /// difference is not cosmetic -- HC_M2_0606_P's trait 1427571_at has 99 /// values but the group has 67 animals, and the ranking changes /// completely between the two sets. pub fn retain_within(&mut self, allowed: &[String]) -> usize { let mut names = Vec::with_capacity(self.names.len()); let mut values = Vec::with_capacity(self.values.len()); for (name, value) in self.names.iter().zip(self.values.iter()) { if allowed.iter().any(|candidate| candidate == name) { names.push(name.clone()); values.push(*value); } } self.names = names; self.values = values; self.len() } } /// Correlate a target trait (given as (individual, value) pairs) against the /// reference, returning `None` when there is too little overlap to say /// anything. /// /// The alignment is delegated to the engine's own parser: it drops, together /// with the matching reference value, any target value that does not parse -- /// which is how a missing observation is expressed here, as the string "NA". pub fn correlate_one(reference: &Reference, method: Method, pairs: &[(String, Option)]) -> Option { correlate_named(reference, method, "", pairs) } /// As [`correlate_one`], but naming the result -- what the row-level entry /// point uses. pub fn correlate_named( reference: &Reference, method: Method, name: &str, pairs: &[(String, Option)], ) -> Option { if reference.is_empty() { return None; } // The target's values in the reference's order; an individual it lacks is // "NA", which the engine's parser drops together with its reference value. let mut row: Vec = Vec::with_capacity(reference.len() + 1); row.push(name.to_string()); row.extend( reference .names .iter() .map(|individual| match value_of(pairs, individual) { Some(value) => value.to_string(), None => "NA".to_string(), }), ); let row: Vec<&str> = row.iter().map(String::as_str).collect(); let parsed = parse_rows_with_names(reference.x_vals(), &row); let n = parsed.x_vals.len(); if n < MIN_OVERLAP { return None; } let (r, p) = match method { Method::Pearson => Pearson::new(n).correlate(&parsed.x_vals, &parsed.y_vals), Method::Spearman => Spearman::new(n).correlate(&parsed.x_vals, &parsed.y_vals), }; // A perfect correlation: the engine's r can come out as 1.0000000000000002, // so 1 - r^2 is negative and its p-value is NaN. The live GN2 API answers // 0.0 for the same row (scipy's pearsonr), so clamp to the reference's // output rather than exposing the engine's last-bit artefact. let (r, p) = if r.abs() >= 1.0 { (r.signum(), 0.0) } else { (r, p) }; Some(Hit { name: if parsed.row_name.is_empty() { name.to_string() } else { parsed.row_name }, r, p, n, }) } /// Correlate every candidate trait against the reference, ordered and /// truncated as the policy says. /// /// `rows` is one `(trait name, individual, value)` record per value: the shape /// the bulk fetch returns, streamed rather than grouped so the caller can feed /// it from a database cursor. pub fn correlate_many( reference: &Reference, method: Method, rows: impl IntoIterator)>, top_n: usize, ) -> Vec { // Group by trait, keeping the fetch's own order within a trait. let mut by_trait: Vec<(String, Vec<(String, Option)>)> = Vec::new(); let mut positions: std::collections::HashMap = std::collections::HashMap::new(); for (trait_name, individual, value) in rows { let position = match positions.get(&trait_name) { Some(position) => *position, None => { by_trait.push((trait_name.clone(), Vec::new())); let position = by_trait.len() - 1; positions.insert(trait_name, position); position } }; by_trait[position].1.push((individual, value)); } let mut hits: Vec = by_trait .iter() .filter_map(|(name, pairs)| correlate_named(reference, method, name, pairs)) .collect(); // The engine's ordering, reused rather than reimplemented: |r| descending, // NaNs last. let mut sortable: Vec<(String, f64, f64, i32)> = hits .drain(..) .map(|hit| (hit.name, hit.r, hit.p, hit.n as i32)) .collect(); custom_float_sorter(&mut sortable); let take = top_n.clamp(1, MAX_TOP_N); sortable.truncate(take); sortable .into_iter() .map(|(name, r, p, n)| Hit { name, r, p, n: n as usize, }) .collect() } /// One individual's value in a trait's pairs, if it has a usable one. fn value_of(pairs: &[(String, Option)], individual: &str) -> Option { pairs .iter() .find(|(name, _)| name == individual) .and_then(|(_, value)| *value) .filter(|value| value.is_finite()) } #[cfg(test)] mod tests { use super::*; fn expected_names(count: usize) -> Vec { (0..count).map(|index| format!("s{index}")).collect() } fn reference(pairs: &[(&str, f64)]) -> Reference { Reference::from_pairs( pairs .iter() .map(|(name, value)| (name.to_string(), Some(*value))), ) } /// A perfectly monotone relationship correlates 1 under both methods; the /// point of the two-method test is that `Spearman` is reachable at all, /// which it was not before 12c2a37. #[test] fn both_methods_answer_with_the_engine() { let names = ["a", "b", "c", "d", "e", "f"]; let x: Vec<(&str, f64)> = names.iter().enumerate().map(|(i, n)| (*n, i as f64)).collect(); let reference = reference(&x); let y: Vec<(String, Option)> = names .iter() .enumerate() .map(|(i, n)| (n.to_string(), Some(2.0 * i as f64))) .collect(); let pearson = correlate_one(&reference, Method::Pearson, &y).expect("pearson"); let spearman = correlate_one(&reference, Method::Spearman, &y).expect("spearman"); assert!((pearson.r - 1.0).abs() < 1e-12, "{pearson:?}"); assert!((spearman.r - 1.0).abs() < 1e-12, "{spearman:?}"); assert_eq!(pearson.n, 6); } /// Below the minimum overlap the row does not exist at all: this is the /// decision that keeps a two-point +-1 out of the ranking. #[test] fn fewer_than_six_pairs_is_not_a_correlation() { let reference = reference(&[("a", 1.0), ("b", 2.0), ("c", 3.0), ("d", 4.0), ("e", 5.0)]); let y: Vec<(String, Option)> = ["a", "b", "c", "d", "e"] .iter() .map(|n| (n.to_string(), Some(1.0))) .collect(); assert!(correlate_one(&reference, Method::Pearson, &y).is_none()); } /// Individuals are matched by name, never by position: a target trait that /// lists them in another order still lines up, and one that lacks an /// individual loses that pair only -- six of seven survive, which is /// exactly the minimum. #[test] fn alignment_is_by_individual_not_by_position() { let names = ["a", "b", "c", "d", "e", "f", "g"]; let reference = reference( &names .iter() .enumerate() .map(|(i, name)| (*name, i as f64 + 1.0)) .collect::>(), ); // Reversed, and 'd' is absent: six pairs, all perfectly monotone. let y: Vec<(String, Option)> = ["g", "f", "e", "c", "b", "a"] .iter() .map(|name| { let value = names.iter().position(|n| n == name).unwrap() as f64 + 1.0; (name.to_string(), Some(value)) }) .collect(); let hit = correlate_one(&reference, Method::Pearson, &y).expect("six pairs is enough"); assert_eq!(hit.n, 6); assert!((hit.r - 1.0).abs() < 1e-12, "reversed order, same pairs: {hit:?}"); } /// The ordering policy: |r| descending with NaNs last, top-N applied after. #[test] fn ordering_is_by_magnitude_and_top_n_truncates_afterwards() { let reference = reference(&[("a", 1.0), ("b", 2.0), ("c", 3.0), ("d", 4.0), ("e", 5.0), ("f", 6.0)]); // Three candidates: strong positive, strong negative, weak. let mut rows: Vec<(String, String, Option)> = Vec::new(); for (individual, value) in [("a", 1.0), ("b", 2.0), ("c", 3.0), ("d", 4.0), ("e", 5.0), ("f", 6.0)] { rows.push(("strong_pos".to_string(), individual.to_string(), Some(value))); } for (individual, value) in [("a", 6.0), ("b", 5.0), ("c", 4.0), ("d", 3.0), ("e", 2.0), ("f", 1.0)] { rows.push(("strong_neg".to_string(), individual.to_string(), Some(value))); } for (individual, value) in [("a", 1.0), ("b", 1.0), ("c", 1.0), ("d", 1.0), ("e", 1.0), ("f", 1.5)] { rows.push(("weak".to_string(), individual.to_string(), Some(value))); } let hits = correlate_many(&reference, Method::Pearson, rows, 2); assert_eq!(hits.len(), 2, "{hits:?}"); // -1 and +1 tie on magnitude; the weak one must be absent. assert!(hits.iter().all(|hit| hit.r.abs() > 0.99), "{hits:?}"); assert!(!hits.iter().any(|hit| hit.name == "weak"), "{hits:?}"); let all = correlate_many( &reference, Method::Pearson, vec![ ("weak".to_string(), "a".to_string(), Some(1.0)), ("weak".to_string(), "b".to_string(), Some(1.0)), ("weak".to_string(), "c".to_string(), Some(1.0)), ("weak".to_string(), "d".to_string(), Some(1.0)), ("weak".to_string(), "e".to_string(), Some(1.0)), ("weak".to_string(), "f".to_string(), Some(1.5)), ], 10, ); assert_eq!(all.len(), 1, "{all:?}"); } // ---- ported from the reference implementations ------------------------- // // These are the cases genenetwork2 and genenetwork3 assert, copied so that // GN4 answers the same questions the same way. Their own suites are // mostly mocks -- `compute_corr_coeff_p_value` is patched out, so the // numbers themselves are never asserted -- which is why what is portable // is the semantics around the arithmetic, not the arithmetic. /// genenetwork3 `tests/unit/computations/test_correlation.py:: /// test_normalize_values` -- only pairs where *both* sides have a value /// survive; an empty or unparseable side drops the pair. #[test] fn genenetwork3_normalize_values() { // (a_values, b_values) -> the pairs that survive let cases: [(&[Option], &[Option], Vec<(f64, f64)>); 3] = [ ( &[Some(2.3), None, None, Some(3.2), Some(4.1), Some(5.0)], &[Some(3.4), Some(7.2), Some(1.3), None, Some(6.2), Some(4.1)], vec![(2.3, 3.4), (4.1, 6.2), (5.0, 4.1)], ), ( &[Some(2.3), None, Some(1.3), None], &[None, None, None, Some(1.2)], vec![], ), (&[], &[], vec![]), ]; for (a_values, b_values, expected) in cases { let reference = Reference::from_pairs(expected_names(a_values.len()).into_iter().zip(a_values.iter().copied())); let pairs: Vec<(String, Option)> = expected_names(b_values.len()) .into_iter() .zip(b_values.iter().copied()) .collect(); let index = pairs .iter() .filter_map(|(name, value)| value.map(|value| (name.clone(), value))) .collect::>(); let kept: Vec<(f64, f64)> = reference .names .iter() .filter_map(|name| { let y = index .iter() .find(|(candidate, _)| candidate == name) .map(|(_, value)| *value)?; let x = reference.values[reference .names .iter() .position(|candidate| candidate == name)?]; Some((x, y)) }) .collect(); assert_eq!(kept, expected, "a={a_values:?} b={b_values:?}"); } } /// genenetwork3 `tests/unit/computations/test_rust_correlation.py:: /// test_generate_input` -- the row format the engine reads: name first, /// then one field per observation, and a missing value is an *empty* /// field, not a skipped one. #[test] fn genenetwork3_dataset_rows_keep_their_positions() { let x = [1.0, 2.0, 3.0]; // ["14_at",12.1,14.1,None] and friends, as the engine writes them // ("14_at,12.1,14.1,"). let parsed = parse_rows_with_names(&x, &["14_at", "12.1", "14.1", ""]); assert_eq!(parsed.row_name, "14_at"); assert_eq!(parsed.x_vals, vec![1.0, 2.0]); assert_eq!(parsed.y_vals, vec![12.1, 14.1]); let parsed = parse_rows_with_names(&x, &["17_at", "12.1", "14.1", "11.4"]); assert_eq!(parsed.x_vals, vec![1.0, 2.0, 3.0]); assert_eq!(parsed.y_vals, vec![12.1, 14.1, 11.4]); } /// genenetwork3 `tests/unit/computations/test_rust_correlation.py:: /// test_parse_results`, with its own fixture file /// `tests/unit/computations/data/correlation/sorted_results.txt`: the /// engine's order is |r| descending, positives and negatives interleaved. #[test] fn genenetwork3_sorted_results_fixture_orders_by_magnitude() { // Exactly the fixture file's rows, deliberately already sorted. let mut rows = vec![ ("63.62".to_string(), 0.97, 0.00, 12), ("19".to_string(), -0.96, 0.22, 12), ("77.92".to_string(), -0.94, 0.31, 12), ("84.04".to_string(), 0.94, 0.11, 12), ("23".to_string(), -0.91, 0.11, 12), ]; custom_float_sorter(&mut rows); let names: Vec<&str> = rows.iter().map(|row| row.0.as_str()).collect(); assert_eq!(names, vec!["63.62", "19", "77.92", "84.04", "23"]); } /// genenetwork3 `tests/unit/computations/test_rust_correlation.py:: /// test_get_samples` and `test_get_samples_no_excluded`: an unparseable /// sample value (" x") never reaches a correlation, and only shared sample /// keys are used. /// /// GN3's own fixture has four shared samples, which is below its minimum /// overlap -- its test asserts `get_samples`, not a correlation. The same /// case here has to keep at least six pairs, so the fixture is widened and /// what is asserted is that the unparseable value costs exactly its own /// pair and nothing else. #[test] fn genenetwork3_unparseable_and_unshared_samples_are_dropped() { let individuals = ["BXD", "BXD3", "BXD4", "BXD5", "BXD6", "BXD11", "BXD31", "BXD99"]; let reference = Reference::from_pairs( individuals .iter() .enumerate() .map(|(index, name)| (name.to_string(), Some(index as f64 + 1.0))), ); // The target knows all eight, one of them with an unparseable value // (" x" in GN3's fixture), and one GN3-side sample is missing. let mut target: Vec<(String, Option)> = individuals .iter() .enumerate() .map(|(index, name)| (name.to_string(), Some(index as f64 + 1.0 * 2.0))) .collect(); target[2].1 = None; // BXD4: unparseable target.retain(|(name, _)| name != "BXD99"); // BXD99: not shared let hit = correlate_one(&reference, Method::Pearson, &target) .expect("seven shared pairs, one of them unusable: six pairs"); assert_eq!( hit.n, 6, "the unparseable value must cost exactly its own pair, and the unshared \ individual must not be invented" ); assert!((hit.r - 1.0).abs() < 1e-9, "monotone on the surviving pairs: {hit:?}"); } /// The overlap boundary GN3 encodes as `num_overlap > 5`, which is where /// our MIN_OVERLAP = 6 comes from. #[test] fn genenetwork3_overlap_boundary_is_six() { let reference = reference( &[("a", 1.0), ("b", 2.0), ("c", 3.0), ("d", 4.0), ("e", 5.0), ("f", 6.0), ("g", 7.0)], ); let pairs = |count: usize| -> Vec<(String, Option)> { ["a", "b", "c", "d", "e", "f", "g"] .iter() .take(count) .map(|name| (name.to_string(), Some(1.0))) .collect() }; assert!(correlate_one(&reference, Method::Pearson, &pairs(5)).is_none(), "5 pairs"); // Six constant values give NaN r, which the engine reports and GN3 // drops -- but the *boundary* is what is being asserted here. assert_eq!(MIN_OVERLAP, 6); let hit = correlate_one(&reference, Method::Pearson, &pairs(6)).map(|hit| hit.n); assert_eq!(hit, Some(6), "six pairs is enough"); } #[test] fn bicor_is_refused_by_name_and_pearson_is_not() { assert_eq!(Method::parse("pearson").unwrap(), Method::Pearson); assert_eq!(Method::parse(" Spearman ").unwrap(), Method::Spearman); let error = Method::parse("bicor").unwrap_err(); assert!(error.contains("bicor") && error.contains("pearson"), "{error}"); assert!(Method::parse("kendall").is_err()); } }