//! Partial correlation: two traits with other traits held constant. //! //! This is GN3's partial axis (`POST /api/correlation/partial`, whose maths is //! `pingouin.partial_corr`). The definition is the standard one: regress both //! traits on the control traits and correlate what is left -- //! //! ```text //! r_xy|z = corr(x - X_z b_x, y - X_z b_y) //! ``` //! //! -- which for a single control reduces to the textbook identity //! //! ```text //! r_xy|z = (r_xy - r_xz r_yz) / sqrt((1 - r_xz^2)(1 - r_yz^2)) //! ``` //! //! that the tests below pin the implementation to. The regression is by normal //! equations with an intercept and Gaussian elimination with partial pivoting: //! the control set is a handful of traits, so the system is tiny, and avoiding //! a linear-algebra dependency keeps the Guix package as it is. //! //! Three things differ from GN3 on purpose, all recorded rather than silent: //! //! * **Observations.** GN3 accepts a correlation from **4** complete //! observations. With one control that leaves one degree of freedom, so a //! "correlation" is any value in [-1, 1] and its p-value means nothing. We //! keep the six-pair floor the rest of the API uses, and additionally require //! `n >= controls + 3` so the t statistic has at least one degree of freedom. //! * **The Spearman form.** GN3 asks pingouin for `method="spearman"`, which //! rank-transforms the values and then regresses. So do we: ranks of the //! trait, the target and every control, then the same least squares. //! * **Ties.** Ranks are averaged (`1.5` for two tied smallest values), which //! is what GSL's `spearman` -- and therefore the sample axis -- does. use crate::correlation::{self, Hit, Method, Reference, MIN_OVERLAP}; /// One trait's partial correlation against the reference trait, with the /// zero-order (uncontrolled) correlation alongside it. /// /// The zero-order pair is not decoration: a partial correlation that differs /// wildly from it, or that is *larger* in magnitude, is the first thing a /// reader should look at, and GN3 returns both for that reason. #[derive(Debug, Clone, PartialEq)] pub struct PartialHit { pub name: String, /// The partial correlation coefficient. pub r: f64, /// Its two-sided p-value, on `n - controls - 2` degrees of freedom. pub p: f64, /// Complete observations the correlation used. pub n: usize, /// The correlation with nothing held constant. pub zero_order_r: f64, pub zero_order_p: f64, } /// Two-sided p-value of Student's t. /// /// `P(|T| > |t|)` for `t` on `df` degrees of freedom, through the regularised /// incomplete beta function (Numerical Recipes' `betacf`/`betai`; the same /// continued fraction GSL and R use). pub fn t_two_sided_p(t: f64, df: f64) -> f64 { if !t.is_finite() || df <= 0.0 { return f64::NAN; } if t == 0.0 { return 1.0; } let t2 = t * t; let p_one_sided = 0.5 * betai(0.5 * df, 0.5, df / (df + t2)); (2.0 * p_one_sided).clamp(0.0, 1.0) } /// Regularised incomplete beta function `I_x(a, b)`. fn betai(a: f64, b: f64, x: f64) -> f64 { if x <= 0.0 { return 0.0; } if x >= 1.0 { return 1.0; } let ln_factor = ln_gamma(a + b) - ln_gamma(a) - ln_gamma(b) + a * x.ln() + b * (1.0 - x).ln(); let factor = ln_factor.exp(); if x < (a + 1.0) / (a + b + 2.0) { factor * betacf(a, b, x) / a } else { 1.0 - factor * betacf(b, a, 1.0 - x) / b } } /// Continued fraction for the incomplete beta function (Lentz's method). fn betacf(a: f64, b: f64, x: f64) -> f64 { const MAX_ITERATIONS: usize = 200; const EPSILON: f64 = 3.0e-16; const TINY: f64 = 1.0e-300; let qab = a + b; let qap = a + 1.0; let qam = a - 1.0; let mut c = 1.0; let mut d = 1.0 - qab * x / qap; if d.abs() < TINY { d = TINY; } d = 1.0 / d; let mut h = d; for m in 1..=MAX_ITERATIONS { let m = m as f64; let m2 = 2.0 * m; // Even step. let aa = m * (b - m) * x / ((qam + m2) * (a + m2)); d = 1.0 + aa * d; if d.abs() < TINY { d = TINY; } c = 1.0 + aa / c; if c.abs() < TINY { c = TINY; } d = 1.0 / d; h *= d * c; // Odd step. let aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2)); d = 1.0 + aa * d; if d.abs() < TINY { d = TINY; } c = 1.0 + aa / c; if c.abs() < TINY { c = TINY; } d = 1.0 / d; let delta = d * c; h *= delta; if (delta - 1.0).abs() < EPSILON { break; } } h } /// `ln Γ(x)` (Lanczos approximation, as in Numerical Recipes' `gammln`). /// /// The sign of the `tmp` term matters more than it looks: flipping it still /// gives a smooth, monotone function, so nothing crashes -- the p-values just /// come out wrong, which the checkpoint tests below catch (t = 2.228 on 10 /// degrees of freedom must be the 5% two-sided cut-off). fn ln_gamma(x: f64) -> f64 { const COEFFICIENTS: [f64; 6] = [ 76.180_091_729_471_46, -86.505_320_329_416_77, 24.014_098_240_830_91, -1.231_739_572_450_155, 0.120_865_097_386_617_9e-2, -0.539_523_938_495_3e-5, ]; if x < 0.5 { // Reflection: Γ(x)Γ(1-x) = π / sin(πx) return (std::f64::consts::PI / (std::f64::consts::PI * x).sin()).ln() - ln_gamma(1.0 - x); } let y = x; let tmp = (y + 5.5) - (y + 0.5) * (y + 5.5).ln(); let mut series = 1.000_000_000_190_015; for (index, coefficient) in COEFFICIENTS.iter().enumerate() { series += coefficient / (y + 1.0 + index as f64); } -tmp + (2.506_628_274_631_000_5 * series / y).ln() } /// Average ranks, 1-based, ties sharing the mean of their positions. pub fn ranks(values: &[f64]) -> Vec { let mut order: Vec = (0..values.len()).collect(); order.sort_by(|a, b| values[*a].partial_cmp(&values[*b]).unwrap_or(std::cmp::Ordering::Equal)); let mut ranks = vec![0.0; values.len()]; let mut start = 0; while start < order.len() { let mut end = start; while end + 1 < order.len() && values[order[end + 1]] == values[order[start]] { end += 1; } let rank = (start + end) as f64 / 2.0 + 1.0; for position in start..=end { ranks[order[position]] = rank; } start = end + 1; } ranks } /// Residuals of `y` after least squares on `controls` *with an intercept*. /// /// Returns `None` when the control columns are collinear with the intercept /// (two controls naming the same trait, say): the fit is then not unique, and a /// silent answer from an arbitrary solution would be worse than a refusal. pub fn residuals(y: &[f64], controls: &[Vec]) -> Option> { let n = y.len(); let k = controls.len(); if controls.iter().any(|column| column.len() != n) { return None; } if k == 0 { // Nothing held constant: the residual is the value minus its mean. let mean = y.iter().sum::() / n as f64; return Some(y.iter().map(|value| value - mean).collect()); } let width = k + 1; // Design row: [1, z_1, ..., z_k] let design = |row: usize| { let mut values = Vec::with_capacity(width); values.push(1.0); for column in controls { values.push(column[row]); } values }; // Normal equations: (X'X) b = X'y let mut matrix = vec![vec![0.0; width + 1]; width]; for row in 0..n { let x_row = design(row); for i in 0..width { for j in 0..width { matrix[i][j] += x_row[i] * x_row[j]; } matrix[i][width] += x_row[i] * y[row]; } } let coefficients = solve(matrix)?; Some( (0..n) .map(|row| { let x_row = design(row); let fitted: f64 = coefficients.iter().zip(&x_row).map(|(b, x)| b * x).sum(); y[row] - fitted }) .collect(), ) } /// Gaussian elimination with partial pivoting on an augmented matrix. /// /// `None` when a pivot vanishes relative to the matrix scale -- the collinear /// case above. fn solve(mut matrix: Vec>) -> Option> { let size = matrix.len(); let scale = matrix .iter() .flat_map(|row| row.iter()) .fold(0.0_f64, |acc, value| acc.max(value.abs())); if scale == 0.0 { return None; } for column in 0..size { let pivot = (column..size).max_by(|a, b| { matrix[*a][column] .abs() .partial_cmp(&matrix[*b][column].abs()) .unwrap_or(std::cmp::Ordering::Equal) })?; if matrix[pivot][column].abs() <= scale * 1e-12 { return None; } matrix.swap(column, pivot); for row in (column + 1)..size { let factor = matrix[row][column] / matrix[column][column]; for entry in column..=size { matrix[row][entry] -= factor * matrix[column][entry]; } } } let mut solution = vec![0.0; size]; for row in (0..size).rev() { let mut sum = matrix[row][size]; for column in (row + 1)..size { sum -= matrix[row][column] * solution[column]; } solution[row] = sum / matrix[row][row]; } Some(solution) } /// One partial correlation, from aligned (already complete-case) vectors. /// /// `primary` and `target` are the two traits, `controls` the traits held /// constant; all four vectors must be the same length and free of gaps. pub fn partial_one( primary: &[f64], target: &[f64], controls: &[Vec], method: Method, ) -> Option<(f64, f64, f64, f64)> { let n = primary.len(); if n < MIN_OVERLAP || n < controls.len() + 3 || target.len() != n { return None; } let (primary, target, controls) = match method { Method::Pearson => ( primary.to_vec(), target.to_vec(), controls.to_vec(), ), Method::Spearman => ( ranks(primary), ranks(target), controls.iter().map(|column| ranks(column)).collect(), ), }; // Zero-order first, with the engine that the sample axis uses, so the two // numbers in a reply are comparable. let zero_order = correlation::correlate_named_value(&primary, &target, n, method)?; let x_residual = residuals(&primary, &controls)?; let y_residual = residuals(&target, &controls)?; let degrees_of_freedom = (n - controls.len() - 2) as f64; let mut r = correlation::pearson_of(&x_residual, &y_residual); if !r.is_finite() { return None; } // The same last-bit artefact the sample axis clamps: a self-correlation // arrives as 1.0000000000000002, which makes 1 - r^2 negative and its // p-value NaN. let p = if r.abs() >= 1.0 { r = r.signum(); 0.0 } else { let t = r * (degrees_of_freedom / (1.0 - r * r)).sqrt(); t_two_sided_p(t, degrees_of_freedom) }; Some((r, p, zero_order.0, zero_order.1)) } /// Every target trait's partial correlation against the reference trait. /// /// The alignment is per target: the individuals that have a value for the /// reference trait, for *every* control, and for that target. A target that /// leaves fewer than [`MIN_OVERLAP`] such individuals is skipped, and the count /// travels in the reply's note. pub fn correlate_partial( reference: &Reference, controls: &[Reference], rows: impl IntoIterator)>, method: Method, top_n: usize, ) -> Vec { let mut grouped: Vec<(String, Vec<(String, Option)>)> = Vec::new(); let mut positions: std::collections::HashMap = std::collections::HashMap::new(); for (trait_name, individual, value) in rows { let position = match positions.get(&trait_name) { Some(position) => *position, None => { grouped.push((trait_name.clone(), Vec::new())); let position = grouped.len() - 1; positions.insert(trait_name, position); position } }; grouped[position].1.push((individual, value)); } let mut hits: Vec = grouped .iter() .filter_map(|(name, pairs)| { let target = Reference::from_pairs(pairs.iter().cloned()); // Complete cases across the reference, the target and every // control, in the reference's own order. let mut primary = Vec::new(); let mut target_values = Vec::new(); let mut control_values: Vec> = vec![Vec::new(); controls.len()]; for (index, individual) in reference.names.iter().enumerate() { let Some(y) = target .names .iter() .position(|name| name == individual) .map(|position| target.values[position]) else { continue; }; let mut control_row = Vec::with_capacity(controls.len()); let mut complete = true; for control in controls { match control .names .iter() .position(|name| name == individual) .map(|position| control.values[position]) { Some(value) => control_row.push(value), None => { complete = false; break; } } } if !complete { continue; } primary.push(reference.values[index]); target_values.push(y); for (column, value) in control_row.into_iter().enumerate() { control_values[column].push(value); } } let (r, p, zero_order_r, zero_order_p) = partial_one(&primary, &target_values, &control_values, method)?; Some(PartialHit { name: name.clone(), r, p, n: primary.len(), zero_order_r, zero_order_p, }) }) .collect(); // |partial r| descending, undefined last, then the same top-N as the other // axes: a caller comparing axes sees one ordering rule. hits.sort_by(|a, b| { let left = if a.r.is_nan() { f64::NEG_INFINITY } else { a.r.abs() }; let right = if b.r.is_nan() { f64::NEG_INFINITY } else { b.r.abs() }; right .partial_cmp(&left) .unwrap_or(std::cmp::Ordering::Equal) }); hits.truncate(top_n); hits } /// The sample axis' own pairs as `Hit`s, for callers that want the shape. pub fn as_hits(hits: Vec) -> Vec { hits.into_iter() .map(|hit| Hit { name: hit.name, r: hit.r, p: hit.p, n: hit.n, }) .collect() } #[cfg(test)] mod tests { use super::*; /// Two controls, correlated with each other and with both traits: the /// textbook case where the partial r differs sharply from the zero-order /// one. fn fixture() -> (Vec, Vec, Vec>) { let z1: Vec = (0..20).map(|i| i as f64).collect(); let z2: Vec = (0..20).map(|i| ((i * 7) % 13) as f64).collect(); // x = 2 z1 + noise, y = 2 z1 + noise: nearly everything is the control. let x: Vec = (0..20) .map(|i| 2.0 * z1[i] + 0.30 * ((i * 5) % 7) as f64) .collect(); let y: Vec = (0..20) .map(|i| 2.0 * z1[i] + 0.25 * ((i * 3) % 5) as f64) .collect(); (x, y, vec![z1, z2]) } #[test] fn one_control_matches_the_textbook_identity() { let (x, y, controls) = fixture(); let (r, p, _zero_r, _zero_p) = partial_one(&x, &y, &controls[..1], Method::Pearson) .expect("twenty observations, one control"); // r_xy|z = (r_xy - r_xz r_yz) / sqrt((1 - r_xz^2)(1 - r_yz^2)) let corr = |a: &[f64], b: &[f64]| { correlation::pearson_of( &a.iter().map(|v| v - a.iter().sum::() / a.len() as f64).collect::>(), &b.iter().map(|v| v - b.iter().sum::() / b.len() as f64).collect::>(), ) }; let (r_xy, r_xz, r_yz) = (corr(&x, &y), corr(&x, &controls[0]), corr(&y, &controls[0])); let expected = (r_xy - r_xz * r_yz) / ((1.0 - r_xz * r_xz) * (1.0 - r_yz * r_yz)).sqrt(); assert!( (r - expected).abs() < 1e-12, "partial {r} vs identity {expected}" ); // n = 20, one control: 17 degrees of freedom. assert!((p - t_two_sided_p(r * (17.0f64 / (1.0 - r * r)).sqrt(), 17.0)).abs() < 1e-15); } #[test] fn no_controls_reproduces_the_sample_axis_exactly() { let (x, y, _controls) = fixture(); let (r, p, zero_r, zero_p) = partial_one(&x, &y, &[], Method::Pearson).expect("no controls"); let engine = correlation::correlate_named_value(&x, &y, x.len(), Method::Pearson) .expect("engine"); // The same coefficient bit for bit; the p-value through a different // route to the same Student t (our betai vs the engine's tdist_Q). assert!((r - engine.0).abs() < 1e-15, "k = 0 is the sample axis: {r} vs {}", engine.0); assert!( (p - engine.1).abs() <= engine.1 * 1e-9 + 1e-15, "p {p} vs engine {}", engine.1 ); assert_eq!((zero_r, zero_p), (engine.0, engine.1)); } #[test] fn controlling_for_the_shared_cause_collapses_the_correlation() { let (x, y, controls) = fixture(); let zero = correlation::correlate_named_value(&x, &y, x.len(), Method::Pearson) .expect("engine") .0; let partial = partial_one(&x, &y, &controls[..1], Method::Pearson).expect("one control").0; assert!(zero.abs() > 0.8, "zero-order is strong: {zero}"); assert!( partial.abs() < 0.75, "holding the shared cause constant must shrink it, got {partial}" ); } #[test] fn t_two_sided_p_agrees_with_the_engine_at_n_minus_two() { // The engine's p for a correlation with n pairs is Student's t on n-2 // degrees of freedom; this is the same function, so they must agree to // rounding for any r. for (index, r) in [0.05, 0.3, 0.7, -0.9, 0.99].iter().enumerate() { let n = 12 + index; let df = (n - 2) as f64; let t = r * (df / (1.0 - r * r)).sqrt(); let ours = t_two_sided_p(t, df); assert!(ours.is_finite() && (0.0..=1.0).contains(&ours), "r {r}: {ours}"); // Symmetry and monotonicity, which a broken beta would violate. assert!((t_two_sided_p(-t, df) - ours).abs() < 1e-15); } // Known value: t = 2.228 on 10 df is the 5% two-sided cut-off. assert!((t_two_sided_p(2.228, 10.0) - 0.05).abs() < 5e-4, "{:?}", t_two_sided_p(2.228, 10.0)); // Two degrees of freedom has a closed form: P = 1 - t/sqrt(t^2+2). let t: f64 = 1.7; let closed = 1.0 - t / (t * t + 2.0).sqrt(); assert!((t_two_sided_p(t, 2.0) - closed).abs() < 1e-12); } #[test] fn ranks_share_ties_evenly() { assert_eq!(ranks(&[10.0, 20.0, 20.0, 30.0]), vec![1.0, 2.5, 2.5, 4.0]); } #[test] fn collinear_controls_are_refused_rather_than_guessed() { let x: Vec = (0..20).map(|i| i as f64).collect(); let z = x.clone(); assert!(residuals(&x, &[x.clone(), z]).is_none()); } #[test] fn too_few_observations_for_the_controls_is_no_correlation() { let x: Vec = (0..4).map(|i| i as f64).collect(); let y: Vec = (0..4).map(|i| (i * 2) as f64).collect(); let z: Vec = (0..4).map(|i| (i % 2) as f64).collect(); assert!(partial_one(&x, &y, &[z], Method::Pearson).is_none(), "4 pairs is below the floor"); } #[test] fn the_ordering_is_by_magnitude_with_undefined_last() { let hits = vec![ PartialHit { name: "a".into(), r: 0.3, p: 0.1, n: 20, zero_order_r: 0.4, zero_order_p: 0.1 }, PartialHit { name: "b".into(), r: -0.9, p: 0.1, n: 20, zero_order_r: 0.4, zero_order_p: 0.1 }, PartialHit { name: "c".into(), r: f64::NAN, p: f64::NAN, n: 20, zero_order_r: 0.4, zero_order_p: 0.1 }, PartialHit { name: "d".into(), r: 0.5, p: 0.1, n: 20, zero_order_r: 0.4, zero_order_p: 0.1 }, ]; let mut sorted = hits.clone(); sorted.sort_by(|a, b| { let left = if a.r.is_nan() { f64::NEG_INFINITY } else { a.r.abs() }; let right = if b.r.is_nan() { f64::NEG_INFINITY } else { b.r.abs() }; right.partial_cmp(&left).unwrap_or(std::cmp::Ordering::Equal) }); assert_eq!( sorted.iter().map(|hit| hit.name.as_str()).collect::>(), vec!["b", "d", "a", "c"] ); } }