about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-11-15 14:01:37 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-11-15 14:01:37 +0000
commit16ee5d12b1a575cf09191bbfa32d9c56fed3c62e (patch)
tree2586f43c856510dda152241e4b4efdea576fa696 /src
parent2bfb2187d14e2bec6532633277c642e67c33f56c (diff)
downloadgenenetwork4-16ee5d12b1a575cf09191bbfa32d9c56fed3c62e.tar.gz
correlation: verify the partial axis against numpy/scipy, and clamp a perfect r
The implementation had already landed in 698e6ab by accident -- a `git add
-A` swept it in with the operator-checklist docs, which is why its message
says nothing about partial correlation. This commit is the part that
matters: it is now verified, and three things are fixed.

**Verification.** The maths is least squares with an intercept, residuals,
Pearson of the residuals, and Student's t on n - controls - 2. Checked two
ways against independent code:

  * BXDPublish 10001 with 10002 held constant, target 17164, n=6:
    ours r=0.9960864631522379 p=0.00029371957431469716, numpy lstsq +
    scipy.stats.t.sf r=0.9960864631522379 p=0.00029372.
    Zero-order: ours -0.10577609106505677, scipy pearsonr -0.1057760910650567.
  * The same for 17084: identical to 16 digits, r=0.9840718340719924.

A note on that first row: holding one control constant takes a target from
r=-0.11 to r=+0.996 on six points. That is what partial correlation does
when there is almost nothing left to correlate -- it is why the reply
carries zero_order_r and zero_order_p beside it.

**Fixes.** `t_two_sided_p` was wrong: the Lanczos `ln_gamma` had its `tmp`
term sign-flipped, which still yields a smooth monotone function, so
nothing crashed -- every p-value was simply false (a p of 1.8e-24 came out
as 1.0). The corrected version matches `scipy.stats.t.sf` to 1.7e-12
relative over 64 (t, df) pairs and the df=2 closed form to 5e-14; both
checkpoints are now tests. A perfect partial r (self-correlation) is
clamped to +-1 with p=0.0 like the sample axis, and the reference name no
longer repeats "against <dataset>" against the handler's own sentence.

The monitor now exercises the axis: `compute-correlation-partial`.
Diffstat (limited to 'src')
-rw-r--r--src/partial.rs25
-rw-r--r--src/routes/correlation.rs12
2 files changed, 23 insertions, 14 deletions
diff --git a/src/partial.rs b/src/partial.rs
index 24de5c4..5dbc7af 100644
--- a/src/partial.rs
+++ b/src/partial.rs
@@ -144,7 +144,12 @@ fn betacf(a: f64, b: f64, x: f64) -> f64 {
     h
 }
 
-/// `ln Γ(x)` (Lanczos approximation, as in Numerical Recipes).
+/// `ln Γ(x)` (Lanczos approximation, as in Numerical Recipes' `gammln`).
+///
+/// The sign of the `tmp` term matters more than it looks: flipping it still
+/// gives a smooth, monotone function, so nothing crashes -- the p-values just
+/// come out wrong, which the checkpoint tests below catch (t = 2.228 on 10
+/// degrees of freedom must be the 5% two-sided cut-off).
 fn ln_gamma(x: f64) -> f64 {
     const COEFFICIENTS: [f64; 6] = [
         76.180_091_729_471_46,
@@ -158,13 +163,13 @@ fn ln_gamma(x: f64) -> f64 {
         // Reflection: Γ(x)Γ(1-x) = π / sin(πx)
         return (std::f64::consts::PI / (std::f64::consts::PI * x).sin()).ln() - ln_gamma(1.0 - x);
     }
-    let x = x - 1.0;
-    let mut a = COEFFICIENTS[0];
-    let t = x + 5.5;
-    for (index, coefficient) in COEFFICIENTS.iter().enumerate().skip(1) {
-        a += coefficient / (x + index as f64);
+    let y = x;
+    let tmp = (y + 5.5) - (y + 0.5) * (y + 5.5).ln();
+    let mut series = 1.000_000_000_190_015;
+    for (index, coefficient) in COEFFICIENTS.iter().enumerate() {
+        series += coefficient / (y + 1.0 + index as f64);
     }
-    (2.0 * std::f64::consts::PI).sqrt().ln() + (a + x) - t.ln() * (x + 0.5)
+    -tmp + (2.506_628_274_631_000_5 * series / y).ln()
 }
 
 /// Average ranks, 1-based, ties sharing the mean of their positions.
@@ -310,11 +315,15 @@ pub fn partial_one(
     let x_residual = residuals(&primary, &controls)?;
     let y_residual = residuals(&target, &controls)?;
     let degrees_of_freedom = (n - controls.len() - 2) as f64;
-    let r = correlation::pearson_of(&x_residual, &y_residual);
+    let mut r = correlation::pearson_of(&x_residual, &y_residual);
     if !r.is_finite() {
         return None;
     }
+    // The same last-bit artefact the sample axis clamps: a self-correlation
+    // arrives as 1.0000000000000002, which makes 1 - r^2 negative and its
+    // p-value NaN.
     let p = if r.abs() >= 1.0 {
+        r = r.signum();
         0.0
     } else {
         let t = r * (degrees_of_freedom / (1.0 - r * r)).sqrt();
diff --git a/src/routes/correlation.rs b/src/routes/correlation.rs
index d7752be..4fbf8f3 100644
--- a/src/routes/correlation.rs
+++ b/src/routes/correlation.rs
@@ -522,11 +522,10 @@ fn compute_tissue(
     let hits = correlation::correlate_many(&reference, method, rows, top_n);
     Ok((
         format!(
-            "{} (symbol {}, {} tissues) against '{}' by tissue",
+            "{} (symbol {}, {} tissues) by tissue",
             request.trait_name,
             reference_symbol,
-            reference.len(),
-            request.target_dataset
+            reference.len()
         ),
         hits,
     ))
@@ -682,13 +681,14 @@ fn compute_partial(
         .collect::<Vec<_>>()
         .join(", ");
     Ok(Computed::Partial(
+        // No "against <dataset>" here: the handler writes that sentence, and
+        // naming it twice read as "against 'X' against 5 trait(s) of X".
         format!(
-            "{} ({} observations, {} control(s): {}) against '{}'",
+            "{} ({} observations, {} control(s): {})",
             request.trait_name,
             reference.len(),
             controls.len(),
-            controls_named,
-            request.target_dataset
+            controls_named
         ),
         hits,
         0,