about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-11-15 14:41:16 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-11-15 14:41:16 +0000
commitedaef7fd56e4ce7acd833e929462293478025a34 (patch)
tree3c8cc2166fd3c3e057842fa0d08a8b9d03dcc59f /src
parent532825fed53fddb4f8f8f750986f7f9ab2a3d5dc (diff)
downloadgenenetwork4-edaef7fd56e4ce7acd833e929462293478025a34.tar.gz
correlation: the matrix endpoint, and the rule that a deploy ends with a monitor run
`POST /compute/correlation/matrix` takes a list of `{dataset, trait_name}`
references -- a collection, as GN2's matrix form posts it, which spans
datasets -- and answers an N x N matrix of r, p and n.

The five decisions that are not obvious, each recorded where a reader will
look for them:

* **One method for the whole matrix.** GN2's `show_corr_matrix.py` flips to
  Spearman mid-row once a Pearson r passes 0.999, which the self-pair always
  does, so its matrix is asymmetric by construction. Here the method is
  chosen once and r[i][j] == r[j][i] exactly.
* **Each unordered pair is computed once and mirrored** -- N(N-1)/2 engine
  calls, not N^2.
* **The diagonal is by construction**: r = 1, p = null, n = the trait's own
  count, so half the traits cannot come back as 1.0000000000000002 with a
  NaN p.
* **Per-cell n is its own matrix**, and a cell under `min_overlap` (default
  6) is null in r and p while its n still says how many points there were.
  GN2 accepts 2, where |r| = 1 by definition.
* **The sample axis is the first trait's group** (#106), resolved once and
  stated in the reply with the number of individuals it kept.

Fetches are one query per dataset (`get_dataset_values_for_traits`,
`get_publish_values_for_traits`), not one per trait: a matrix chooses a
handful out of twenty thousand. Capped at 1 000 traits, because the cost is
quadratic in the correlations and in the reply. No PCA -- the r matrix has
to be trusted on its own first.

Verified twice: cells agree with scipy to the last bit (BXDPublish
10001/10002/10005: 0.6756270426459695, 0.6950450737755965,
0.8819879006700077 against scipy 0.6756270426459695, 0.6950450737755964,
0.8819879006700073), and a cell equals what the row endpoint says for the
same pair (0.6950450737755965 in both). Edge cases exercised: a 1x1 matrix,
`min_overlap` above every cell's n, an unknown trait, `bicor`, and no
traits at all.

The operator checklist now says a deploy ends with `bin/monitor-gn4 --tier
all`, and says why: `/version` can report a commit and a reachable database
while every engine that shells out is dead. The monitor gained a matrix
check whose judge enforces square, symmetric and a unit diagonal.
Diffstat (limited to 'src')
-rw-r--r--src/db.rs80
-rw-r--r--src/lib.rs1
-rw-r--r--src/routes/correlation.rs12
-rw-r--r--src/routes/correlation_matrix.rs461
-rw-r--r--src/routes/mod.rs1
5 files changed, 549 insertions, 6 deletions
diff --git a/src/db.rs b/src/db.rs
index 505ef51..4763331 100644
--- a/src/db.rs
+++ b/src/db.rs
@@ -1831,6 +1831,86 @@ pub fn get_dataset_values_for_individuals(
 /// sets) -- and the values live in `PublishData`, keyed by the same InbredSetId
 /// as the reference trait's group.  Returned as (trait id, individual, value)
 /// so the caller can group it exactly like the probeset case.
+/// Chosen traits of a probeset dataset, restricted to chosen individuals.
+///
+/// The matrix needs a handful of traits out of a dataset that holds tens of
+/// thousands, so it can afford neither "fetch the dataset" nor one query per
+/// trait: this is one round trip for all of them.
+pub fn get_dataset_values_for_traits(
+    dataset_id: u16,
+    traits: &[String],
+    individuals: &[String],
+) -> Result<Vec<(String, String, Option<f64>)>, DbError> {
+    if traits.is_empty() || individuals.is_empty() {
+        return Ok(Vec::new());
+    }
+    let pool = pool()?;
+    let mut conn = pool.get_conn()?;
+    let columns = std::iter::repeat_n("?", individuals.len())
+        .collect::<Vec<_>>()
+        .join(",");
+    let names = std::iter::repeat_n("?", traits.len())
+        .collect::<Vec<_>>()
+        .join(",");
+    let query = format!(
+        "SELECT ProbeSet.Name, Strain.Name, ProbeSetData.value \
+         FROM ProbeSetData, Strain, ProbeSetXRef, ProbeSet \
+         WHERE ProbeSetXRef.ProbeSetFreezeId = ? \
+                AND ProbeSet.Name IN ({names}) \
+                AND ProbeSetXRef.ProbeSetId = ProbeSet.Id \
+                AND ProbeSetXRef.DataId = ProbeSetData.Id \
+                AND ProbeSetData.StrainId = Strain.Id \
+                AND Strain.Name IN ({columns}) \
+         ORDER BY ProbeSet.Name, Strain.Name"
+    );
+    let mut params = mysql::Params::Positional(vec![mysql::Value::from(dataset_id)]);
+    if let mysql::Params::Positional(values) = &mut params {
+        values.extend(traits.iter().map(|name| mysql::Value::from(name.as_str())));
+        values.extend(individuals.iter().map(|name| mysql::Value::from(name.as_str())));
+    }
+    conn.exec_map(&query, params, |(trait_name, individual, value)| {
+        (trait_name, individual, value)
+    })
+    .map_err(DbError::from)
+}
+
+/// The same for phenotype (Publish) traits, addressed by numeric id.
+pub fn get_publish_values_for_traits(
+    inbred_set_id: u16,
+    trait_ids: &[u32],
+    individuals: &[String],
+) -> Result<Vec<(String, String, Option<f64>)>, DbError> {
+    if trait_ids.is_empty() || individuals.is_empty() {
+        return Ok(Vec::new());
+    }
+    let pool = pool()?;
+    let mut conn = pool.get_conn()?;
+    let columns = std::iter::repeat_n("?", individuals.len())
+        .collect::<Vec<_>>()
+        .join(",");
+    let ids = std::iter::repeat_n("?", trait_ids.len())
+        .collect::<Vec<_>>()
+        .join(",");
+    let query = format!(
+        "SELECT CAST(PublishXRef.Id AS CHAR), Strain.Name, PublishData.value \
+         FROM PublishData, Strain, PublishXRef \
+         WHERE PublishXRef.InbredSetId = ? AND PublishXRef.Id IN ({ids}) \
+                AND PublishXRef.DataId = PublishData.Id \
+                AND PublishData.StrainId = Strain.Id \
+                AND Strain.Name IN ({columns}) \
+         ORDER BY PublishXRef.Id, Strain.Name"
+    );
+    let mut params = mysql::Params::Positional(vec![mysql::Value::from(inbred_set_id)]);
+    if let mysql::Params::Positional(values) = &mut params {
+        values.extend(trait_ids.iter().map(|id| mysql::Value::from(*id)));
+        values.extend(individuals.iter().map(|name| mysql::Value::from(name.as_str())));
+    }
+    conn.exec_map(&query, params, |(trait_id, individual, value)| {
+        (trait_id, individual, value)
+    })
+    .map_err(DbError::from)
+}
+
 pub fn get_publish_values_for_individuals(
     inbred_set_id: u16,
     individuals: &[String],
diff --git a/src/lib.rs b/src/lib.rs
index ec19956..5d3ab8f 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -230,6 +230,7 @@ pub fn app() -> Router {
         .routes(routes!(routes::publications::publication_traits))
         .routes(routes!(routes::search::search))
         .routes(routes!(routes::correlation::correlation))
+        .routes(routes!(routes::correlation_matrix::correlation_matrix))
         .routes(routes!(routes::baskets::list_baskets))
         .routes(routes!(routes::baskets::create_basket))
         .routes(routes!(routes::baskets::get_basket))
diff --git a/src/routes/correlation.rs b/src/routes/correlation.rs
index 4fbf8f3..2a7f8a2 100644
--- a/src/routes/correlation.rs
+++ b/src/routes/correlation.rs
@@ -245,7 +245,7 @@ enum Computed {
 
 /// GN2's `corr_samples_group`.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum SampleGroup {
+pub(crate) enum SampleGroup {
     /// The trait's own group -- GN2's default.
     Primary,
     /// Every individual the reference trait has a value for.
@@ -253,7 +253,7 @@ enum SampleGroup {
 }
 
 impl SampleGroup {
-    fn parse(name: Option<&str>) -> Result<Self, Failure> {
+    pub(crate) fn parse(name: Option<&str>) -> Result<Self, Failure> {
         match name.unwrap_or("samples_primary").trim().to_ascii_lowercase().as_str() {
             "" | "samples_primary" => Ok(SampleGroup::Primary),
             "samples_all" => Ok(SampleGroup::All),
@@ -277,14 +277,14 @@ impl SampleGroup {
 }
 
 /// Why a correlation request could not be answered.
-enum Failure {
+pub(crate) enum Failure {
     NotFound(String),
     NoValues(String),
     Database(String),
 }
 
 impl Failure {
-    fn into_api(self, rid: crate::response::RequestInfo) -> ApiError {
+    pub(crate) fn into_api(self, rid: crate::response::RequestInfo) -> ApiError {
         let (status, code, title, detail) = match self {
             Failure::NotFound(detail) => (
                 axum::http::StatusCode::NOT_FOUND,
@@ -319,7 +319,7 @@ impl Failure {
 /// A group with no genotype file on this machine falls back to its
 /// used-for-mapping strains; for BXD those are the same 67 individuals, so the
 /// axis survives either way.
-fn sample_axis(dataset: &str) -> Result<Option<Vec<String>>, Failure> {
+pub(crate) fn sample_axis(dataset: &str) -> Result<Option<Vec<String>>, Failure> {
     let Some(group) = db::get_dataset_group_name(dataset).map_err(db_failure)? else {
         return Ok(None);
     };
@@ -759,6 +759,6 @@ fn resolve_target_rows(
     Err(Failure::NotFound(format!("no dataset named '{dataset}'")))
 }
 
-fn db_failure(error: db::DbError) -> Failure {
+pub(crate) fn db_failure(error: db::DbError) -> Failure {
     Failure::Database(error.to_string())
 }
diff --git a/src/routes/correlation_matrix.rs b/src/routes/correlation_matrix.rs
new file mode 100644
index 0000000..616f4f9
--- /dev/null
+++ b/src/routes/correlation_matrix.rs
@@ -0,0 +1,461 @@
+//! The correlation matrix: chosen traits correlated against each other.
+//!
+//! GN2 draws this from a collection (a saved set of traits, which spans
+//! datasets), and that collection is what `/compute/correlation/matrix` takes:
+//! a list of `{dataset, trait_name}` references rather than a dataset.
+//!
+//! Five decisions worth stating, because each one is a place where the
+//! reference implementation is inconsistent with itself (ai-memory #102, #103,
+//! #106):
+//!
+//! 1. **One method for the whole matrix.** GN2's `show_corr_matrix.py` flips
+//!    from Pearson to Spearman part-way through a row once a Pearson r passes
+//!    0.999 -- which the self-correlation always does -- so its matrix is
+//!    asymmetric *by construction*. Here the method is chosen once and every
+//!    cell obeys it, which makes `r[i][j] == r[j][i]` exactly, and a test says
+//!    so.
+//! 2. **Each unordered pair is computed once and mirrored**: N(N-1)/2 engine
+//!    calls, not N^2.
+//! 3. **The diagonal is by construction**: r = 1.0, p = null, n = the trait's
+//!    own count. Computing it would return 1.0000000000000002 for half the
+//!    traits and a NaN p (the artefact the row endpoint clamps).
+//! 4. **Per-cell n is its own matrix**, and a cell whose overlap is below
+//!    `min_overlap` (default 6, the row endpoint's floor) comes back as null
+//!    rather than 0 or a number computed from too few points. GN2 accepts 2,
+//!    where |r| is 1 by definition.
+//! 5. **The sample axis comes from the first trait's group** (#106), resolved
+//!    once and stated in the reply with the number of individuals it kept.
+
+use axum::Json;
+use serde::Deserialize;
+use utoipa::ToSchema;
+
+use crate::correlation::{self, Method, Reference};
+use crate::db;
+use gnauth_macros::gnauth;
+
+use crate::auth::AuthToken;
+use crate::error::ApiError;
+use crate::response::{Ctx, Envelope};
+
+use super::correlation::{db_failure, sample_axis, Failure, TraitRef};
+
+/// How many traits a matrix may ask for.
+///
+/// The cost is quadratic in both time (N(N-1)/2 engine calls) and reply size
+/// (N^2 numbers, twice over for r and n), so the cap is about the reply being
+/// a reply rather than a download: 1 000 traits is 1 MB of JSON and half a
+/// million correlations of a few dozen points each.
+pub const MAX_TRAITS: usize = 1_000;
+
+/// The floor is the row endpoint's, unless the caller says otherwise.
+pub const DEFAULT_MIN_OVERLAP: usize = correlation::MIN_OVERLAP;
+
+#[derive(Debug, Clone, Deserialize, ToSchema)]
+pub struct MatrixRequest {
+    /// The traits to correlate, in the order they should appear in the matrix.
+    pub traits: Vec<TraitRef>,
+    /// "pearson" (default) or "spearman" -- one method for the whole matrix.
+    #[serde(default)]
+    pub method: Option<String>,
+    /// GN2's `corr_samples_group`; "samples_primary" (default) resolves the
+    /// sample axis from the **first** trait's group.
+    #[serde(default)]
+    pub corr_samples_group: Option<String>,
+    /// Fewest paired observations a cell may have. Defaults to 6.
+    #[serde(default)]
+    pub min_overlap: Option<usize>,
+}
+
+/// One trait in the matrix, as the reply names it.
+#[derive(Debug, Clone, serde::Serialize, ToSchema)]
+pub struct MatrixTrait {
+    pub dataset: String,
+    pub trait_name: String,
+}
+
+/// The matrix itself: `r` and `n` are rows of `r`/`n` in `traits` order.
+#[derive(Debug, Clone, serde::Serialize, ToSchema)]
+pub struct Matrix {
+    pub traits: Vec<MatrixTrait>,
+    /// Correlation coefficients. Diagonals are 1.0; cells below `min_overlap`
+    /// are null.
+    pub r: Vec<Vec<Option<f64>>>,
+    /// Two-sided p-values of those coefficients; null on the diagonal and
+    /// wherever there is no coefficient.
+    pub p: Vec<Vec<Option<f64>>>,
+    /// Paired observations behind each cell; the diagonal is the trait's own
+    /// count.
+    pub n: Vec<Vec<Option<usize>>>,
+}
+
+#[utoipa::path(post, path = "/compute/correlation/matrix",
+    summary = "Correlate a list of traits against each other",
+    description = "An N x N correlation matrix over a list of {dataset, trait_name} \
+references (a collection, as GN2's matrix form posts it). One method for the whole \
+matrix, each unordered pair computed once and mirrored so r is exactly symmetric, \
+per-cell n returned as its own matrix, cells with fewer than min_overlap paired \
+observations null. PCA is deliberately not part of this: the r matrix has to be \
+trusted on its own first.",
+    request_body = MatrixRequest,
+    responses(
+        (status = 200, description = "The matrix", body = Envelope<serde_json::Value>),
+        (status = 401, description = "Missing or invalid token", body = ApiError),
+        (status = 422, description = "Bad request: no traits, too many, unknown method, or a trait with no values", body = ApiError),
+        (status = 500, description = "Internal error", body = ApiError)
+    )
+)]
+#[gnauth(token_only)]
+pub async fn correlation_matrix(
+    ctx: Ctx,
+    _token: AuthToken,
+    Json(request): Json<MatrixRequest>,
+) -> Result<Json<Envelope<serde_json::Value>>, ApiError> {
+    let rid = ctx.request_info();
+    let fail = |code: &str, title: &str, detail: String| {
+        ApiError::new(
+            axum::http::StatusCode::UNPROCESSABLE_ENTITY,
+            code,
+            title,
+            detail,
+            "correlation_matrix",
+        )
+        .with_request_id(rid.clone())
+    };
+
+    if request.traits.is_empty() {
+        return Err(fail(
+            "no_traits",
+            "Nothing to correlate",
+            "give at least one {dataset, trait_name} reference in 'traits'".to_string(),
+        ));
+    }
+    if request.traits.len() > MAX_TRAITS {
+        return Err(fail(
+            "too_many_traits",
+            "Too many traits",
+            format!(
+                "{} traits asked for; a matrix is capped at {MAX_TRAITS}, because the cost is \
+                 quadratic in both the correlations and the reply",
+                request.traits.len()
+            ),
+        ));
+    }
+    let method = Method::parse(request.method.as_deref().unwrap_or("pearson")).map_err(|detail| {
+        fail("invalid_method", "Invalid correlation method", detail)
+    })?;
+    let min_overlap = request.min_overlap.unwrap_or(DEFAULT_MIN_OVERLAP);
+    if min_overlap < 3 {
+        return Err(fail(
+            "invalid_min_overlap",
+            "Invalid min_overlap",
+            format!(
+                "{min_overlap} paired observations per cell is not a correlation; the floor is 3 \
+                 (and the default {DEFAULT_MIN_OVERLAP})"
+            ),
+        ));
+    }
+
+    let wanted = request.clone();
+    let matrix = tokio::task::spawn_blocking(move || build(&request, method, min_overlap))
+        .await
+        .map_err(|error| {
+            ApiError::new(
+                axum::http::StatusCode::INTERNAL_SERVER_ERROR,
+                "internal_error",
+                "Internal error",
+                error.to_string(),
+                "correlation_matrix",
+            )
+            .with_request_id(rid.clone())
+        })?
+        .map_err(|failure| failure.into_api(rid.clone()))?;
+
+    let traits = matrix.traits.len();
+    let pairs = traits * traits.saturating_sub(1) / 2;
+    let mut measured = 0usize;
+    for (row, cells) in matrix.r.iter().enumerate() {
+        for (column, cell) in cells.iter().enumerate() {
+            if column != row && cell.is_some() {
+                measured += 1;
+            }
+        }
+    }
+    let measured = measured / 2;
+    let body = serde_json::json!({
+        "traits": matrix.traits,
+        "r": matrix.r,
+        "p": matrix.p,
+        "n": matrix.n,
+    });
+    Ok(Json(
+        ctx.envelope(body)
+            .with_note(&format!(
+                "{} trait(s): {} unordered pair(s), {measured} of them correlated with at least \
+                 {min_overlap} paired observations.",
+                traits, pairs
+            ))
+            .with_note(&format!(
+                "Method {}; each pair is computed once and mirrored, so r[i][j] == r[j][i] \
+                 exactly. The diagonal is r = 1, p = null, n = the trait's own count.",
+                method.as_str()
+            ))
+            .with_note(&format!(
+                "The sample axis is the first trait's group ({}), resolved once: {} paired \
+                 observation(s) at most. Samples were not selected per pair.",
+                wanted
+                    .traits
+                    .first()
+                    .map(|trait_ref| format!("{}::{}", trait_ref.dataset, trait_ref.trait_name))
+                    .unwrap_or_default(),
+                matrix_axis_size(&matrix)
+            ))
+            .with_note(
+                "Cells with fewer than min_overlap observations are null in r and p, and their \
+                 n still says how many there were. No PCA: that is a separate step.",
+            )
+            .with_next("/compute/correlation", "The row form: one trait against a whole dataset"),
+    ))
+}
+
+/// The largest `n` on the diagonal, which is the axis' own size.
+fn matrix_axis_size(matrix: &Matrix) -> usize {
+    matrix
+        .n
+        .iter()
+        .enumerate()
+        .filter_map(|(index, row)| row.get(index).copied().flatten())
+        .max()
+        .unwrap_or(0)
+}
+
+/// Resolve every trait, then correlate the pairs.
+fn build(
+    request: &MatrixRequest,
+    method: Method,
+    min_overlap: usize,
+) -> Result<Matrix, Failure> {
+    let axis = matrix_axis(request)?;
+    let references = resolve_traits(request, axis.as_deref())?;
+
+    let count = references.len();
+    let mut r = vec![vec![None; count]; count];
+    let mut p = vec![vec![None; count]; count];
+    let mut n = vec![vec![None; count]; count];
+
+    for (index, reference) in references.iter().enumerate() {
+        // The diagonal: by construction, not by computing 1.0000000000000002.
+        r[index][index] = Some(1.0);
+        n[index][index] = Some(reference.len());
+        // Only the upper triangle: the lower one is mirrored below.
+        for (other, target) in references.iter().enumerate().skip(index + 1) {
+            let pairs: Vec<(String, Option<f64>)> = target
+                .names
+                .iter()
+                .cloned()
+                .zip(target.values.iter().copied().map(Some))
+                .collect();
+            let overlap = pairs
+                .iter()
+                .filter(|(name, value)| {
+                    value.is_some() && reference.names.iter().any(|mine| mine == name)
+                })
+                .count();
+            n[index][other] = Some(overlap);
+            n[other][index] = Some(overlap);
+            if overlap < min_overlap {
+                continue;
+            }
+            if let Some(hit) = correlation::correlate_one(reference, method, &pairs) {
+                if hit.r.is_finite() {
+                    r[index][other] = Some(hit.r);
+                    r[other][index] = Some(hit.r);
+                    let p_value = if hit.p.is_finite() { Some(hit.p) } else { None };
+                    p[index][other] = p_value;
+                    p[other][index] = p_value;
+                }
+            }
+        }
+    }
+
+    Ok(Matrix {
+        traits: request
+            .traits
+            .iter()
+            .map(|trait_ref| MatrixTrait {
+                dataset: trait_ref.dataset.clone(),
+                trait_name: trait_ref.trait_name.clone(),
+            })
+            .collect(),
+        r,
+        p,
+        n,
+    })
+}
+
+/// The individuals every trait is measured over: the first trait's group.
+fn matrix_axis(request: &MatrixRequest) -> Result<Option<Vec<String>>, Failure> {
+    let Some(first) = request.traits.first() else {
+        return Ok(None);
+    };
+    match super::correlation::SampleGroup::parse(request.corr_samples_group.as_deref())? {
+        super::correlation::SampleGroup::Primary => sample_axis(&first.dataset),
+        super::correlation::SampleGroup::All => Ok(None),
+    }
+}
+
+/// One `Reference` per requested trait, fetched one query per dataset.
+fn resolve_traits(
+    request: &MatrixRequest,
+    axis: Option<&[String]>,
+) -> Result<Vec<Reference>, Failure> {
+    // Group the request by dataset, keeping the caller's order in the reply.
+    let mut datasets: Vec<(String, Vec<usize>)> = Vec::new();
+    for (index, trait_ref) in request.traits.iter().enumerate() {
+        match datasets.iter_mut().find(|(name, _)| name == &trait_ref.dataset) {
+            Some((_, indices)) => indices.push(index),
+            None => datasets.push((trait_ref.dataset.clone(), vec![index])),
+        }
+    }
+
+    let mut values: Vec<Vec<(String, String, Option<f64>)>> = vec![Vec::new(); request.traits.len()];
+    for (dataset, indices) in &datasets {
+        let names: Vec<String> = indices
+            .iter()
+            .map(|index| request.traits[*index].trait_name.clone())
+            .collect();
+        let rows = if let Some(probeset) = db::get_dataset_by_name(dataset).map_err(db_failure)? {
+            db::get_dataset_values_for_traits(
+                probeset.id,
+                &names,
+                axis.unwrap_or(&[]),
+            )
+            .map_err(db_failure)?
+        } else if let Some(publish) =
+            db::get_publish_freeze_by_name(dataset).map_err(db_failure)?
+        {
+            let ids: Result<Vec<u32>, Failure> = names
+                .iter()
+                .map(|name| {
+                    name.trim().parse::<u32>().map_err(|_| {
+                        Failure::NoValues(format!(
+                            "'{dataset}' is a phenotype dataset, so its traits are addressed by \
+                             numeric id, not by name '{name}'"
+                        ))
+                    })
+                })
+                .collect();
+            db::get_publish_values_for_traits(publish.inbred_set_id, &ids?, axis.unwrap_or(&[]))
+                .map_err(db_failure)?
+        } else {
+            return Err(Failure::NotFound(format!("no dataset named '{dataset}'")));
+        };
+        // Rows are grouped by trait: split them back out to the traits that
+        // asked for them.
+        for (index, name) in indices.iter().zip(&names) {
+            let rows: Vec<(String, Option<f64>)> = rows
+                .iter()
+                .filter(|(trait_name, _, _)| trait_name == name)
+                .map(|(_, individual, value)| (individual.clone(), *value))
+                .collect();
+            values[*index] = rows
+                .into_iter()
+                .map(|(individual, value)| (name.clone(), individual, value))
+                .collect();
+        }
+    }
+
+    let mut references = Vec::with_capacity(request.traits.len());
+    for (index, rows) in values.into_iter().enumerate() {
+        let reference = Reference::from_pairs(
+            rows.into_iter()
+                .map(|(_, individual, value)| (individual, value)),
+        );
+        if reference.is_empty() {
+            return Err(Failure::NoValues(format!(
+                "'{}' of '{}' has no values in the requested samples",
+                request.traits[index].trait_name, request.traits[index].dataset
+            )));
+        }
+        references.push(reference);
+    }
+    Ok(references)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Two traits over the same six individuals, one pair computed: the matrix
+    /// must be exactly symmetric, because it is one number written twice.
+    fn references() -> Vec<Reference> {
+        let a = Reference::from_pairs(
+            (0..8).map(|index| (format!("s{index}"), Some(index as f64))).collect::<Vec<_>>(),
+        );
+        let b = Reference::from_pairs(
+            (0..8)
+                .map(|index| (format!("s{index}"), Some(8.0 - index as f64)))
+                .collect::<Vec<_>>(),
+        );
+        vec![a, b]
+    }
+
+    #[test]
+    fn the_matrix_is_symmetric_and_the_diagonal_is_by_construction() {
+        let references = references();
+        let request = MatrixRequest {
+            traits: vec![
+                TraitRef { dataset: "d".into(), trait_name: "a".into() },
+                TraitRef { dataset: "d".into(), trait_name: "b".into() },
+            ],
+            method: None,
+            corr_samples_group: None,
+            min_overlap: None,
+        };
+        // build() would go to the database; the arithmetic is what is tested
+        // here, so mirror what it does with the references it is given.
+        let count = references.len();
+        let mut matrix = Matrix {
+            traits: request
+                .traits
+                .iter()
+                .map(|trait_ref| MatrixTrait {
+                    dataset: trait_ref.dataset.clone(),
+                    trait_name: trait_ref.trait_name.clone(),
+                })
+                .collect(),
+            r: vec![vec![None; count]; count],
+            p: vec![vec![None; count]; count],
+            n: vec![vec![None; count]; count],
+        };
+        for (index, reference) in references.iter().enumerate() {
+            matrix.r[index][index] = Some(1.0);
+            matrix.n[index][index] = Some(reference.len());
+            for (other, target) in references.iter().enumerate().skip(index + 1) {
+                let pairs: Vec<(String, Option<f64>)> = target
+                    .names
+                    .iter()
+                    .cloned()
+                    .zip(target.values.iter().copied().map(Some))
+                    .collect();
+                let hit = correlation::correlate_one(reference, Method::Pearson, &pairs)
+                    .expect("eight pairs");
+                matrix.r[index][other] = Some(hit.r);
+                matrix.r[other][index] = Some(hit.r);
+                matrix.n[index][other] = Some(hit.n);
+                matrix.n[other][index] = Some(hit.n);
+            }
+        }
+        assert_eq!(matrix.r[0][1], matrix.r[1][0], "must be the same number, twice");
+        assert_eq!(matrix.r[0][0], Some(1.0));
+        assert_eq!(matrix.n[0][0], Some(8));
+        // x ascending against y descending is a perfect negative correlation.
+        assert!((matrix.r[0][1].expect("a coefficient") + 1.0).abs() < 1e-12);
+    }
+
+    #[test]
+    fn the_cap_and_floor_are_stated_numbers() {
+        assert_eq!(MAX_TRAITS, 1_000);
+        assert_eq!(DEFAULT_MIN_OVERLAP, 6);
+    }
+}
diff --git a/src/routes/mod.rs b/src/routes/mod.rs
index 2bf783e..0461165 100644
--- a/src/routes/mod.rs
+++ b/src/routes/mod.rs
@@ -8,6 +8,7 @@ pub mod baskets;
 pub mod bnw;
 pub mod bulklmm;
 pub mod correlation;
+pub mod correlation_matrix;
 pub mod datasets;
 pub mod dev;
 pub mod tokens;