diff --git a/bin/monitor-gn4 b/bin/monitor-gn4
index 01f38f8..609fdb8 100755
--- a/bin/monitor-gn4
+++ b/bin/monitor-gn4
@@ -650,6 +650,18 @@ def build_checks(found, tiers):
"method": "pearson", "pca": True},
judge=has_pca,
hint="no external tool: pure Rust + MariaDB"))
+ if found.control_trait:
+ # Clustered ordering: average linkage on 1 - r, for the heatmap.
+ add(Check("compute-correlation-heatmap", "compute", "POST",
+ "/compute/correlation/matrix",
+ "clustered order + tree (pure Rust)",
+ body={"traits": [
+ {"dataset": found.publish, "trait_name": found.trait},
+ {"dataset": found.publish,
+ "trait_name": found.control_trait}],
+ "method": "pearson", "cluster": True},
+ judge=has_clustering,
+ hint="no external tool: pure Rust + MariaDB"))
add(Check("compute-gemma", "compute", "POST", "/compute/gemma",
"gemma-wrapper + gemma + R (R_LIBS_SITE)",
body={"group": g, "lmm": 9, "samples": samples},
@@ -724,6 +736,21 @@ def build_checks(found, tiers):
body=pca_plot_body(found), expect=200,
hint="PCA_PLOTTER and python3",
judge=has_svg))
+ if found.control_trait:
+ add(Check("plot-heatmap", "plots", "POST", "/plot/heatmap",
+ "clustered matrix -> heatmap svg (HEATMAP_PLOTTER)",
+ body=heatmap_plot_body(found), expect=200,
+ hint="HEATMAP_PLOTTER and python3", judge=has_svg))
+ if found.samples:
+ # The Manhattan plot: a GEMMA run's reply, posted back as-is.
+ add(Check("plot-gemma", "plots", "POST", "/plot/gemma",
+ "GEMMA result -> Manhattan svg (GEMMA_PLOTTER)",
+ body=gemma_plot_body(found), expect=200,
+ hint="GEMMA_PLOTTER and python3", judge=has_svg))
+ add(Check("plot-bnw", "plots", "POST", "/plot/bnw",
+ "BNW result -> network svg (BNW_PLOTTER)",
+ body=bnw_plot_body(), expect=200,
+ hint="BNW_PLOTTER and python3", judge=has_svg))
add(Check("plot-genotypes", "plots", "POST", "/plot/genotypes",
"peak-genotypes + values -> svg (GENOTYPE_PLOTTER)",
body=genotype_plot_body(found), expect=200,
@@ -758,6 +785,22 @@ def perturbed(body):
return None
+def has_clustering(reply):
+ """A clustered order that is a permutation, with n-1 merges."""
+ data = (reply.json or {}).get("data") or {}
+ heatmap = data.get("heatmap") or {}
+ order = heatmap.get("order") or []
+ merges = heatmap.get("merges") or []
+ traits = data.get("traits") or []
+ if len(order) != len(traits):
+ return FAIL, "order lists %d, traits %d" % (len(order), len(traits))
+ if sorted(order) != list(range(len(traits))):
+ return FAIL, "order is not a permutation: %s" % order
+ if len(merges) != max(len(traits) - 1, 0):
+ return FAIL, "%d merges for %d traits" % (len(merges), len(traits))
+ return OK, "%d traits, max distance %.3f" % (len(traits), heatmap.get("max_height") or 0.0)
+
+
def has_pca(reply):
"""A PCA block whose eigenvalues sum to the trait count."""
data = (reply.json or {}).get("data") or {}
@@ -816,6 +859,40 @@ def has_svg(reply):
return FAIL, "not an SVG: " + str(body)[:80]
+def _post_result(client, path, body, what):
+ reply = client.call("POST", path, body, timeout=600)
+ if reply.status != 200:
+ code, detail = reply.fields()
+ raise Skip("could not produce a %s to plot: %s %s"
+ % (what, code or reply.status, str(detail)[:80]))
+ return reply.json or {}
+
+
+def heatmap_plot_body(found):
+ def build(client):
+ return _post_result(client, "/compute/correlation/matrix",
+ {"traits": [
+ {"dataset": found.publish, "trait_name": found.trait},
+ {"dataset": found.publish,
+ "trait_name": found.control_trait}],
+ "method": "pearson", "cluster": True}, "clustered matrix")
+ return build
+
+
+def gemma_plot_body(found):
+ def build(client):
+ return _post_result(client, "/compute/gemma",
+ {"group": found.group, "lmm": 9, "samples": found.samples},
+ "GEMMA result")
+ return build
+
+
+def bnw_plot_body():
+ def build(client):
+ return _post_result(client, "/compute/bnw", {"values": bnw_matrix()}, "BNW result")
+ return build
+
+
def pca_plot_body(found):
"""Ask the matrix for a PCA, then post that reply to the plotter."""
def build(client):
diff --git a/scripts/plot-heatmap.py b/scripts/plot-heatmap.py
new file mode 100644
index 0000000..791e3d3
--- /dev/null
+++ b/scripts/plot-heatmap.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Clustered heatmap of a GeneNetwork4 correlation matrix, as one SVG panel.
+
+Input is the JSON response of POST /compute/correlation/matrix with
+`"cluster": true` (saved as-is). The cells are the r matrix drawn in the
+clustered order, with the dendrogram that produced that order alongside, so a
+reader can see *why* two traits are neighbours rather than trusting the order.
+
+ python3 plot-heatmap.py matrix.json -o heatmap.svg --trait collection-name
+
+Standard library only; writes plain SVG (light/dark aware).
+"""
+import argparse
+import json
+import sys
+
+W, H = 880, 720
+LEFT_TREE = 150 # dendrogram width
+TOP, BOTTOM = 78, 116 # header, legend
+RIGHT_LABELS = 250 # trait names on the right
+
+
+def heatmap_of(doc):
+ data = doc.get("data") if isinstance(doc, dict) else None
+ for candidate in (data, doc):
+ if isinstance(candidate, dict) and isinstance(candidate.get("heatmap"), dict):
+ return candidate["heatmap"], candidate
+ return None, data if isinstance(data, dict) else {}
+
+
+def colour(value):
+ """Diverging scale for r in [-1, 1]: blue negative, red positive."""
+ v = max(-1.0, min(1.0, value))
+ if v >= 0:
+ # white -> red
+ r, g, b = 250, int(250 - 150 * v), int(250 - 170 * v)
+ else:
+ # white -> blue
+ t = -v
+ r, g, b = int(250 - 160 * t), int(250 - 100 * t), 250
+ return f"#{r:02x}{g:02x}{b:02x}"
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+ ap.add_argument("result", help="saved /compute/correlation/matrix response (JSON), cluster: true")
+ ap.add_argument("-o", "--out", default="heatmap.svg")
+ ap.add_argument("--trait", default="", help="collection label for the title")
+ ap.add_argument("--title", default=None)
+ a = ap.parse_args()
+
+ doc = json.load(open(a.result))
+ clustering, data = heatmap_of(doc)
+ if not clustering or not clustering.get("order"):
+ sys.exit(f"{a.result}: no heatmap.order -- post the matrix reply with \"cluster\": true")
+ traits = data.get("traits") or []
+ r = data.get("r") or []
+ order = clustering["order"]
+ n = len(order)
+ if n < 2:
+ sys.exit("a heatmap needs at least two traits")
+ names = [t.get("trait_name", "?") for t in traits]
+ datasets = [t.get("dataset", "") for t in traits]
+
+ # Cell size: keep the whole matrix in the panel, labels only if they fit.
+ size = min((H - TOP - BOTTOM) / n, (W - LEFT_TREE - RIGHT_LABELS) / n)
+ size = max(size, 1.0)
+ grid = size * n
+ x0 = LEFT_TREE
+ y0 = TOP
+
+ label = a.trait or "correlation matrix"
+ title = a.title or f"{label}: clustered correlation heatmap"
+ values = [r[i][j] for i in order for j in order if r[i][j] is not None]
+ sub = (f"{n} trait(s), average linkage on 1 - r ({len(clustering.get('merges') or [])} merges, "
+ f"max distance {clustering.get('max_height', 0):.3f})"
+ + (f" · {len(values)} of {n * n} cells shown" if len(values) != n * n else ""))
+ desc = (f"Clustered heatmap of {n} traits. Cell colour is the correlation coefficient, "
+ f"blue negative and red positive, in the order the average-linkage tree produced.")
+
+ s = [f'''<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {W} {H}" width="{W}" height="{H}" role="img" aria-labelledby="t d" font-family="system-ui,-apple-system,Segoe UI,sans-serif">
+<title id="t">{title}</title>
+<desc id="d">{desc}</desc>
+<style>
+:root{{--surface:#fcfcfb;--ink:#0b0b0b;--ink2:#52514e;--grid:#e6e5e1;--tree:#52514e;--na:#dedcd6}}
+@media (prefers-color-scheme:dark){{:root{{--surface:#1a1a19;--ink:#f0efec;--ink2:#c3c2b7;--grid:#333330;--tree:#a9a89f;--na:#2a2a28}}}}
+text{{fill:var(--ink2);font-size:12px}} .ttl{{fill:var(--ink);font-size:16px;font-weight:600}}
+</style>
+<rect width="{W}" height="{H}" fill="var(--surface)"/>
+<text class="ttl" x="{x0}" y="26">{title}</text>
+<text x="{x0}" y="46">{sub}</text>''']
+
+ # Cells.
+ for row, i in enumerate(order):
+ for column, j in enumerate(order):
+ value = r[i][j]
+ x = x0 + column * size
+ y = y0 + row * size
+ if value is None:
+ fill = "var(--na)"
+ label = "no correlation"
+ else:
+ fill = colour(value)
+ label = f"r = {value:+.3f}"
+ s.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{size + 0.4:.1f}" height="{size + 0.4:.1f}" '
+ f'fill="{fill}"><title>{names[i]} × {names[j]}: {label}</title></rect>')
+
+ # Dendrogram on the left: leaves in the same order, heights from the tree.
+ leaves = {trait_index: position for position, trait_index in enumerate(order)}
+ nodes = {}
+ max_height = max(clustering.get("max_height") or 0.0, 1e-12)
+ x_max = LEFT_TREE - 18
+ x_min = 14
+
+ def pos_x(height):
+ return x_max - (height / max_height) * (x_max - x_min)
+
+ for index, trait_index in enumerate(order):
+ nodes[trait_index] = (x_max, y0 + (leaves[trait_index] + 0.5) * size)
+
+ for merge_index, merge in enumerate(clustering.get("merges") or []):
+ left, right = merge["left"], merge["right"]
+ if left not in nodes or right not in nodes:
+ continue
+ (lx, ly), (rx, ry) = nodes[left], nodes[right]
+ x = pos_x(merge["height"])
+ s.append(f'<path d="M{lx:.1f} {ly:.1f} H{x:.1f} V{ry:.1f} H{rx:.1f}" fill="none" '
+ f'stroke="var(--tree)" stroke-width="1"/>')
+ nodes[n + merge_index] = (x, (ly + ry) / 2)
+
+ # Labels: trait names on the right, dataset only when it differs.
+ label_font = 11 if size < 14 else 12
+ for row, i in enumerate(order):
+ y = y0 + (row + 0.5) * size + 3.5
+ text = names[i]
+ s.append(f'<text x="{x0 + grid + 8:.1f}" y="{y:.1f}" font-size="{label_font}">{text}</text>')
+ if datasets[i]:
+ s.append(f'<text x="{x0 + grid + 8 + 92:.1f}" y="{y:.1f}" font-size="{label_font - 1}" '
+ f'opacity="0.7">{datasets[i]}</text>')
+
+ # Legend: the diverging scale, drawn as a strip.
+ lx, ly, lw, lh = x0, H - 74, 220, 12
+ steps = 40
+ for step in range(steps):
+ value = -1.0 + 2.0 * step / (steps - 1)
+ s.append(f'<rect x="{lx + step * lw / steps:.1f}" y="{ly}" width="{lw / steps + 0.5:.1f}" '
+ f'height="{lh}" fill="{colour(value)}"/>')
+ s.append(f'<text x="{lx}" y="{ly + lh + 14}">-1</text>')
+ s.append(f'<text x="{lx + lw / 2:.1f}" y="{ly + lh + 14}" text-anchor="middle">0</text>')
+ s.append(f'<text x="{lx + lw:.1f}" y="{ly + lh + 14}" text-anchor="end">+1</text>')
+ s.append(f'<text x="{lx + lw + 14}" y="{ly + 10}">correlation coefficient r</text>')
+ s.append(f'<text x="{lx}" y="{H - 18}">clustered order, both axes: average linkage on 1 - r</text>')
+ s.append("</svg>")
+ open(a.out, "w").write("\n".join(s) + "\n")
+ print(f"{a.out}: {n}x{n} heatmap, {len(clustering.get('merges') or [])} merges, "
+ f"cell {size:.1f}px")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/config.rs b/src/config.rs
index a4a078c..c6a1553 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -173,6 +173,8 @@ pub struct Config {
pub bnw_plotter: String,
/// Script that draws the scree plot of a correlation-matrix PCA (PCA_PLOTTER).
pub pca_plotter: String,
+ /// Script that draws the clustered heatmap of a correlation matrix (HEATMAP_PLOTTER).
+ pub heatmap_plotter: String,
/// Directory holding BNW's worked example datasets (`BNW_EXAMPLES_DIR`),
/// served by `/compute/bnw/examples`. Empty means the endpoint reports
/// that no examples are configured.
@@ -413,6 +415,10 @@ impl Config {
"PCA_PLOTTER".to_string(),
"scripts/plot-pca.py".to_string(),
);
+ values.insert(
+ "HEATMAP_PLOTTER".to_string(),
+ "scripts/plot-heatmap.py".to_string(),
+ );
// Per-user baskets: one SQLite file each (src/baskets.rs). Never
// MariaDB -- a user's edited column is theirs.
values.insert(
@@ -480,6 +486,7 @@ impl Config {
"BASKET_DIR",
"GENOTYPE_PLOTTER",
"PCA_PLOTTER",
+ "HEATMAP_PLOTTER",
"BNW_CMD",
"BNW_BIN_DIR",
"BNW_CACHE_DIR",
@@ -546,6 +553,7 @@ impl Config {
bnw_cache_dir: values["BNW_CACHE_DIR"].clone(),
bnw_plotter: values["BNW_PLOTTER"].clone(),
pca_plotter: values["PCA_PLOTTER"].clone(),
+ heatmap_plotter: values["HEATMAP_PLOTTER"].clone(),
bnw_examples_dir: values["BNW_EXAMPLES_DIR"].clone(),
bulklmm_cmd: values["BULKLMM_CMD"].clone(),
bulklmm_env: values["BULKLMM_ENV"].clone(),
diff --git a/src/heatmap.rs b/src/heatmap.rs
new file mode 100644
index 0000000..f179980
--- /dev/null
+++ b/src/heatmap.rs
@@ -0,0 +1,214 @@
+//! Clustered ordering of a correlation matrix, for a heatmap.
+//!
+//! The matrix is symmetric, so one ordering serves both axes: cluster the
+//! traits, then draw the r matrix in that order and a neighbour is a trait that
+//! actually looks like its neighbour.
+//!
+//! Decisions, all stated because a clustering nobody can reproduce is worse
+//! than no clustering:
+//!
+//! * **Distance is `1 - r`**, so a pair at r = 1 sits at 0 and a pair at
+//! r = -1 at 2. That is the usual choice for correlation matrices and it
+//! keeps anti-correlated traits far apart, which is what a reader expects
+//! from a heatmap of *correlations*.
+//! * **Average linkage (UPGMA).** Single linkage chains along a gradient and
+//! complete linkage makes one huge flat cluster from anything with noise in
+//! it; average is the compromise that behaves on real expression traits. It
+//! is also what GN1's heatmaps used (R's `hclust(method="average")`).
+//! * **Deterministic ties**: when two merges have the same height the lower
+//! index wins, and the leaf order comes out of the tree by always placing the
+//! earlier-clustered side first. Two runs on the same matrix give the same
+//! order, which is what makes the plot diffable.
+//!
+//! The algorithm is the textbook O(n^3) one: n is at most `MAX_PCA_TRAITS`, and
+//! clarity beats a faster distance-update scheme that has to be trusted.
+
+/// One merge in the tree, enough to draw a dendrogram.
+#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)]
+pub struct Merge {
+ /// Cluster ids 0..n-1 are the traits themselves; n and up are merges, in
+ /// the order they appear in this list.
+ pub left: usize,
+ pub right: usize,
+ /// Distance at which the two were joined (`1 - r`, so 0 is identical).
+ pub height: f64,
+}
+
+/// The clustered order and the tree that produced it.
+#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)]
+pub struct Clustering {
+ /// Trait indices in drawing order (row and column order both).
+ pub order: Vec<usize>,
+ pub merges: Vec<Merge>,
+ /// Largest distance seen, so a plotter can scale the dendrogram.
+ pub max_height: f64,
+}
+
+/// Average-linkage clustering of a correlation matrix.
+///
+/// Returns `None` for a matrix that is not square, is larger than the cap, or
+/// has a cell that is not a number -- a gap means the two traits' distance is
+/// unknown, and quietly treating it as 1 would invent a structure.
+pub fn cluster(r: &[Vec<Option<f64>>], cap: usize) -> Option<Clustering> {
+ let n = r.len();
+ if n < 2 || n > cap || r.iter().any(|row| row.len() != n) {
+ return None;
+ }
+ let mut distance = vec![vec![0.0; n]; n];
+ for i in 0..n {
+ for j in 0..n {
+ let value = r[i][j]?;
+ if !value.is_finite() {
+ return None;
+ }
+ distance[i][j] = 1.0 - value;
+ }
+ }
+
+ // Active clusters: members and the merge tree.
+ let mut members: Vec<Vec<usize>> = (0..n).map(|index| vec![index]).collect();
+ let mut active: Vec<bool> = vec![true; n];
+ let mut sizes: Vec<usize> = vec![1; n];
+ let mut dist = distance;
+ let mut merges: Vec<Merge> = Vec::new();
+ let mut max_height = 0.0_f64;
+
+ for _ in 1..n {
+ // Nearest pair of active clusters, lowest index pair on a tie.
+ let mut best = (f64::INFINITY, 0usize, 0usize);
+ for i in 0..dist.len() {
+ if !active[i] {
+ continue;
+ }
+ for j in (i + 1)..dist.len() {
+ if !active[j] {
+ continue;
+ }
+ let d = dist[i][j];
+ if d < best.0 {
+ best = (d, i, j);
+ }
+ }
+ }
+ let (height, left, right) = best;
+ if !height.is_finite() {
+ return None;
+ }
+ max_height = max_height.max(height);
+ // Ids in this array ARE the tree ids: traits are 0..n-1 and merge m is
+ // n+m, created in this order, so the pair can be recorded as it stands.
+ let new_id = dist.len();
+ merges.push(Merge { left, right, height });
+
+ // Average linkage: the new distance is the size-weighted mean.
+ let mut row = vec![0.0; new_id + 1];
+ for k in 0..new_id {
+ if !active[k] || k == left || k == right {
+ continue;
+ }
+ row[k] = (dist[left][k] * sizes[left] as f64 + dist[right][k] * sizes[right] as f64)
+ / (sizes[left] + sizes[right]) as f64;
+ }
+ // Grow the matrix to new_id + 1 rows before writing into the new row:
+ // pushing the row first is what makes dist[new_id] exist.
+ for existing in dist.iter_mut() {
+ existing.push(0.0);
+ }
+ dist.push(row.clone());
+ for k in 0..new_id {
+ dist[new_id][k] = row[k];
+ dist[k][new_id] = row[k];
+ }
+
+ active[left] = false;
+ active[right] = false;
+ active.push(true);
+ sizes.push(sizes[left] + sizes[right]);
+ members.push(Vec::new());
+ let mut merged = members[left].clone();
+ merged.extend(members[right].iter().copied());
+ members[new_id] = merged;
+ }
+
+ // Leaf order: walk the tree, earlier-clustered side first (deterministic).
+ let mut order = Vec::with_capacity(n);
+ fn walk(id: usize, n: usize, merges: &[Merge], order: &mut Vec<usize>) {
+ if id < n {
+ order.push(id);
+ return;
+ }
+ let index = id - n;
+ let merge = &merges[index];
+ walk(merge.left, n, merges, order);
+ walk(merge.right, n, merges, order);
+ }
+ walk(dist.len() - 1, n, &merges, &mut order);
+ if order.len() != n {
+ return None;
+ }
+ Some(Clustering {
+ order,
+ merges,
+ max_height,
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn matrix(values: &[&[f64]]) -> Vec<Vec<Option<f64>>> {
+ values
+ .iter()
+ .map(|row| row.iter().map(|v| Some(*v)).collect())
+ .collect()
+ }
+
+ #[test]
+ fn two_blocks_are_ordered_together() {
+ // 0 and 1 correlate strongly with each other and with nothing else;
+ // same for 2 and 3. A good ordering keeps each pair adjacent.
+ let r = matrix(&[
+ &[1.0, 0.9, 0.0, 0.1],
+ &[0.9, 1.0, 0.1, 0.0],
+ &[0.0, 0.1, 1.0, 0.85],
+ &[0.1, 0.0, 0.85, 1.0],
+ ]);
+ let clustering = cluster(&r, 300).expect("4x4");
+ let order = clustering.order;
+ let position = |trait_index: usize| order.iter().position(|x| *x == trait_index).unwrap();
+ assert_eq!(position(0).abs_diff(position(1)), 1, "0 and 1 adjacent: {order:?}");
+ assert_eq!(position(2).abs_diff(position(3)), 1, "2 and 3 adjacent: {order:?}");
+ assert_eq!(order.len(), 4);
+ assert_eq!(clustering.merges.len(), 3);
+ // The first merges are the within-block ones, so they are the closest.
+ assert!(clustering.merges[0].height < 0.2);
+ assert!(clustering.merges[2].height >= clustering.merges[0].height);
+ }
+
+ #[test]
+ fn the_order_is_reproducible() {
+ let r = matrix(&[
+ &[1.0, 0.4, -0.2],
+ &[0.4, 1.0, 0.1],
+ &[-0.2, 0.1, 1.0],
+ ]);
+ let first = cluster(&r, 300).expect("3x3");
+ let second = cluster(&r, 300).expect("3x3");
+ assert_eq!(first.order, second.order);
+ assert_eq!(first.merges.len(), second.merges.len());
+ }
+
+ #[test]
+ fn a_gap_refuses_rather_than_invents() {
+ let mut r = matrix(&[&[1.0, 0.5], &[0.5, 1.0]]);
+ r[0][1] = None;
+ assert!(cluster(&r, 300).is_none(), "a null cell is not a distance of 1");
+ }
+
+ #[test]
+ fn the_cap_is_honoured() {
+ let r = matrix(&[&[1.0, 0.5, 0.2], &[0.5, 1.0, 0.3], &[0.2, 0.3, 1.0]]);
+ assert!(cluster(&r, 2).is_none());
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 1973652..976b2f1 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -22,6 +22,7 @@ pub mod fitqtl;
pub mod frequencies;
pub mod genoupload;
pub mod genotypes;
+pub mod heatmap;
pub mod normalize;
mod plot;
mod error;
@@ -260,6 +261,7 @@ pub fn app() -> Router {
.routes(routes!(routes::plot::plot_bulklmm))
.routes(routes!(routes::plot::plot_bnw))
.routes(routes!(routes::plot::plot_pca))
+ .routes(routes!(routes::plot::plot_heatmap))
.routes(routes!(routes::phewas::group_phewas))
.routes(routes!(routes::frequencies::group_version_frequencies))
.routes(routes!(routes::frequencies::group_marker_frequencies))
diff --git a/src/routes/correlation_matrix.rs b/src/routes/correlation_matrix.rs
index 2e6c883..9a433b6 100644
--- a/src/routes/correlation_matrix.rs
+++ b/src/routes/correlation_matrix.rs
@@ -65,6 +65,11 @@ pub struct MatrixRequest {
/// Fewest paired observations a cell may have. Defaults to 6.
#[serde(default)]
pub min_overlap: Option<usize>,
+ /// Also return a clustered ordering (and its tree) for a heatmap: average
+ /// linkage on `1 - r`. The matrix is symmetric, so one order serves both
+ /// axes.
+ #[serde(default)]
+ pub cluster: Option<bool>,
/// Also return a PCA of the matrix (eigenvalues, explained variance,
/// loadings over the traits whose every cell is valid). Off by default:
/// the r matrix is worth reading on its own first.
@@ -201,6 +206,16 @@ pub async fn correlation_matrix(
if let Some(pca) = &pca {
body["pca"] = serde_json::to_value(pca).unwrap_or(serde_json::Value::Null);
}
+ let clustering = if wanted.cluster.unwrap_or(false) {
+ // Refused rather than guessed when a cell is missing: an unknown
+ // distance treated as 1 would invent a structure and a plot of it.
+ crate::heatmap::cluster(&matrix.r, crate::pca::MAX_PCA_TRAITS)
+ } else {
+ None
+ };
+ if let Some(clustering) = &clustering {
+ body["heatmap"] = serde_json::to_value(clustering).unwrap_or(serde_json::Value::Null);
+ }
Ok(Json(
ctx.envelope(body)
.with_note(&format!(
@@ -227,6 +242,19 @@ pub async fn correlation_matrix(
"Cells with fewer than min_overlap observations are null in r and p, and their \
n still says how many there were.",
)
+ .with_note(&match (&clustering, wanted.cluster.unwrap_or(false)) {
+ (Some(clustering), _) => format!(
+ "Clustering: average linkage on 1 - r, {} merge(s), largest distance {:.3}; \
+ `heatmap.order` is the drawing order for both axes.",
+ clustering.merges.len(),
+ clustering.max_height
+ ),
+ (None, true) => "No clustering: the matrix has a gap (a null cell), so some \
+ distances are unknown -- drop those traits and ask again."
+ .to_string(),
+ (None, false) => "No clustering: pass \"cluster\": true for a heatmap order."
+ .to_string(),
+ })
.with_note(&match &pca {
Some(pca) if pca.components.is_empty() => format!(
"PCA: not computed -- {} trait(s) were usable and it needs at least 2, and \
@@ -448,6 +476,7 @@ mod tests {
method: None,
corr_samples_group: None,
min_overlap: None,
+ cluster: None,
pca: None,
};
// build() would go to the database; the arithmetic is what is tested
diff --git a/src/routes/plot.rs b/src/routes/plot.rs
index fd56abc..2b4c043 100644
--- a/src/routes/plot.rs
+++ b/src/routes/plot.rs
@@ -817,6 +817,103 @@ pub async fn plot_pca(
Ok(svg_response(rendered))
}
+
+#[utoipa::path(post, path = "/plot/heatmap",
+ summary = "Draw the clustered heatmap of a correlation matrix, as an SVG",
+ description = "Post the response of POST /compute/correlation/matrix back (as-is) after \
+asking it for `\"cluster\": true`, and get the figure scripts/plot-heatmap.py draws: the r \
+matrix in the average-linkage order, one dendrogram showing why those traits are neighbours, \
+and a diverging colour scale with blue negative and red positive. Nothing is recomputed — the \
+order comes from the reply. image/svg+xml, not the JSON envelope; `x-plot-summary` carries the \
+plotter's one-line summary.",
+ params(PlotParams),
+ request_body = serde_json::Value,
+ responses(
+ (status = 200, description = "The SVG", body = String, content_type = "image/svg+xml"),
+ Unauthorized,
+ (status = 422, description = "The posted body carries no clustered order", body = ApiError),
+ (status = 502, description = "The plotter failed", body = ApiError)
+ )
+)]
+#[gnauth(token_only)]
+pub async fn plot_heatmap(
+ ctx: Ctx,
+ _token: AuthToken,
+ Query(plot_params): Query<PlotParams>,
+ Json(posted): Json<serde_json::Value>,
+) -> Result<Response, ApiError> {
+ let rid = ctx.request_info();
+ if let Err(detail) = check_heatmap_result(&posted) {
+ return Err(ApiError::new(
+ StatusCode::UNPROCESSABLE_ENTITY,
+ "not_a_heatmap_result",
+ "Not a clustered matrix",
+ detail,
+ "plot_heatmap",
+ )
+ .with_request_id(rid));
+ }
+ let config = Config::load();
+ let mut extra: Vec<String> = Vec::new();
+ if let Some(title) = plot_params.title.as_deref() {
+ extra.push("--title".to_string());
+ extra.push(title.to_string());
+ }
+ let borrowed: Vec<&str> = extra.iter().map(String::as_str).collect();
+ let rendered = plot::render(
+ &config.plot_python,
+ &config.heatmap_plotter,
+ std::path::Path::new(&config.gemma_cache_dir),
+ &posted,
+ plot_params
+ .trait_label
+ .as_deref()
+ .or(plot_params.title.as_deref())
+ .unwrap_or(""),
+ &borrowed,
+ )
+ .await
+ .map_err(|error| {
+ ApiError::new(
+ StatusCode::BAD_GATEWAY,
+ "plot_failed",
+ "The plotter failed",
+ error.to_string(),
+ "plot_heatmap",
+ )
+ .with_request_id(rid)
+ })?;
+ Ok(svg_response(rendered))
+}
+
+/// The posted body must carry `heatmap.order` from the matrix endpoint.
+fn check_heatmap_result(result: &serde_json::Value) -> Result<(), String> {
+ let data = result.get("data").unwrap_or(result);
+ let heatmap = data
+ .get("heatmap")
+ .ok_or("the posted body has no `heatmap` object: ask /compute/correlation/matrix for \
+ \"cluster\": true first")?;
+ let order = heatmap
+ .get("order")
+ .and_then(|order| order.as_array())
+ .ok_or("`heatmap.order` is missing")?;
+ if order.len() < 2 {
+ return Err("a heatmap needs at least two traits".to_string());
+ }
+ let traits = data
+ .get("traits")
+ .and_then(|traits| traits.as_array())
+ .ok_or("the posted body has no `traits` array")?;
+ if traits.len() != order.len() {
+ return Err(format!(
+ "`heatmap.order` lists {} trait(s) but `traits` has {}",
+ order.len(),
+ traits.len()
+ ));
+ }
+ Ok(())
+}
+
/// The posted body must carry `pca.components`, from the matrix endpoint.
fn check_pca_result(result: &serde_json::Value) -> Result<(), String> {
let pca = result
|