diff options
| -rwxr-xr-x | bin/monitor-gn4 | 77 | ||||
| -rw-r--r-- | scripts/plot-heatmap.py | 160 | ||||
| -rw-r--r-- | src/config.rs | 8 | ||||
| -rw-r--r-- | src/heatmap.rs | 214 | ||||
| -rw-r--r-- | src/lib.rs | 2 | ||||
| -rw-r--r-- | src/routes/correlation_matrix.rs | 29 | ||||
| -rw-r--r-- | src/routes/plot.rs | 97 |
7 files changed, 587 insertions, 0 deletions
diff --git a/bin/monitor-gn4 b/bin/monitor-gn4 index 01f38f8..609fdb8 100755 --- a/bin/monitor-gn4 +++ b/bin/monitor-gn4 @@ -650,6 +650,18 @@ def build_checks(found, tiers): "method": "pearson", "pca": True}, judge=has_pca, hint="no external tool: pure Rust + MariaDB")) + if found.control_trait: + # Clustered ordering: average linkage on 1 - r, for the heatmap. + add(Check("compute-correlation-heatmap", "compute", "POST", + "/compute/correlation/matrix", + "clustered order + tree (pure Rust)", + body={"traits": [ + {"dataset": found.publish, "trait_name": found.trait}, + {"dataset": found.publish, + "trait_name": found.control_trait}], + "method": "pearson", "cluster": True}, + judge=has_clustering, + hint="no external tool: pure Rust + MariaDB")) add(Check("compute-gemma", "compute", "POST", "/compute/gemma", "gemma-wrapper + gemma + R (R_LIBS_SITE)", body={"group": g, "lmm": 9, "samples": samples}, @@ -724,6 +736,21 @@ def build_checks(found, tiers): body=pca_plot_body(found), expect=200, hint="PCA_PLOTTER and python3", judge=has_svg)) + if found.control_trait: + add(Check("plot-heatmap", "plots", "POST", "/plot/heatmap", + "clustered matrix -> heatmap svg (HEATMAP_PLOTTER)", + body=heatmap_plot_body(found), expect=200, + hint="HEATMAP_PLOTTER and python3", judge=has_svg)) + if found.samples: + # The Manhattan plot: a GEMMA run's reply, posted back as-is. + add(Check("plot-gemma", "plots", "POST", "/plot/gemma", + "GEMMA result -> Manhattan svg (GEMMA_PLOTTER)", + body=gemma_plot_body(found), expect=200, + hint="GEMMA_PLOTTER and python3", judge=has_svg)) + add(Check("plot-bnw", "plots", "POST", "/plot/bnw", + "BNW result -> network svg (BNW_PLOTTER)", + body=bnw_plot_body(), expect=200, + hint="BNW_PLOTTER and python3", judge=has_svg)) add(Check("plot-genotypes", "plots", "POST", "/plot/genotypes", "peak-genotypes + values -> svg (GENOTYPE_PLOTTER)", body=genotype_plot_body(found), expect=200, @@ -758,6 +785,22 @@ def perturbed(body): return None +def has_clustering(reply): + """A clustered order that is a permutation, with n-1 merges.""" + data = (reply.json or {}).get("data") or {} + heatmap = data.get("heatmap") or {} + order = heatmap.get("order") or [] + merges = heatmap.get("merges") or [] + traits = data.get("traits") or [] + if len(order) != len(traits): + return FAIL, "order lists %d, traits %d" % (len(order), len(traits)) + if sorted(order) != list(range(len(traits))): + return FAIL, "order is not a permutation: %s" % order + if len(merges) != max(len(traits) - 1, 0): + return FAIL, "%d merges for %d traits" % (len(merges), len(traits)) + return OK, "%d traits, max distance %.3f" % (len(traits), heatmap.get("max_height") or 0.0) + + def has_pca(reply): """A PCA block whose eigenvalues sum to the trait count.""" data = (reply.json or {}).get("data") or {} @@ -816,6 +859,40 @@ def has_svg(reply): return FAIL, "not an SVG: " + str(body)[:80] +def _post_result(client, path, body, what): + reply = client.call("POST", path, body, timeout=600) + if reply.status != 200: + code, detail = reply.fields() + raise Skip("could not produce a %s to plot: %s %s" + % (what, code or reply.status, str(detail)[:80])) + return reply.json or {} + + +def heatmap_plot_body(found): + def build(client): + return _post_result(client, "/compute/correlation/matrix", + {"traits": [ + {"dataset": found.publish, "trait_name": found.trait}, + {"dataset": found.publish, + "trait_name": found.control_trait}], + "method": "pearson", "cluster": True}, "clustered matrix") + return build + + +def gemma_plot_body(found): + def build(client): + return _post_result(client, "/compute/gemma", + {"group": found.group, "lmm": 9, "samples": found.samples}, + "GEMMA result") + return build + + +def bnw_plot_body(): + def build(client): + return _post_result(client, "/compute/bnw", {"values": bnw_matrix()}, "BNW result") + return build + + def pca_plot_body(found): """Ask the matrix for a PCA, then post that reply to the plotter.""" def build(client): diff --git a/scripts/plot-heatmap.py b/scripts/plot-heatmap.py new file mode 100644 index 0000000..791e3d3 --- /dev/null +++ b/scripts/plot-heatmap.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Clustered heatmap of a GeneNetwork4 correlation matrix, as one SVG panel. + +Input is the JSON response of POST /compute/correlation/matrix with +`"cluster": true` (saved as-is). The cells are the r matrix drawn in the +clustered order, with the dendrogram that produced that order alongside, so a +reader can see *why* two traits are neighbours rather than trusting the order. + + python3 plot-heatmap.py matrix.json -o heatmap.svg --trait collection-name + +Standard library only; writes plain SVG (light/dark aware). +""" +import argparse +import json +import sys + +W, H = 880, 720 +LEFT_TREE = 150 # dendrogram width +TOP, BOTTOM = 78, 116 # header, legend +RIGHT_LABELS = 250 # trait names on the right + + +def heatmap_of(doc): + data = doc.get("data") if isinstance(doc, dict) else None + for candidate in (data, doc): + if isinstance(candidate, dict) and isinstance(candidate.get("heatmap"), dict): + return candidate["heatmap"], candidate + return None, data if isinstance(data, dict) else {} + + +def colour(value): + """Diverging scale for r in [-1, 1]: blue negative, red positive.""" + v = max(-1.0, min(1.0, value)) + if v >= 0: + # white -> red + r, g, b = 250, int(250 - 150 * v), int(250 - 170 * v) + else: + # white -> blue + t = -v + r, g, b = int(250 - 160 * t), int(250 - 100 * t), 250 + return f"#{r:02x}{g:02x}{b:02x}" + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("result", help="saved /compute/correlation/matrix response (JSON), cluster: true") + ap.add_argument("-o", "--out", default="heatmap.svg") + ap.add_argument("--trait", default="", help="collection label for the title") + ap.add_argument("--title", default=None) + a = ap.parse_args() + + doc = json.load(open(a.result)) + clustering, data = heatmap_of(doc) + if not clustering or not clustering.get("order"): + sys.exit(f"{a.result}: no heatmap.order -- post the matrix reply with \"cluster\": true") + traits = data.get("traits") or [] + r = data.get("r") or [] + order = clustering["order"] + n = len(order) + if n < 2: + sys.exit("a heatmap needs at least two traits") + names = [t.get("trait_name", "?") for t in traits] + datasets = [t.get("dataset", "") for t in traits] + + # Cell size: keep the whole matrix in the panel, labels only if they fit. + size = min((H - TOP - BOTTOM) / n, (W - LEFT_TREE - RIGHT_LABELS) / n) + size = max(size, 1.0) + grid = size * n + x0 = LEFT_TREE + y0 = TOP + + label = a.trait or "correlation matrix" + title = a.title or f"{label}: clustered correlation heatmap" + values = [r[i][j] for i in order for j in order if r[i][j] is not None] + sub = (f"{n} trait(s), average linkage on 1 - r ({len(clustering.get('merges') or [])} merges, " + f"max distance {clustering.get('max_height', 0):.3f})" + + (f" · {len(values)} of {n * n} cells shown" if len(values) != n * n else "")) + desc = (f"Clustered heatmap of {n} traits. Cell colour is the correlation coefficient, " + f"blue negative and red positive, in the order the average-linkage tree produced.") + + s = [f'''<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {W} {H}" width="{W}" height="{H}" role="img" aria-labelledby="t d" font-family="system-ui,-apple-system,Segoe UI,sans-serif"> +<title id="t">{title}</title> +<desc id="d">{desc}</desc> +<style> +:root{{--surface:#fcfcfb;--ink:#0b0b0b;--ink2:#52514e;--grid:#e6e5e1;--tree:#52514e;--na:#dedcd6}} +@media (prefers-color-scheme:dark){{:root{{--surface:#1a1a19;--ink:#f0efec;--ink2:#c3c2b7;--grid:#333330;--tree:#a9a89f;--na:#2a2a28}}}} +text{{fill:var(--ink2);font-size:12px}} .ttl{{fill:var(--ink);font-size:16px;font-weight:600}} +</style> +<rect width="{W}" height="{H}" fill="var(--surface)"/> +<text class="ttl" x="{x0}" y="26">{title}</text> +<text x="{x0}" y="46">{sub}</text>'''] + + # Cells. + for row, i in enumerate(order): + for column, j in enumerate(order): + value = r[i][j] + x = x0 + column * size + y = y0 + row * size + if value is None: + fill = "var(--na)" + label = "no correlation" + else: + fill = colour(value) + label = f"r = {value:+.3f}" + s.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{size + 0.4:.1f}" height="{size + 0.4:.1f}" ' + f'fill="{fill}"><title>{names[i]} × {names[j]}: {label}</title></rect>') + + # Dendrogram on the left: leaves in the same order, heights from the tree. + leaves = {trait_index: position for position, trait_index in enumerate(order)} + nodes = {} + max_height = max(clustering.get("max_height") or 0.0, 1e-12) + x_max = LEFT_TREE - 18 + x_min = 14 + + def pos_x(height): + return x_max - (height / max_height) * (x_max - x_min) + + for index, trait_index in enumerate(order): + nodes[trait_index] = (x_max, y0 + (leaves[trait_index] + 0.5) * size) + + for merge_index, merge in enumerate(clustering.get("merges") or []): + left, right = merge["left"], merge["right"] + if left not in nodes or right not in nodes: + continue + (lx, ly), (rx, ry) = nodes[left], nodes[right] + x = pos_x(merge["height"]) + s.append(f'<path d="M{lx:.1f} {ly:.1f} H{x:.1f} V{ry:.1f} H{rx:.1f}" fill="none" ' + f'stroke="var(--tree)" stroke-width="1"/>') + nodes[n + merge_index] = (x, (ly + ry) / 2) + + # Labels: trait names on the right, dataset only when it differs. + label_font = 11 if size < 14 else 12 + for row, i in enumerate(order): + y = y0 + (row + 0.5) * size + 3.5 + text = names[i] + s.append(f'<text x="{x0 + grid + 8:.1f}" y="{y:.1f}" font-size="{label_font}">{text}</text>') + if datasets[i]: + s.append(f'<text x="{x0 + grid + 8 + 92:.1f}" y="{y:.1f}" font-size="{label_font - 1}" ' + f'opacity="0.7">{datasets[i]}</text>') + + # Legend: the diverging scale, drawn as a strip. + lx, ly, lw, lh = x0, H - 74, 220, 12 + steps = 40 + for step in range(steps): + value = -1.0 + 2.0 * step / (steps - 1) + s.append(f'<rect x="{lx + step * lw / steps:.1f}" y="{ly}" width="{lw / steps + 0.5:.1f}" ' + f'height="{lh}" fill="{colour(value)}"/>') + s.append(f'<text x="{lx}" y="{ly + lh + 14}">-1</text>') + s.append(f'<text x="{lx + lw / 2:.1f}" y="{ly + lh + 14}" text-anchor="middle">0</text>') + s.append(f'<text x="{lx + lw:.1f}" y="{ly + lh + 14}" text-anchor="end">+1</text>') + s.append(f'<text x="{lx + lw + 14}" y="{ly + 10}">correlation coefficient r</text>') + s.append(f'<text x="{lx}" y="{H - 18}">clustered order, both axes: average linkage on 1 - r</text>') + s.append("</svg>") + open(a.out, "w").write("\n".join(s) + "\n") + print(f"{a.out}: {n}x{n} heatmap, {len(clustering.get('merges') or [])} merges, " + f"cell {size:.1f}px") + + +if __name__ == "__main__": + main() diff --git a/src/config.rs b/src/config.rs index a4a078c..c6a1553 100644 --- a/src/config.rs +++ b/src/config.rs @@ -173,6 +173,8 @@ pub struct Config { pub bnw_plotter: String, /// Script that draws the scree plot of a correlation-matrix PCA (PCA_PLOTTER). pub pca_plotter: String, + /// Script that draws the clustered heatmap of a correlation matrix (HEATMAP_PLOTTER). + pub heatmap_plotter: String, /// Directory holding BNW's worked example datasets (`BNW_EXAMPLES_DIR`), /// served by `/compute/bnw/examples`. Empty means the endpoint reports /// that no examples are configured. @@ -413,6 +415,10 @@ impl Config { "PCA_PLOTTER".to_string(), "scripts/plot-pca.py".to_string(), ); + values.insert( + "HEATMAP_PLOTTER".to_string(), + "scripts/plot-heatmap.py".to_string(), + ); // Per-user baskets: one SQLite file each (src/baskets.rs). Never // MariaDB -- a user's edited column is theirs. values.insert( @@ -480,6 +486,7 @@ impl Config { "BASKET_DIR", "GENOTYPE_PLOTTER", "PCA_PLOTTER", + "HEATMAP_PLOTTER", "BNW_CMD", "BNW_BIN_DIR", "BNW_CACHE_DIR", @@ -546,6 +553,7 @@ impl Config { bnw_cache_dir: values["BNW_CACHE_DIR"].clone(), bnw_plotter: values["BNW_PLOTTER"].clone(), pca_plotter: values["PCA_PLOTTER"].clone(), + heatmap_plotter: values["HEATMAP_PLOTTER"].clone(), bnw_examples_dir: values["BNW_EXAMPLES_DIR"].clone(), bulklmm_cmd: values["BULKLMM_CMD"].clone(), bulklmm_env: values["BULKLMM_ENV"].clone(), diff --git a/src/heatmap.rs b/src/heatmap.rs new file mode 100644 index 0000000..f179980 --- /dev/null +++ b/src/heatmap.rs @@ -0,0 +1,214 @@ +//! Clustered ordering of a correlation matrix, for a heatmap. +//! +//! The matrix is symmetric, so one ordering serves both axes: cluster the +//! traits, then draw the r matrix in that order and a neighbour is a trait that +//! actually looks like its neighbour. +//! +//! Decisions, all stated because a clustering nobody can reproduce is worse +//! than no clustering: +//! +//! * **Distance is `1 - r`**, so a pair at r = 1 sits at 0 and a pair at +//! r = -1 at 2. That is the usual choice for correlation matrices and it +//! keeps anti-correlated traits far apart, which is what a reader expects +//! from a heatmap of *correlations*. +//! * **Average linkage (UPGMA).** Single linkage chains along a gradient and +//! complete linkage makes one huge flat cluster from anything with noise in +//! it; average is the compromise that behaves on real expression traits. It +//! is also what GN1's heatmaps used (R's `hclust(method="average")`). +//! * **Deterministic ties**: when two merges have the same height the lower +//! index wins, and the leaf order comes out of the tree by always placing the +//! earlier-clustered side first. Two runs on the same matrix give the same +//! order, which is what makes the plot diffable. +//! +//! The algorithm is the textbook O(n^3) one: n is at most `MAX_PCA_TRAITS`, and +//! clarity beats a faster distance-update scheme that has to be trusted. + +/// One merge in the tree, enough to draw a dendrogram. +#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] +pub struct Merge { + /// Cluster ids 0..n-1 are the traits themselves; n and up are merges, in + /// the order they appear in this list. + pub left: usize, + pub right: usize, + /// Distance at which the two were joined (`1 - r`, so 0 is identical). + pub height: f64, +} + +/// The clustered order and the tree that produced it. +#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] +pub struct Clustering { + /// Trait indices in drawing order (row and column order both). + pub order: Vec<usize>, + pub merges: Vec<Merge>, + /// Largest distance seen, so a plotter can scale the dendrogram. + pub max_height: f64, +} + +/// Average-linkage clustering of a correlation matrix. +/// +/// Returns `None` for a matrix that is not square, is larger than the cap, or +/// has a cell that is not a number -- a gap means the two traits' distance is +/// unknown, and quietly treating it as 1 would invent a structure. +pub fn cluster(r: &[Vec<Option<f64>>], cap: usize) -> Option<Clustering> { + let n = r.len(); + if n < 2 || n > cap || r.iter().any(|row| row.len() != n) { + return None; + } + let mut distance = vec![vec![0.0; n]; n]; + for i in 0..n { + for j in 0..n { + let value = r[i][j]?; + if !value.is_finite() { + return None; + } + distance[i][j] = 1.0 - value; + } + } + + // Active clusters: members and the merge tree. + let mut members: Vec<Vec<usize>> = (0..n).map(|index| vec![index]).collect(); + let mut active: Vec<bool> = vec![true; n]; + let mut sizes: Vec<usize> = vec![1; n]; + let mut dist = distance; + let mut merges: Vec<Merge> = Vec::new(); + let mut max_height = 0.0_f64; + + for _ in 1..n { + // Nearest pair of active clusters, lowest index pair on a tie. + let mut best = (f64::INFINITY, 0usize, 0usize); + for i in 0..dist.len() { + if !active[i] { + continue; + } + for j in (i + 1)..dist.len() { + if !active[j] { + continue; + } + let d = dist[i][j]; + if d < best.0 { + best = (d, i, j); + } + } + } + let (height, left, right) = best; + if !height.is_finite() { + return None; + } + max_height = max_height.max(height); + // Ids in this array ARE the tree ids: traits are 0..n-1 and merge m is + // n+m, created in this order, so the pair can be recorded as it stands. + let new_id = dist.len(); + merges.push(Merge { left, right, height }); + + // Average linkage: the new distance is the size-weighted mean. + let mut row = vec![0.0; new_id + 1]; + for k in 0..new_id { + if !active[k] || k == left || k == right { + continue; + } + row[k] = (dist[left][k] * sizes[left] as f64 + dist[right][k] * sizes[right] as f64) + / (sizes[left] + sizes[right]) as f64; + } + // Grow the matrix to new_id + 1 rows before writing into the new row: + // pushing the row first is what makes dist[new_id] exist. + for existing in dist.iter_mut() { + existing.push(0.0); + } + dist.push(row.clone()); + for k in 0..new_id { + dist[new_id][k] = row[k]; + dist[k][new_id] = row[k]; + } + + active[left] = false; + active[right] = false; + active.push(true); + sizes.push(sizes[left] + sizes[right]); + members.push(Vec::new()); + let mut merged = members[left].clone(); + merged.extend(members[right].iter().copied()); + members[new_id] = merged; + } + + // Leaf order: walk the tree, earlier-clustered side first (deterministic). + let mut order = Vec::with_capacity(n); + fn walk(id: usize, n: usize, merges: &[Merge], order: &mut Vec<usize>) { + if id < n { + order.push(id); + return; + } + let index = id - n; + let merge = &merges[index]; + walk(merge.left, n, merges, order); + walk(merge.right, n, merges, order); + } + walk(dist.len() - 1, n, &merges, &mut order); + if order.len() != n { + return None; + } + Some(Clustering { + order, + merges, + max_height, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn matrix(values: &[&[f64]]) -> Vec<Vec<Option<f64>>> { + values + .iter() + .map(|row| row.iter().map(|v| Some(*v)).collect()) + .collect() + } + + #[test] + fn two_blocks_are_ordered_together() { + // 0 and 1 correlate strongly with each other and with nothing else; + // same for 2 and 3. A good ordering keeps each pair adjacent. + let r = matrix(&[ + &[1.0, 0.9, 0.0, 0.1], + &[0.9, 1.0, 0.1, 0.0], + &[0.0, 0.1, 1.0, 0.85], + &[0.1, 0.0, 0.85, 1.0], + ]); + let clustering = cluster(&r, 300).expect("4x4"); + let order = clustering.order; + let position = |trait_index: usize| order.iter().position(|x| *x == trait_index).unwrap(); + assert_eq!(position(0).abs_diff(position(1)), 1, "0 and 1 adjacent: {order:?}"); + assert_eq!(position(2).abs_diff(position(3)), 1, "2 and 3 adjacent: {order:?}"); + assert_eq!(order.len(), 4); + assert_eq!(clustering.merges.len(), 3); + // The first merges are the within-block ones, so they are the closest. + assert!(clustering.merges[0].height < 0.2); + assert!(clustering.merges[2].height >= clustering.merges[0].height); + } + + #[test] + fn the_order_is_reproducible() { + let r = matrix(&[ + &[1.0, 0.4, -0.2], + &[0.4, 1.0, 0.1], + &[-0.2, 0.1, 1.0], + ]); + let first = cluster(&r, 300).expect("3x3"); + let second = cluster(&r, 300).expect("3x3"); + assert_eq!(first.order, second.order); + assert_eq!(first.merges.len(), second.merges.len()); + } + + #[test] + fn a_gap_refuses_rather_than_invents() { + let mut r = matrix(&[&[1.0, 0.5], &[0.5, 1.0]]); + r[0][1] = None; + assert!(cluster(&r, 300).is_none(), "a null cell is not a distance of 1"); + } + + #[test] + fn the_cap_is_honoured() { + let r = matrix(&[&[1.0, 0.5, 0.2], &[0.5, 1.0, 0.3], &[0.2, 0.3, 1.0]]); + assert!(cluster(&r, 2).is_none()); + } +} diff --git a/src/lib.rs b/src/lib.rs index 1973652..976b2f1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,6 +22,7 @@ pub mod fitqtl; pub mod frequencies; pub mod genoupload; pub mod genotypes; +pub mod heatmap; pub mod normalize; mod plot; mod error; @@ -260,6 +261,7 @@ pub fn app() -> Router { .routes(routes!(routes::plot::plot_bulklmm)) .routes(routes!(routes::plot::plot_bnw)) .routes(routes!(routes::plot::plot_pca)) + .routes(routes!(routes::plot::plot_heatmap)) .routes(routes!(routes::phewas::group_phewas)) .routes(routes!(routes::frequencies::group_version_frequencies)) .routes(routes!(routes::frequencies::group_marker_frequencies)) diff --git a/src/routes/correlation_matrix.rs b/src/routes/correlation_matrix.rs index 2e6c883..9a433b6 100644 --- a/src/routes/correlation_matrix.rs +++ b/src/routes/correlation_matrix.rs @@ -65,6 +65,11 @@ pub struct MatrixRequest { /// Fewest paired observations a cell may have. Defaults to 6. #[serde(default)] pub min_overlap: Option<usize>, + /// Also return a clustered ordering (and its tree) for a heatmap: average + /// linkage on `1 - r`. The matrix is symmetric, so one order serves both + /// axes. + #[serde(default)] + pub cluster: Option<bool>, /// Also return a PCA of the matrix (eigenvalues, explained variance, /// loadings over the traits whose every cell is valid). Off by default: /// the r matrix is worth reading on its own first. @@ -201,6 +206,16 @@ pub async fn correlation_matrix( if let Some(pca) = &pca { body["pca"] = serde_json::to_value(pca).unwrap_or(serde_json::Value::Null); } + let clustering = if wanted.cluster.unwrap_or(false) { + // Refused rather than guessed when a cell is missing: an unknown + // distance treated as 1 would invent a structure and a plot of it. + crate::heatmap::cluster(&matrix.r, crate::pca::MAX_PCA_TRAITS) + } else { + None + }; + if let Some(clustering) = &clustering { + body["heatmap"] = serde_json::to_value(clustering).unwrap_or(serde_json::Value::Null); + } Ok(Json( ctx.envelope(body) .with_note(&format!( @@ -227,6 +242,19 @@ pub async fn correlation_matrix( "Cells with fewer than min_overlap observations are null in r and p, and their \ n still says how many there were.", ) + .with_note(&match (&clustering, wanted.cluster.unwrap_or(false)) { + (Some(clustering), _) => format!( + "Clustering: average linkage on 1 - r, {} merge(s), largest distance {:.3}; \ + `heatmap.order` is the drawing order for both axes.", + clustering.merges.len(), + clustering.max_height + ), + (None, true) => "No clustering: the matrix has a gap (a null cell), so some \ + distances are unknown -- drop those traits and ask again." + .to_string(), + (None, false) => "No clustering: pass \"cluster\": true for a heatmap order." + .to_string(), + }) .with_note(&match &pca { Some(pca) if pca.components.is_empty() => format!( "PCA: not computed -- {} trait(s) were usable and it needs at least 2, and \ @@ -448,6 +476,7 @@ mod tests { method: None, corr_samples_group: None, min_overlap: None, + cluster: None, pca: None, }; // build() would go to the database; the arithmetic is what is tested diff --git a/src/routes/plot.rs b/src/routes/plot.rs index fd56abc..2b4c043 100644 --- a/src/routes/plot.rs +++ b/src/routes/plot.rs @@ -817,6 +817,103 @@ pub async fn plot_pca( Ok(svg_response(rendered)) } + +#[utoipa::path(post, path = "/plot/heatmap", + summary = "Draw the clustered heatmap of a correlation matrix, as an SVG", + description = "Post the response of POST /compute/correlation/matrix back (as-is) after \ +asking it for `\"cluster\": true`, and get the figure scripts/plot-heatmap.py draws: the r \ +matrix in the average-linkage order, one dendrogram showing why those traits are neighbours, \ +and a diverging colour scale with blue negative and red positive. Nothing is recomputed — the \ +order comes from the reply. image/svg+xml, not the JSON envelope; `x-plot-summary` carries the \ +plotter's one-line summary.", + params(PlotParams), + request_body = serde_json::Value, + responses( + (status = 200, description = "The SVG", body = String, content_type = "image/svg+xml"), + Unauthorized, + (status = 422, description = "The posted body carries no clustered order", body = ApiError), + (status = 502, description = "The plotter failed", body = ApiError) + ) +)] +#[gnauth(token_only)] +pub async fn plot_heatmap( + ctx: Ctx, + _token: AuthToken, + Query(plot_params): Query<PlotParams>, + Json(posted): Json<serde_json::Value>, +) -> Result<Response, ApiError> { + let rid = ctx.request_info(); + if let Err(detail) = check_heatmap_result(&posted) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "not_a_heatmap_result", + "Not a clustered matrix", + detail, + "plot_heatmap", + ) + .with_request_id(rid)); + } + let config = Config::load(); + let mut extra: Vec<String> = Vec::new(); + if let Some(title) = plot_params.title.as_deref() { + extra.push("--title".to_string()); + extra.push(title.to_string()); + } + let borrowed: Vec<&str> = extra.iter().map(String::as_str).collect(); + let rendered = plot::render( + &config.plot_python, + &config.heatmap_plotter, + std::path::Path::new(&config.gemma_cache_dir), + &posted, + plot_params + .trait_label + .as_deref() + .or(plot_params.title.as_deref()) + .unwrap_or(""), + &borrowed, + ) + .await + .map_err(|error| { + ApiError::new( + StatusCode::BAD_GATEWAY, + "plot_failed", + "The plotter failed", + error.to_string(), + "plot_heatmap", + ) + .with_request_id(rid) + })?; + Ok(svg_response(rendered)) +} + +/// The posted body must carry `heatmap.order` from the matrix endpoint. +fn check_heatmap_result(result: &serde_json::Value) -> Result<(), String> { + let data = result.get("data").unwrap_or(result); + let heatmap = data + .get("heatmap") + .ok_or("the posted body has no `heatmap` object: ask /compute/correlation/matrix for \ + \"cluster\": true first")?; + let order = heatmap + .get("order") + .and_then(|order| order.as_array()) + .ok_or("`heatmap.order` is missing")?; + if order.len() < 2 { + return Err("a heatmap needs at least two traits".to_string()); + } + let traits = data + .get("traits") + .and_then(|traits| traits.as_array()) + .ok_or("the posted body has no `traits` array")?; + if traits.len() != order.len() { + return Err(format!( + "`heatmap.order` lists {} trait(s) but `traits` has {}", + order.len(), + traits.len() + )); + } + Ok(()) +} + /// The posted body must carry `pca.components`, from the matrix endpoint. fn check_pca_result(result: &serde_json::Value) -> Result<(), String> { let pca = result |
