From ea3dd926211e8db4a0d7d3739a606440b888e8b0 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 16 Nov 2026 17:17:11 +0000 Subject: correlation matrix: clustered heatmap and its plot `"cluster": true` adds `heatmap: {order, merges, max_height}` to the matrix reply and `POST /plot/heatmap` draws it: the r matrix in that order, the dendrogram that produced it, and a diverging scale with blue negative and red positive. A reader can see *why* two traits are neighbours instead of trusting an ordering. Clustering decisions, all recorded because an unrepeatable clustering is worse than none: * distance `1 - r` -- the usual choice for correlations, and it keeps anti-correlated traits apart, which is what a heatmap of correlations should show; * **average linkage (UPGMA)** -- single linkage chains along a gradient and complete linkage makes one flat blob out of noisy traits; average is the compromise GN1's R heatmaps used too; * **deterministic ties and leaf order** (lower index first), so two runs on one matrix produce the same picture and plots stay diffable; * a matrix with a **gap is refused** rather than clustered: an unknown distance treated as 1 would invent a structure, and the reply's note says to drop those traits and ask again. Pairing with the PCA work this is the visual half of a collection: scree plot for how much structure there is, clustered heatmap for where it is. Monitor, as asked: `compute-correlation-heatmap` (order must be a permutation with n-1 merges), `plot-heatmap`, and the plot endpoints that had no check at all -- `plot-gemma` and `plot-bnw`, each posting a real compute result back. Plots tier is now five checks, all ok; full run 47 ok, 0 failed, 1 skipped. --- scripts/plot-heatmap.py | 160 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 scripts/plot-heatmap.py (limited to 'scripts/plot-heatmap.py') diff --git a/scripts/plot-heatmap.py b/scripts/plot-heatmap.py new file mode 100644 index 0000000..791e3d3 --- /dev/null +++ b/scripts/plot-heatmap.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Clustered heatmap of a GeneNetwork4 correlation matrix, as one SVG panel. + +Input is the JSON response of POST /compute/correlation/matrix with +`"cluster": true` (saved as-is). The cells are the r matrix drawn in the +clustered order, with the dendrogram that produced that order alongside, so a +reader can see *why* two traits are neighbours rather than trusting the order. + + python3 plot-heatmap.py matrix.json -o heatmap.svg --trait collection-name + +Standard library only; writes plain SVG (light/dark aware). +""" +import argparse +import json +import sys + +W, H = 880, 720 +LEFT_TREE = 150 # dendrogram width +TOP, BOTTOM = 78, 116 # header, legend +RIGHT_LABELS = 250 # trait names on the right + + +def heatmap_of(doc): + data = doc.get("data") if isinstance(doc, dict) else None + for candidate in (data, doc): + if isinstance(candidate, dict) and isinstance(candidate.get("heatmap"), dict): + return candidate["heatmap"], candidate + return None, data if isinstance(data, dict) else {} + + +def colour(value): + """Diverging scale for r in [-1, 1]: blue negative, red positive.""" + v = max(-1.0, min(1.0, value)) + if v >= 0: + # white -> red + r, g, b = 250, int(250 - 150 * v), int(250 - 170 * v) + else: + # white -> blue + t = -v + r, g, b = int(250 - 160 * t), int(250 - 100 * t), 250 + return f"#{r:02x}{g:02x}{b:02x}" + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("result", help="saved /compute/correlation/matrix response (JSON), cluster: true") + ap.add_argument("-o", "--out", default="heatmap.svg") + ap.add_argument("--trait", default="", help="collection label for the title") + ap.add_argument("--title", default=None) + a = ap.parse_args() + + doc = json.load(open(a.result)) + clustering, data = heatmap_of(doc) + if not clustering or not clustering.get("order"): + sys.exit(f"{a.result}: no heatmap.order -- post the matrix reply with \"cluster\": true") + traits = data.get("traits") or [] + r = data.get("r") or [] + order = clustering["order"] + n = len(order) + if n < 2: + sys.exit("a heatmap needs at least two traits") + names = [t.get("trait_name", "?") for t in traits] + datasets = [t.get("dataset", "") for t in traits] + + # Cell size: keep the whole matrix in the panel, labels only if they fit. + size = min((H - TOP - BOTTOM) / n, (W - LEFT_TREE - RIGHT_LABELS) / n) + size = max(size, 1.0) + grid = size * n + x0 = LEFT_TREE + y0 = TOP + + label = a.trait or "correlation matrix" + title = a.title or f"{label}: clustered correlation heatmap" + values = [r[i][j] for i in order for j in order if r[i][j] is not None] + sub = (f"{n} trait(s), average linkage on 1 - r ({len(clustering.get('merges') or [])} merges, " + f"max distance {clustering.get('max_height', 0):.3f})" + + (f" · {len(values)} of {n * n} cells shown" if len(values) != n * n else "")) + desc = (f"Clustered heatmap of {n} traits. Cell colour is the correlation coefficient, " + f"blue negative and red positive, in the order the average-linkage tree produced.") + + s = [f''' +{title} +{desc} + + +{title} +{sub}'''] + + # Cells. + for row, i in enumerate(order): + for column, j in enumerate(order): + value = r[i][j] + x = x0 + column * size + y = y0 + row * size + if value is None: + fill = "var(--na)" + label = "no correlation" + else: + fill = colour(value) + label = f"r = {value:+.3f}" + s.append(f'{names[i]} × {names[j]}: {label}') + + # Dendrogram on the left: leaves in the same order, heights from the tree. + leaves = {trait_index: position for position, trait_index in enumerate(order)} + nodes = {} + max_height = max(clustering.get("max_height") or 0.0, 1e-12) + x_max = LEFT_TREE - 18 + x_min = 14 + + def pos_x(height): + return x_max - (height / max_height) * (x_max - x_min) + + for index, trait_index in enumerate(order): + nodes[trait_index] = (x_max, y0 + (leaves[trait_index] + 0.5) * size) + + for merge_index, merge in enumerate(clustering.get("merges") or []): + left, right = merge["left"], merge["right"] + if left not in nodes or right not in nodes: + continue + (lx, ly), (rx, ry) = nodes[left], nodes[right] + x = pos_x(merge["height"]) + s.append(f'') + nodes[n + merge_index] = (x, (ly + ry) / 2) + + # Labels: trait names on the right, dataset only when it differs. + label_font = 11 if size < 14 else 12 + for row, i in enumerate(order): + y = y0 + (row + 0.5) * size + 3.5 + text = names[i] + s.append(f'{text}') + if datasets[i]: + s.append(f'{datasets[i]}') + + # Legend: the diverging scale, drawn as a strip. + lx, ly, lw, lh = x0, H - 74, 220, 12 + steps = 40 + for step in range(steps): + value = -1.0 + 2.0 * step / (steps - 1) + s.append(f'') + s.append(f'-1') + s.append(f'0') + s.append(f'+1') + s.append(f'correlation coefficient r') + s.append(f'clustered order, both axes: average linkage on 1 - r') + s.append("") + open(a.out, "w").write("\n".join(s) + "\n") + print(f"{a.out}: {n}x{n} heatmap, {len(clustering.get('merges') or [])} merges, " + f"cell {size:.1f}px") + + +if __name__ == "__main__": + main() -- cgit 1.4.1