diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-16 17:17:11 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-16 17:17:11 +0000 |
| commit | ea3dd926211e8db4a0d7d3739a606440b888e8b0 (patch) | |
| tree | 31b58cd0126960e70a76e45dc2c3b7c9a1c6d0b6 /bin/monitor-gn4 | |
| parent | 33e837e771e07936c808fb3aeb3540525691718c (diff) | |
| download | genenetwork4-main.tar.gz | |
`"cluster": true` adds `heatmap: {order, merges, max_height}` to the matrix
reply and `POST /plot/heatmap` draws it: the r matrix in that order, the
dendrogram that produced it, and a diverging scale with blue negative and red
positive. A reader can see *why* two traits are neighbours instead of trusting
an ordering.
Clustering decisions, all recorded because an unrepeatable clustering is worse
than none:
* distance `1 - r` -- the usual choice for correlations, and it keeps
anti-correlated traits apart, which is what a heatmap of correlations should
show;
* **average linkage (UPGMA)** -- single linkage chains along a gradient and
complete linkage makes one flat blob out of noisy traits; average is the
compromise GN1's R heatmaps used too;
* **deterministic ties and leaf order** (lower index first), so two runs on one
matrix produce the same picture and plots stay diffable;
* a matrix with a **gap is refused** rather than clustered: an unknown distance
treated as 1 would invent a structure, and the reply's note says to drop
those traits and ask again.
Pairing with the PCA work this is the visual half of a collection: scree plot
for how much structure there is, clustered heatmap for where it is.
Monitor, as asked: `compute-correlation-heatmap` (order must be a permutation
with n-1 merges), `plot-heatmap`, and the plot endpoints that had no check at
all -- `plot-gemma` and `plot-bnw`, each posting a real compute result back.
Plots tier is now five checks, all ok; full run 47 ok, 0 failed, 1 skipped.
Diffstat (limited to 'bin/monitor-gn4')
| -rwxr-xr-x | bin/monitor-gn4 | 77 |
1 files changed, 77 insertions, 0 deletions
diff --git a/bin/monitor-gn4 b/bin/monitor-gn4 index 01f38f8..609fdb8 100755 --- a/bin/monitor-gn4 +++ b/bin/monitor-gn4 @@ -650,6 +650,18 @@ def build_checks(found, tiers): "method": "pearson", "pca": True}, judge=has_pca, hint="no external tool: pure Rust + MariaDB")) + if found.control_trait: + # Clustered ordering: average linkage on 1 - r, for the heatmap. + add(Check("compute-correlation-heatmap", "compute", "POST", + "/compute/correlation/matrix", + "clustered order + tree (pure Rust)", + body={"traits": [ + {"dataset": found.publish, "trait_name": found.trait}, + {"dataset": found.publish, + "trait_name": found.control_trait}], + "method": "pearson", "cluster": True}, + judge=has_clustering, + hint="no external tool: pure Rust + MariaDB")) add(Check("compute-gemma", "compute", "POST", "/compute/gemma", "gemma-wrapper + gemma + R (R_LIBS_SITE)", body={"group": g, "lmm": 9, "samples": samples}, @@ -724,6 +736,21 @@ def build_checks(found, tiers): body=pca_plot_body(found), expect=200, hint="PCA_PLOTTER and python3", judge=has_svg)) + if found.control_trait: + add(Check("plot-heatmap", "plots", "POST", "/plot/heatmap", + "clustered matrix -> heatmap svg (HEATMAP_PLOTTER)", + body=heatmap_plot_body(found), expect=200, + hint="HEATMAP_PLOTTER and python3", judge=has_svg)) + if found.samples: + # The Manhattan plot: a GEMMA run's reply, posted back as-is. + add(Check("plot-gemma", "plots", "POST", "/plot/gemma", + "GEMMA result -> Manhattan svg (GEMMA_PLOTTER)", + body=gemma_plot_body(found), expect=200, + hint="GEMMA_PLOTTER and python3", judge=has_svg)) + add(Check("plot-bnw", "plots", "POST", "/plot/bnw", + "BNW result -> network svg (BNW_PLOTTER)", + body=bnw_plot_body(), expect=200, + hint="BNW_PLOTTER and python3", judge=has_svg)) add(Check("plot-genotypes", "plots", "POST", "/plot/genotypes", "peak-genotypes + values -> svg (GENOTYPE_PLOTTER)", body=genotype_plot_body(found), expect=200, @@ -758,6 +785,22 @@ def perturbed(body): return None +def has_clustering(reply): + """A clustered order that is a permutation, with n-1 merges.""" + data = (reply.json or {}).get("data") or {} + heatmap = data.get("heatmap") or {} + order = heatmap.get("order") or [] + merges = heatmap.get("merges") or [] + traits = data.get("traits") or [] + if len(order) != len(traits): + return FAIL, "order lists %d, traits %d" % (len(order), len(traits)) + if sorted(order) != list(range(len(traits))): + return FAIL, "order is not a permutation: %s" % order + if len(merges) != max(len(traits) - 1, 0): + return FAIL, "%d merges for %d traits" % (len(merges), len(traits)) + return OK, "%d traits, max distance %.3f" % (len(traits), heatmap.get("max_height") or 0.0) + + def has_pca(reply): """A PCA block whose eigenvalues sum to the trait count.""" data = (reply.json or {}).get("data") or {} @@ -816,6 +859,40 @@ def has_svg(reply): return FAIL, "not an SVG: " + str(body)[:80] +def _post_result(client, path, body, what): + reply = client.call("POST", path, body, timeout=600) + if reply.status != 200: + code, detail = reply.fields() + raise Skip("could not produce a %s to plot: %s %s" + % (what, code or reply.status, str(detail)[:80])) + return reply.json or {} + + +def heatmap_plot_body(found): + def build(client): + return _post_result(client, "/compute/correlation/matrix", + {"traits": [ + {"dataset": found.publish, "trait_name": found.trait}, + {"dataset": found.publish, + "trait_name": found.control_trait}], + "method": "pearson", "cluster": True}, "clustered matrix") + return build + + +def gemma_plot_body(found): + def build(client): + return _post_result(client, "/compute/gemma", + {"group": found.group, "lmm": 9, "samples": found.samples}, + "GEMMA result") + return build + + +def bnw_plot_body(): + def build(client): + return _post_result(client, "/compute/bnw", {"values": bnw_matrix()}, "BNW result") + return build + + def pca_plot_body(found): """Ask the matrix for a PCA, then post that reply to the plotter.""" def build(client): |
