diff options
| author | Pjotr <wrk@bosa.lan> | 2026-11-11 12:37:26 +0000 |
|---|---|---|
| committer | Pjotr <wrk@bosa.lan> | 2026-11-11 12:37:26 +0000 |
| commit | 84aced61986f229931b5e7738b4c64df45251a85 (patch) | |
| tree | 8561fd6c863be1ccf828716817e1a34cb8db0f67 /scripts/plot-bulklmm.py | |
| parent | 3022797b55bae96322439770ea48e74eb2e0eb8d (diff) | |
| download | genenetwork4-84aced61986f229931b5e7738b4c64df45251a85.tar.gz | |
bulklmm: exact marker-block parallelism, and what the cores are actually doing
Reported: a BulkLMM run does not look multicore. It is not a
misconfiguration. BulkLMM's single-trait `scan()` -- what this endpoint
calls -- has no `Threads.@threads` at all; the package threads
`bulkscan()` (many traits) and bulkscan_helpers, not scan. The
parallelism that was visible before was BLAS (OpenBLAS defaulting to all
96 cores here), which is why `user` time ran 4x `real` while Julia
threads made no difference: --threads 1 13.7 s, --threads 4 12.3 s.
* The driver splits the markers into independent blocks and scans them
on Julia threads (--blocks, default = BULKLMM_THREADS). It is exact:
the LOD for each marker is independent given the null variance
components, which depend on y and K only. Verified identical -- peak
LOD 9.589938 and the same permutation thresholds at 1 and 4 blocks --
and permutation thresholds combine by taking each permutation's
maximum across blocks before the quantiles, which is what a
genome-wide threshold is.
* The parsed matrix and the kinship are cached per panel (read 1.3 s ->
0.4 s, kinship 0.9 s -> 0), because both depend only on the cross.
* The summary in `notes` now says what ran: julia threads, blocks, BLAS
threads, and the read/scan split.
* The hand parser is back: CSV.jl into a DataFrame is an order of
magnitude slower for a 21056 x 235 matrix (14-17 s against 2 s).
* BLAS is capped at 4 threads -- 96 was slower (10.5 s against 7.7 s for
one scan).
Measured on BXD.8 trait 16627 (21056 markers): 1 block 20.9 s, 4 blocks
18.3 s, same answer. The gain is modest because the LMM loop is ~8 s of
the run and each block re-estimates its variance components. Where the
cores pay is bulkscan() -- many traits in one process, threaded over
trait blocks -- which is the shape the next endpoint wants; documented in
the README rather than left as folklore.
Diffstat (limited to 'scripts/plot-bulklmm.py')
| -rwxr-xr-x | scripts/plot-bulklmm.py | 131 |
1 files changed, 131 insertions, 0 deletions
diff --git a/scripts/plot-bulklmm.py b/scripts/plot-bulklmm.py new file mode 100755 index 0000000..51c24f8 --- /dev/null +++ b/scripts/plot-bulklmm.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""LOD plot of a GN4 BulkLMM result, as one SVG panel. + +Input is the JSON response of POST /gn4/compute/bulklmm (saved as-is). The +rows are positional -- marker, chr, pos (Mb), lod, beta, h2 -- so this reads +`meta.columns` to find them. Thresholds come from the reply's own `notes` +("alpha 0.05: LOD 3.81"), which is where the endpoint puts them, so posting a +response back draws the picture a peak has to be judged against. + + python3 plot-bulklmm.py scan.json -o scan.svg --trait BXD::16627 + +Standard library only; writes plain SVG (light/dark aware). +""" +import argparse, json, math, re, sys + +def chr_key(c): + return (0, int(c), "") if c.isdigit() else (1, 0, c) + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("result", help="saved /compute/bulklmm response (JSON)") + ap.add_argument("-o", "--out", default="bulklmm_lod.svg") + ap.add_argument("--trait", default="", help="trait label for the title") + ap.add_argument("--title", default=None, help="override the title text") + a = ap.parse_args() + + doc = json.load(open(a.result)) + if "data" not in doc: + sys.exit(f"{a.result}: no 'data' member (error response?): {str(doc)[:200]}") + columns = (doc.get("meta") or {}).get("columns") or ["marker", "chr", "pos", "lod", "beta", "h2"] + rows = [r for r in doc["data"] if isinstance(r, list) and len(r) >= 4] + if not rows: + sys.exit(f"{a.result}: no rows to plot") + idx = {name: i for i, name in enumerate(columns)} + name_i, chr_i, pos_i, lod_i = idx["marker"], idx["chr"], idx["pos"], idx["lod"] + + # Thresholds: the endpoint writes them into `notes` as "alpha 0.05: LOD 3.81". + notes = " ".join(doc.get("notes", [])) + thresholds = [] + for match in re.finditer(r"alpha ([0-9.]+): LOD ([0-9]+(?:\.[0-9]+)?)", notes): + alpha, lod = float(match.group(1)), float(match.group(2)) + if abs(alpha - 0.05) < 1e-9: + thresholds.append((lod, "genome-wide (alpha 0.05)", "6 3")) + elif abs(alpha - 0.01) < 1e-9: + thresholds.append((lod, "alpha 0.01", "3 3")) + seed = re.search(r"seed (\d+)", notes) + h2 = re.search(r"heritability h2 = ([\d.eE+-]+)", notes) + nperm = re.search(r"from (\d+) rotations", notes) + + by = {} + for row in rows: + try: + lod = float(row[lod_i]) + except (TypeError, ValueError): + continue + by.setdefault(str(row[chr_i]), []).append((float(row[pos_i]), lod)) + if not by: + sys.exit(f"{a.result}: no numeric LODs") + order = sorted(by, key=chr_key) + off, tot, gap = {}, 0.0, 8 + for c in order: + by[c].sort() + off[c] = tot + tot += by[c][-1][0] + gap + + top = max(rows, key=lambda r: r[lod_i]) + top_lod = float(top[lod_i]) + ymax = max(2, math.ceil(max([top_lod] + [t[0] for t in thresholds]) + 0.5)) + W, H, L, R, T, B = 1000, 430, 62, 20, 64, 52 + pw, ph = W - L - R, H - T - B + X = lambda c, p: L + (off[c] + p) / tot * pw + Y = lambda v: T + ph - min(v, ymax) / ymax * ph + + label = a.trait or "BulkLMM scan" + title = a.title or f"{label}: LOD along the genome" + sub = "BulkLMM LMM, global kinship" + if h2: + sub += f", h2 {float(h2.group(1)):.3f}" + sub += (f", thresholds from {nperm.group(1)} rotations (seed {seed.group(1)})" + if nperm and seed else ", no permutation threshold") + + parts = [ + f'<svg xmlns="http://www.w3.org/2000/svg" width="{W}" height="{H}" ' + f'viewBox="0 0 {W} {H}" font-family="sans-serif" role="img">', + "<style>" + ".bg{fill:#fff}.fg{fill:#333}.grid{stroke:#e8e8e8}.axis{stroke:#999}" + ".pt{fill:#2b6cb0}.ptalt{fill:#4299e1}.thr{stroke:#c53030;stroke-dasharray:6 3}" + ".thr2{stroke:#dd6b20;stroke-dasharray:3 3}.lab{font-size:11px;fill:#555}" + ".ttl{font-size:15px;font-weight:bold;fill:#111}.sub{font-size:11px;fill:#666}" + "@media (prefers-color-scheme: dark){.bg{fill:#111}.fg{fill:#eee}" + ".grid{stroke:#2a2a2a}.pt{fill:#63b3ed}.ptalt{fill:#90cdf4}" + ".thr{stroke:#fc8181}.thr2{stroke:#f6ad55}.lab{fill:#bbb}.ttl{fill:#fff}.sub{fill:#999}}" + "</style>", + f'<rect class="bg" width="{W}" height="{H}"/>', + f'<text class="ttl" x="{L}" y="28">{title}</text>', + f'<text class="sub" x="{L}" y="46">{sub}</text>', + ] + for step in range(0, ymax + 1, max(1, ymax // 5)): + y = Y(step) + parts.append(f'<line class="grid" x1="{L}" y1="{y:.1f}" x2="{L + pw}" y2="{y:.1f}"/>') + parts.append(f'<text class="lab" x="{L - 8}" y="{y + 4:.1f}" text-anchor="end">{step}</text>') + parts.append(f'<line class="axis" x1="{L}" y1="{T}" x2="{L}" y2="{T + ph}"/>') + for index, c in enumerate(order): + colour = "pt" if index % 2 == 0 else "ptalt" + for pos, lod in by[c]: + parts.append(f'<circle class="{colour}" cx="{X(c, pos):.1f}" cy="{Y(lod):.1f}" r="1.6"/>') + centre = L + (off[c] + by[c][-1][0] / 2) / tot * pw + parts.append(f'<text class="lab" x="{centre:.1f}" y="{T + ph + 16}" ' + f'text-anchor="middle">{c}</text>') + for value, label_text, dash in thresholds: + y = Y(value) + parts.append(f'<line class="thr" x1="{L}" y1="{y:.1f}" x2="{L + pw}" y2="{y:.1f}" ' + f'stroke-dasharray="{dash}"/>') + parts.append(f'<text class="lab" x="{L + pw - 4}" y="{y - 4:.1f}" ' + f'text-anchor="end">{label_text}: LOD {value:.2f}</text>') + parts.append(f'<text class="lab" x="{L - 44}" y="{T - 10}">LOD</text>') + parts.append( + f'<desc>The highest peak is LOD {top_lod:.2f} on chromosome {top[chr_i]} at ' + f'{float(top[pos_i]):.1f} Mb ({top[name_i]}).' + + ("" if not thresholds else + " Genome-wide threshold (alpha 0.05) LOD " + + f"{[t[0] for t in thresholds if 'alpha 0.05' in t[1]][0]:.2f}.") + + "</desc>") + parts.append("</svg>") + open(a.out, "w").write("\n".join(parts)) + print(f"BulkLMM scan. The highest peak is LOD {top_lod:.2f} on chromosome {top[chr_i]} " + f"at {float(top[pos_i]):.1f} Mb ({top[name_i]}).") + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) |
