about summary refs log tree commit diff
path: root/scripts/plot-bulklmm.py
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-11-11 12:37:26 +0000
committerPjotr <wrk@bosa.lan>2026-11-11 12:37:26 +0000
commit84aced61986f229931b5e7738b4c64df45251a85 (patch)
tree8561fd6c863be1ccf828716817e1a34cb8db0f67 /scripts/plot-bulklmm.py
parent3022797b55bae96322439770ea48e74eb2e0eb8d (diff)
downloadgenenetwork4-84aced61986f229931b5e7738b4c64df45251a85.tar.gz
bulklmm: exact marker-block parallelism, and what the cores are actually doing
Reported: a BulkLMM run does not look multicore. It is not a
misconfiguration. BulkLMM's single-trait `scan()` -- what this endpoint
calls -- has no `Threads.@threads` at all; the package threads
`bulkscan()` (many traits) and bulkscan_helpers, not scan. The
parallelism that was visible before was BLAS (OpenBLAS defaulting to all
96 cores here), which is why `user` time ran 4x `real` while Julia
threads made no difference: --threads 1 13.7 s, --threads 4 12.3 s.

  * The driver splits the markers into independent blocks and scans them
    on Julia threads (--blocks, default = BULKLMM_THREADS). It is exact:
    the LOD for each marker is independent given the null variance
    components, which depend on y and K only. Verified identical -- peak
    LOD 9.589938 and the same permutation thresholds at 1 and 4 blocks --
    and permutation thresholds combine by taking each permutation's
    maximum across blocks before the quantiles, which is what a
    genome-wide threshold is.
  * The parsed matrix and the kinship are cached per panel (read 1.3 s ->
    0.4 s, kinship 0.9 s -> 0), because both depend only on the cross.
  * The summary in `notes` now says what ran: julia threads, blocks, BLAS
    threads, and the read/scan split.
  * The hand parser is back: CSV.jl into a DataFrame is an order of
    magnitude slower for a 21056 x 235 matrix (14-17 s against 2 s).
  * BLAS is capped at 4 threads -- 96 was slower (10.5 s against 7.7 s for
    one scan).

Measured on BXD.8 trait 16627 (21056 markers): 1 block 20.9 s, 4 blocks
18.3 s, same answer. The gain is modest because the LMM loop is ~8 s of
the run and each block re-estimates its variance components. Where the
cores pay is bulkscan() -- many traits in one process, threaded over
trait blocks -- which is the shape the next endpoint wants; documented in
the README rather than left as folklore.
Diffstat (limited to 'scripts/plot-bulklmm.py')
-rwxr-xr-xscripts/plot-bulklmm.py131
1 files changed, 131 insertions, 0 deletions
diff --git a/scripts/plot-bulklmm.py b/scripts/plot-bulklmm.py
new file mode 100755
index 0000000..51c24f8
--- /dev/null
+++ b/scripts/plot-bulklmm.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""LOD plot of a GN4 BulkLMM result, as one SVG panel.
+
+Input is the JSON response of POST /gn4/compute/bulklmm (saved as-is). The
+rows are positional -- marker, chr, pos (Mb), lod, beta, h2 -- so this reads
+`meta.columns` to find them. Thresholds come from the reply's own `notes`
+("alpha 0.05: LOD 3.81"), which is where the endpoint puts them, so posting a
+response back draws the picture a peak has to be judged against.
+
+    python3 plot-bulklmm.py scan.json -o scan.svg --trait BXD::16627
+
+Standard library only; writes plain SVG (light/dark aware).
+"""
+import argparse, json, math, re, sys
+
+def chr_key(c):
+    return (0, int(c), "") if c.isdigit() else (1, 0, c)
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    ap.add_argument("result", help="saved /compute/bulklmm response (JSON)")
+    ap.add_argument("-o", "--out", default="bulklmm_lod.svg")
+    ap.add_argument("--trait", default="", help="trait label for the title")
+    ap.add_argument("--title", default=None, help="override the title text")
+    a = ap.parse_args()
+
+    doc = json.load(open(a.result))
+    if "data" not in doc:
+        sys.exit(f"{a.result}: no 'data' member (error response?): {str(doc)[:200]}")
+    columns = (doc.get("meta") or {}).get("columns") or ["marker", "chr", "pos", "lod", "beta", "h2"]
+    rows = [r for r in doc["data"] if isinstance(r, list) and len(r) >= 4]
+    if not rows:
+        sys.exit(f"{a.result}: no rows to plot")
+    idx = {name: i for i, name in enumerate(columns)}
+    name_i, chr_i, pos_i, lod_i = idx["marker"], idx["chr"], idx["pos"], idx["lod"]
+
+    # Thresholds: the endpoint writes them into `notes` as "alpha 0.05: LOD 3.81".
+    notes = " ".join(doc.get("notes", []))
+    thresholds = []
+    for match in re.finditer(r"alpha ([0-9.]+): LOD ([0-9]+(?:\.[0-9]+)?)", notes):
+        alpha, lod = float(match.group(1)), float(match.group(2))
+        if abs(alpha - 0.05) < 1e-9:
+            thresholds.append((lod, "genome-wide (alpha 0.05)", "6 3"))
+        elif abs(alpha - 0.01) < 1e-9:
+            thresholds.append((lod, "alpha 0.01", "3 3"))
+    seed = re.search(r"seed (\d+)", notes)
+    h2 = re.search(r"heritability h2 = ([\d.eE+-]+)", notes)
+    nperm = re.search(r"from (\d+) rotations", notes)
+
+    by = {}
+    for row in rows:
+        try:
+            lod = float(row[lod_i])
+        except (TypeError, ValueError):
+            continue
+        by.setdefault(str(row[chr_i]), []).append((float(row[pos_i]), lod))
+    if not by:
+        sys.exit(f"{a.result}: no numeric LODs")
+    order = sorted(by, key=chr_key)
+    off, tot, gap = {}, 0.0, 8
+    for c in order:
+        by[c].sort()
+        off[c] = tot
+        tot += by[c][-1][0] + gap
+
+    top = max(rows, key=lambda r: r[lod_i])
+    top_lod = float(top[lod_i])
+    ymax = max(2, math.ceil(max([top_lod] + [t[0] for t in thresholds]) + 0.5))
+    W, H, L, R, T, B = 1000, 430, 62, 20, 64, 52
+    pw, ph = W - L - R, H - T - B
+    X = lambda c, p: L + (off[c] + p) / tot * pw
+    Y = lambda v: T + ph - min(v, ymax) / ymax * ph
+
+    label = a.trait or "BulkLMM scan"
+    title = a.title or f"{label}: LOD along the genome"
+    sub = "BulkLMM LMM, global kinship"
+    if h2:
+        sub += f", h2 {float(h2.group(1)):.3f}"
+    sub += (f", thresholds from {nperm.group(1)} rotations (seed {seed.group(1)})"
+            if nperm and seed else ", no permutation threshold")
+
+    parts = [
+        f'<svg xmlns="http://www.w3.org/2000/svg" width="{W}" height="{H}" '
+        f'viewBox="0 0 {W} {H}" font-family="sans-serif" role="img">',
+        "<style>"
+        ".bg{fill:#fff}.fg{fill:#333}.grid{stroke:#e8e8e8}.axis{stroke:#999}"
+        ".pt{fill:#2b6cb0}.ptalt{fill:#4299e1}.thr{stroke:#c53030;stroke-dasharray:6 3}"
+        ".thr2{stroke:#dd6b20;stroke-dasharray:3 3}.lab{font-size:11px;fill:#555}"
+        ".ttl{font-size:15px;font-weight:bold;fill:#111}.sub{font-size:11px;fill:#666}"
+        "@media (prefers-color-scheme: dark){.bg{fill:#111}.fg{fill:#eee}"
+        ".grid{stroke:#2a2a2a}.pt{fill:#63b3ed}.ptalt{fill:#90cdf4}"
+        ".thr{stroke:#fc8181}.thr2{stroke:#f6ad55}.lab{fill:#bbb}.ttl{fill:#fff}.sub{fill:#999}}"
+        "</style>",
+        f'<rect class="bg" width="{W}" height="{H}"/>',
+        f'<text class="ttl" x="{L}" y="28">{title}</text>',
+        f'<text class="sub" x="{L}" y="46">{sub}</text>',
+    ]
+    for step in range(0, ymax + 1, max(1, ymax // 5)):
+        y = Y(step)
+        parts.append(f'<line class="grid" x1="{L}" y1="{y:.1f}" x2="{L + pw}" y2="{y:.1f}"/>')
+        parts.append(f'<text class="lab" x="{L - 8}" y="{y + 4:.1f}" text-anchor="end">{step}</text>')
+    parts.append(f'<line class="axis" x1="{L}" y1="{T}" x2="{L}" y2="{T + ph}"/>')
+    for index, c in enumerate(order):
+        colour = "pt" if index % 2 == 0 else "ptalt"
+        for pos, lod in by[c]:
+            parts.append(f'<circle class="{colour}" cx="{X(c, pos):.1f}" cy="{Y(lod):.1f}" r="1.6"/>')
+        centre = L + (off[c] + by[c][-1][0] / 2) / tot * pw
+        parts.append(f'<text class="lab" x="{centre:.1f}" y="{T + ph + 16}" '
+                     f'text-anchor="middle">{c}</text>')
+    for value, label_text, dash in thresholds:
+        y = Y(value)
+        parts.append(f'<line class="thr" x1="{L}" y1="{y:.1f}" x2="{L + pw}" y2="{y:.1f}" '
+                     f'stroke-dasharray="{dash}"/>')
+        parts.append(f'<text class="lab" x="{L + pw - 4}" y="{y - 4:.1f}" '
+                     f'text-anchor="end">{label_text}: LOD {value:.2f}</text>')
+    parts.append(f'<text class="lab" x="{L - 44}" y="{T - 10}">LOD</text>')
+    parts.append(
+        f'<desc>The highest peak is LOD {top_lod:.2f} on chromosome {top[chr_i]} at '
+        f'{float(top[pos_i]):.1f} Mb ({top[name_i]}).'
+        + ("" if not thresholds else
+           " Genome-wide threshold (alpha 0.05) LOD "
+           + f"{[t[0] for t in thresholds if 'alpha 0.05' in t[1]][0]:.2f}.")
+        + "</desc>")
+    parts.append("</svg>")
+    open(a.out, "w").write("\n".join(parts))
+    print(f"BulkLMM scan. The highest peak is LOD {top_lod:.2f} on chromosome {top[chr_i]} "
+          f"at {float(top[pos_i]):.1f} Mb ({top[name_i]}).")
+    return 0
+
+if __name__ == "__main__":
+    raise SystemExit(main())