about summary refs log tree commit diff
path: root/scripts
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-11-11 12:37:26 +0000
committerPjotr <wrk@bosa.lan>2026-11-11 12:37:26 +0000
commit84aced61986f229931b5e7738b4c64df45251a85 (patch)
tree8561fd6c863be1ccf828716817e1a34cb8db0f67 /scripts
parent3022797b55bae96322439770ea48e74eb2e0eb8d (diff)
downloadgenenetwork4-84aced61986f229931b5e7738b4c64df45251a85.tar.gz
bulklmm: exact marker-block parallelism, and what the cores are actually doing
Reported: a BulkLMM run does not look multicore. It is not a
misconfiguration. BulkLMM's single-trait `scan()` -- what this endpoint
calls -- has no `Threads.@threads` at all; the package threads
`bulkscan()` (many traits) and bulkscan_helpers, not scan. The
parallelism that was visible before was BLAS (OpenBLAS defaulting to all
96 cores here), which is why `user` time ran 4x `real` while Julia
threads made no difference: --threads 1 13.7 s, --threads 4 12.3 s.

  * The driver splits the markers into independent blocks and scans them
    on Julia threads (--blocks, default = BULKLMM_THREADS). It is exact:
    the LOD for each marker is independent given the null variance
    components, which depend on y and K only. Verified identical -- peak
    LOD 9.589938 and the same permutation thresholds at 1 and 4 blocks --
    and permutation thresholds combine by taking each permutation's
    maximum across blocks before the quantiles, which is what a
    genome-wide threshold is.
  * The parsed matrix and the kinship are cached per panel (read 1.3 s ->
    0.4 s, kinship 0.9 s -> 0), because both depend only on the cross.
  * The summary in `notes` now says what ran: julia threads, blocks, BLAS
    threads, and the read/scan split.
  * The hand parser is back: CSV.jl into a DataFrame is an order of
    magnitude slower for a 21056 x 235 matrix (14-17 s against 2 s).
  * BLAS is capped at 4 threads -- 96 was slower (10.5 s against 7.7 s for
    one scan).

Measured on BXD.8 trait 16627 (21056 markers): 1 block 20.9 s, 4 blocks
18.3 s, same answer. The gain is modest because the LMM loop is ~8 s of
the run and each block re-estimates its variance components. Where the
cores pay is bulkscan() -- many traits in one process, threaded over
trait blocks -- which is the shape the next endpoint wants; documented in
the README rather than left as folklore.
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/bulklmm_scan.jl127
-rwxr-xr-xscripts/plot-bulklmm.py131
2 files changed, 237 insertions, 21 deletions
diff --git a/scripts/bulklmm_scan.jl b/scripts/bulklmm_scan.jl
index c3e70cf..0710f1b 100755
--- a/scripts/bulklmm_scan.jl
+++ b/scripts/bulklmm_scan.jl
@@ -23,8 +23,13 @@
 #         --markers m.csv --out scan.csv [--nperm 1000 --perm-out t.csv]
 
 using BulkLMM
+using CSV
+using DataFrames
 using DelimitedFiles
+using Serialization
 using Statistics
+using Base64
+using LinearAlgebra
 
 function fail(message)
     println(stderr, "Error: ", message)
@@ -49,6 +54,8 @@ out_file = argument("--out")
 covar_file = argument("--covar")
 perm_out = argument("--perm-out")
 nperms = parse(Int, argument("--nperm", "0"))
+nblocks = parse(Int, argument("--blocks", string(Threads.nthreads())))
+cache_dir = argument("--cache-dir")
 seed = parse(Int, argument("--seed", "0"))
 reml = flag("--reml")
 assumption = argument("--assumption", "null")
@@ -79,24 +86,48 @@ length(individuals) >= 3 || fail("$(pheno_file): fewer than three individuals wi
 y = values
 
 # The genotype matrix: rows are individuals, columns markers, entries allele
-# probabilities. The header names the individuals, in the same order.
-geno_text = split(read(geno_file, String), '\n')
-geno_header = split(strip(geno_text[1]), ',')
-geno_individuals = String[strip(name) for name in geno_header[2:end]]
-first = findfirst(isequal(geno_text[1]), geno_text) + 1
-rows = geno_text[first:end]
-rows = [row for row in rows if !isempty(strip(row))]
-g = Matrix{Float64}(undef, length(rows), length(geno_individuals))
-marker_names = Vector{String}(undef, length(rows))
-for (i, row) in enumerate(rows)
-    fields = split(strip(row), ',')
-    marker_names[i] = String(fields[1])
-    for j in 1:length(geno_individuals)
-        cell = strip(fields[j + 1])
-        g[i, j] = (isempty(cell) || cell == "NA") ? NaN : something(tryparse(Float64, cell), NaN)
+# probabilities (0, 0.5, 1). Parsed by hand -- `CSV.jl` into a DataFrame is an
+# order of magnitude slower for this shape -- and cached under --cache-dir,
+# because the matrix and the kinship depend only on the panel.
+function read_matrix(geno_file, cache_dir)
+    tag = cache_dir === nothing ? nothing :
+        joinpath(cache_dir, string(hash((normpath(geno_file), filesize(geno_file),
+                                         mtime(geno_file))), ".matrix"))
+    if tag !== nothing && isfile(tag)
+        return deserialize(tag)
     end
+    rows = split(strip(read(geno_file, String)), '\n')
+    header = split(rows[1], ',')
+    geno_individuals = String[strip(name) for name in header[2:end]]
+    marker_names = String[]
+    pieces = Vector{Float64}[]
+    for row in rows[2:end]
+        isempty(strip(row)) && continue
+        fields = split(row, ',')
+        line = Vector{Float64}(undef, length(geno_individuals))
+        for j in 1:length(geno_individuals)
+            cell = strip(fields[j + 1])
+            line[j] = (isempty(cell) || cell == "NA") ? NaN :
+                something(tryparse(Float64, cell), NaN)
+        end
+        push!(marker_names, String(fields[1]))
+        push!(pieces, line)
+    end
+    xs = permutedims(reduce(hcat, pieces))
+    if tag !== nothing
+        try
+            mkpath(cache_dir)
+            serialize(tag, (marker_names, geno_individuals, xs))
+        catch
+        end
+    end
+    return (marker_names, geno_individuals, xs)
 end
+
+raised = time()
+marker_names, geno_individuals, g = read_matrix(geno_file, cache_dir)
 g = permutedims(g)   # BulkLMM wants individuals x markers
+read_seconds = time() - raised
 
 # Keep only the individuals that have both a value and genotypes, in the same
 # order, and fill a marker's missing probability with its column mean.
@@ -149,19 +180,73 @@ println(stderr, "bulklmm: ", length(y), " individuals with a value, ", length(ma
         " markers, covariates: ", covar === nothing ? "none" : string(size(covar, 2)))
 
 # ---- the scan ---------------------------------------------------------
-kinship = calcKinship(g)
+# Kinship depends only on the cross, so it is cached beside the matrix: a second
+# trait on the same panel should pay the scan and nothing else.
+raised = time()
+kinship = nothing
+cache_key = cache_dir === nothing ? nothing :
+    joinpath(cache_dir, string(hash((base64encode(join(marker_names)), string(size(g)))), ".kin"))
+if cache_key !== nothing && isfile(cache_key)
+    kinship = deserialize(cache_key)
+end
+if kinship === nothing
+    kinship = calcKinship(g)
+    if cache_key !== nothing
+        try
+            serialize(cache_key, kinship)
+        catch
+            # A cache that cannot be written is not a reason to fail the scan.
+        end
+    end
+end
+kinship_seconds = time() - raised
 options = Dict{Symbol, Any}(:reml => reml, :assumption => assumption, :method => method,
                             :permutation_test => nperms > 0, :nperms => nperms, :rndseed => seed)
-result = try
-    if covar === nothing
-        scan(y, g, kinship; options...)
-    else
-        scan(y, g, covar, kinship; options...)
+# BulkLMM's single-trait `scan()` is serial -- only `bulkscan()` (many traits)
+# and its helpers use Julia threads -- so one trait would leave every core but
+# one idle except for BLAS. Each marker's LOD is independent given the null
+# variance components (which depend on y and K only, not on the markers), so the
+# matrix is split into blocks and the blocks are scanned in parallel. The answer
+# is the same one the serial scan gives; only the wall clock changes.
+nblocks = clamp(nblocks, 1, max(1, min(Threads.nthreads(), size(g, 2))))
+# BLAS threads: OpenBLAS defaults to every core in the machine (96 here), which
+# on matrices this size is slower than a handful -- measured 10.5 s at 96 against
+# 7.7 s at 4 for one 235x21056 scan. When the markers are split across Julia
+# threads, each block gets one BLAS thread instead: oversubscribing both is what
+# makes a "multicore" run look single-core.
+BLAS.set_num_threads(nblocks > 1 ? 1 : min(4, Threads.nthreads()))
+edges = round.(Int, range(0, size(g, 2); length = nblocks + 1))
+blocks = [(edges[i] + 1, edges[i + 1]) for i in 1:nblocks if edges[i + 1] > edges[i]]
+results = Vector{Any}(undef, length(blocks))
+try
+    Threads.@threads for index in eachindex(blocks)
+        (first, last) = blocks[index]
+        piece = g[:, first:last]
+        results[index] = covar === nothing ? scan(y, piece, kinship; options...) :
+                                           scan(y, piece, covar, kinship; options...)
     end
 catch error
     println(stderr, "Error: ", sprint(showerror, error))
     exit(2)
 end
+result = (
+    lod = vcat((r.lod for r in results)...),
+    b = vcat((r.b for r in results)...),
+    h2_null = results[1].h2_null,
+    sigma2_e = results[1].sigma2_e,
+)
+# Genome-wide thresholds need the maximum over *all* markers in each
+# permutation, so combine the blocks' permutation matrices by taking their
+# per-permutation maxima before the quantiles.
+if nperms > 0 && all(r -> hasproperty(r, :L_perms), results)
+    maxima = reduce((left, right) -> max.(left, right),
+                    (vec(maximum(r.L_perms; dims = 1)) for r in results))
+    result = merge(result, (L_perms = reshape(maxima, 1, :),))
+end
+scan_seconds = time() - raised
+println(stderr, "bulklmm: julia threads ", Threads.nthreads(), ", blocks ", length(blocks),
+        ", BLAS threads ", BLAS.get_num_threads(), " | read ", round(read_seconds, digits = 1),
+        " s, scan ", round(scan_seconds, digits = 1), " s")
 
 h2 = hasproperty(result, :h2_null) ? result.h2_null : NaN
 beta = hasproperty(result, :b) ? result.b : fill(NaN, length(marker_names))
diff --git a/scripts/plot-bulklmm.py b/scripts/plot-bulklmm.py
new file mode 100755
index 0000000..51c24f8
--- /dev/null
+++ b/scripts/plot-bulklmm.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""LOD plot of a GN4 BulkLMM result, as one SVG panel.
+
+Input is the JSON response of POST /gn4/compute/bulklmm (saved as-is). The
+rows are positional -- marker, chr, pos (Mb), lod, beta, h2 -- so this reads
+`meta.columns` to find them. Thresholds come from the reply's own `notes`
+("alpha 0.05: LOD 3.81"), which is where the endpoint puts them, so posting a
+response back draws the picture a peak has to be judged against.
+
+    python3 plot-bulklmm.py scan.json -o scan.svg --trait BXD::16627
+
+Standard library only; writes plain SVG (light/dark aware).
+"""
+import argparse, json, math, re, sys
+
+def chr_key(c):
+    return (0, int(c), "") if c.isdigit() else (1, 0, c)
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    ap.add_argument("result", help="saved /compute/bulklmm response (JSON)")
+    ap.add_argument("-o", "--out", default="bulklmm_lod.svg")
+    ap.add_argument("--trait", default="", help="trait label for the title")
+    ap.add_argument("--title", default=None, help="override the title text")
+    a = ap.parse_args()
+
+    doc = json.load(open(a.result))
+    if "data" not in doc:
+        sys.exit(f"{a.result}: no 'data' member (error response?): {str(doc)[:200]}")
+    columns = (doc.get("meta") or {}).get("columns") or ["marker", "chr", "pos", "lod", "beta", "h2"]
+    rows = [r for r in doc["data"] if isinstance(r, list) and len(r) >= 4]
+    if not rows:
+        sys.exit(f"{a.result}: no rows to plot")
+    idx = {name: i for i, name in enumerate(columns)}
+    name_i, chr_i, pos_i, lod_i = idx["marker"], idx["chr"], idx["pos"], idx["lod"]
+
+    # Thresholds: the endpoint writes them into `notes` as "alpha 0.05: LOD 3.81".
+    notes = " ".join(doc.get("notes", []))
+    thresholds = []
+    for match in re.finditer(r"alpha ([0-9.]+): LOD ([0-9]+(?:\.[0-9]+)?)", notes):
+        alpha, lod = float(match.group(1)), float(match.group(2))
+        if abs(alpha - 0.05) < 1e-9:
+            thresholds.append((lod, "genome-wide (alpha 0.05)", "6 3"))
+        elif abs(alpha - 0.01) < 1e-9:
+            thresholds.append((lod, "alpha 0.01", "3 3"))
+    seed = re.search(r"seed (\d+)", notes)
+    h2 = re.search(r"heritability h2 = ([\d.eE+-]+)", notes)
+    nperm = re.search(r"from (\d+) rotations", notes)
+
+    by = {}
+    for row in rows:
+        try:
+            lod = float(row[lod_i])
+        except (TypeError, ValueError):
+            continue
+        by.setdefault(str(row[chr_i]), []).append((float(row[pos_i]), lod))
+    if not by:
+        sys.exit(f"{a.result}: no numeric LODs")
+    order = sorted(by, key=chr_key)
+    off, tot, gap = {}, 0.0, 8
+    for c in order:
+        by[c].sort()
+        off[c] = tot
+        tot += by[c][-1][0] + gap
+
+    top = max(rows, key=lambda r: r[lod_i])
+    top_lod = float(top[lod_i])
+    ymax = max(2, math.ceil(max([top_lod] + [t[0] for t in thresholds]) + 0.5))
+    W, H, L, R, T, B = 1000, 430, 62, 20, 64, 52
+    pw, ph = W - L - R, H - T - B
+    X = lambda c, p: L + (off[c] + p) / tot * pw
+    Y = lambda v: T + ph - min(v, ymax) / ymax * ph
+
+    label = a.trait or "BulkLMM scan"
+    title = a.title or f"{label}: LOD along the genome"
+    sub = "BulkLMM LMM, global kinship"
+    if h2:
+        sub += f", h2 {float(h2.group(1)):.3f}"
+    sub += (f", thresholds from {nperm.group(1)} rotations (seed {seed.group(1)})"
+            if nperm and seed else ", no permutation threshold")
+
+    parts = [
+        f'<svg xmlns="http://www.w3.org/2000/svg" width="{W}" height="{H}" '
+        f'viewBox="0 0 {W} {H}" font-family="sans-serif" role="img">',
+        "<style>"
+        ".bg{fill:#fff}.fg{fill:#333}.grid{stroke:#e8e8e8}.axis{stroke:#999}"
+        ".pt{fill:#2b6cb0}.ptalt{fill:#4299e1}.thr{stroke:#c53030;stroke-dasharray:6 3}"
+        ".thr2{stroke:#dd6b20;stroke-dasharray:3 3}.lab{font-size:11px;fill:#555}"
+        ".ttl{font-size:15px;font-weight:bold;fill:#111}.sub{font-size:11px;fill:#666}"
+        "@media (prefers-color-scheme: dark){.bg{fill:#111}.fg{fill:#eee}"
+        ".grid{stroke:#2a2a2a}.pt{fill:#63b3ed}.ptalt{fill:#90cdf4}"
+        ".thr{stroke:#fc8181}.thr2{stroke:#f6ad55}.lab{fill:#bbb}.ttl{fill:#fff}.sub{fill:#999}}"
+        "</style>",
+        f'<rect class="bg" width="{W}" height="{H}"/>',
+        f'<text class="ttl" x="{L}" y="28">{title}</text>',
+        f'<text class="sub" x="{L}" y="46">{sub}</text>',
+    ]
+    for step in range(0, ymax + 1, max(1, ymax // 5)):
+        y = Y(step)
+        parts.append(f'<line class="grid" x1="{L}" y1="{y:.1f}" x2="{L + pw}" y2="{y:.1f}"/>')
+        parts.append(f'<text class="lab" x="{L - 8}" y="{y + 4:.1f}" text-anchor="end">{step}</text>')
+    parts.append(f'<line class="axis" x1="{L}" y1="{T}" x2="{L}" y2="{T + ph}"/>')
+    for index, c in enumerate(order):
+        colour = "pt" if index % 2 == 0 else "ptalt"
+        for pos, lod in by[c]:
+            parts.append(f'<circle class="{colour}" cx="{X(c, pos):.1f}" cy="{Y(lod):.1f}" r="1.6"/>')
+        centre = L + (off[c] + by[c][-1][0] / 2) / tot * pw
+        parts.append(f'<text class="lab" x="{centre:.1f}" y="{T + ph + 16}" '
+                     f'text-anchor="middle">{c}</text>')
+    for value, label_text, dash in thresholds:
+        y = Y(value)
+        parts.append(f'<line class="thr" x1="{L}" y1="{y:.1f}" x2="{L + pw}" y2="{y:.1f}" '
+                     f'stroke-dasharray="{dash}"/>')
+        parts.append(f'<text class="lab" x="{L + pw - 4}" y="{y - 4:.1f}" '
+                     f'text-anchor="end">{label_text}: LOD {value:.2f}</text>')
+    parts.append(f'<text class="lab" x="{L - 44}" y="{T - 10}">LOD</text>')
+    parts.append(
+        f'<desc>The highest peak is LOD {top_lod:.2f} on chromosome {top[chr_i]} at '
+        f'{float(top[pos_i]):.1f} Mb ({top[name_i]}).'
+        + ("" if not thresholds else
+           " Genome-wide threshold (alpha 0.05) LOD "
+           + f"{[t[0] for t in thresholds if 'alpha 0.05' in t[1]][0]:.2f}.")
+        + "</desc>")
+    parts.append("</svg>")
+    open(a.out, "w").write("\n".join(parts))
+    print(f"BulkLMM scan. The highest peak is LOD {top_lod:.2f} on chromosome {top[chr_i]} "
+          f"at {float(top[pos_i]):.1f} Mb ({top[name_i]}).")
+    return 0
+
+if __name__ == "__main__":
+    raise SystemExit(main())