diff options
| author | Pjotr <wrk@bosa.lan> | 2026-11-11 12:37:26 +0000 |
|---|---|---|
| committer | Pjotr <wrk@bosa.lan> | 2026-11-11 12:37:26 +0000 |
| commit | 84aced61986f229931b5e7738b4c64df45251a85 (patch) | |
| tree | 8561fd6c863be1ccf828716817e1a34cb8db0f67 /scripts | |
| parent | 3022797b55bae96322439770ea48e74eb2e0eb8d (diff) | |
| download | genenetwork4-84aced61986f229931b5e7738b4c64df45251a85.tar.gz | |
bulklmm: exact marker-block parallelism, and what the cores are actually doing
Reported: a BulkLMM run does not look multicore. It is not a
misconfiguration. BulkLMM's single-trait `scan()` -- what this endpoint
calls -- has no `Threads.@threads` at all; the package threads
`bulkscan()` (many traits) and bulkscan_helpers, not scan. The
parallelism that was visible before was BLAS (OpenBLAS defaulting to all
96 cores here), which is why `user` time ran 4x `real` while Julia
threads made no difference: --threads 1 13.7 s, --threads 4 12.3 s.
* The driver splits the markers into independent blocks and scans them
on Julia threads (--blocks, default = BULKLMM_THREADS). It is exact:
the LOD for each marker is independent given the null variance
components, which depend on y and K only. Verified identical -- peak
LOD 9.589938 and the same permutation thresholds at 1 and 4 blocks --
and permutation thresholds combine by taking each permutation's
maximum across blocks before the quantiles, which is what a
genome-wide threshold is.
* The parsed matrix and the kinship are cached per panel (read 1.3 s ->
0.4 s, kinship 0.9 s -> 0), because both depend only on the cross.
* The summary in `notes` now says what ran: julia threads, blocks, BLAS
threads, and the read/scan split.
* The hand parser is back: CSV.jl into a DataFrame is an order of
magnitude slower for a 21056 x 235 matrix (14-17 s against 2 s).
* BLAS is capped at 4 threads -- 96 was slower (10.5 s against 7.7 s for
one scan).
Measured on BXD.8 trait 16627 (21056 markers): 1 block 20.9 s, 4 blocks
18.3 s, same answer. The gain is modest because the LMM loop is ~8 s of
the run and each block re-estimates its variance components. Where the
cores pay is bulkscan() -- many traits in one process, threaded over
trait blocks -- which is the shape the next endpoint wants; documented in
the README rather than left as folklore.
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/bulklmm_scan.jl | 127 | ||||
| -rwxr-xr-x | scripts/plot-bulklmm.py | 131 |
2 files changed, 237 insertions, 21 deletions
diff --git a/scripts/bulklmm_scan.jl b/scripts/bulklmm_scan.jl index c3e70cf..0710f1b 100755 --- a/scripts/bulklmm_scan.jl +++ b/scripts/bulklmm_scan.jl @@ -23,8 +23,13 @@ # --markers m.csv --out scan.csv [--nperm 1000 --perm-out t.csv] using BulkLMM +using CSV +using DataFrames using DelimitedFiles +using Serialization using Statistics +using Base64 +using LinearAlgebra function fail(message) println(stderr, "Error: ", message) @@ -49,6 +54,8 @@ out_file = argument("--out") covar_file = argument("--covar") perm_out = argument("--perm-out") nperms = parse(Int, argument("--nperm", "0")) +nblocks = parse(Int, argument("--blocks", string(Threads.nthreads()))) +cache_dir = argument("--cache-dir") seed = parse(Int, argument("--seed", "0")) reml = flag("--reml") assumption = argument("--assumption", "null") @@ -79,24 +86,48 @@ length(individuals) >= 3 || fail("$(pheno_file): fewer than three individuals wi y = values # The genotype matrix: rows are individuals, columns markers, entries allele -# probabilities. The header names the individuals, in the same order. -geno_text = split(read(geno_file, String), '\n') -geno_header = split(strip(geno_text[1]), ',') -geno_individuals = String[strip(name) for name in geno_header[2:end]] -first = findfirst(isequal(geno_text[1]), geno_text) + 1 -rows = geno_text[first:end] -rows = [row for row in rows if !isempty(strip(row))] -g = Matrix{Float64}(undef, length(rows), length(geno_individuals)) -marker_names = Vector{String}(undef, length(rows)) -for (i, row) in enumerate(rows) - fields = split(strip(row), ',') - marker_names[i] = String(fields[1]) - for j in 1:length(geno_individuals) - cell = strip(fields[j + 1]) - g[i, j] = (isempty(cell) || cell == "NA") ? NaN : something(tryparse(Float64, cell), NaN) +# probabilities (0, 0.5, 1). Parsed by hand -- `CSV.jl` into a DataFrame is an +# order of magnitude slower for this shape -- and cached under --cache-dir, +# because the matrix and the kinship depend only on the panel. +function read_matrix(geno_file, cache_dir) + tag = cache_dir === nothing ? nothing : + joinpath(cache_dir, string(hash((normpath(geno_file), filesize(geno_file), + mtime(geno_file))), ".matrix")) + if tag !== nothing && isfile(tag) + return deserialize(tag) end + rows = split(strip(read(geno_file, String)), '\n') + header = split(rows[1], ',') + geno_individuals = String[strip(name) for name in header[2:end]] + marker_names = String[] + pieces = Vector{Float64}[] + for row in rows[2:end] + isempty(strip(row)) && continue + fields = split(row, ',') + line = Vector{Float64}(undef, length(geno_individuals)) + for j in 1:length(geno_individuals) + cell = strip(fields[j + 1]) + line[j] = (isempty(cell) || cell == "NA") ? NaN : + something(tryparse(Float64, cell), NaN) + end + push!(marker_names, String(fields[1])) + push!(pieces, line) + end + xs = permutedims(reduce(hcat, pieces)) + if tag !== nothing + try + mkpath(cache_dir) + serialize(tag, (marker_names, geno_individuals, xs)) + catch + end + end + return (marker_names, geno_individuals, xs) end + +raised = time() +marker_names, geno_individuals, g = read_matrix(geno_file, cache_dir) g = permutedims(g) # BulkLMM wants individuals x markers +read_seconds = time() - raised # Keep only the individuals that have both a value and genotypes, in the same # order, and fill a marker's missing probability with its column mean. @@ -149,19 +180,73 @@ println(stderr, "bulklmm: ", length(y), " individuals with a value, ", length(ma " markers, covariates: ", covar === nothing ? "none" : string(size(covar, 2))) # ---- the scan --------------------------------------------------------- -kinship = calcKinship(g) +# Kinship depends only on the cross, so it is cached beside the matrix: a second +# trait on the same panel should pay the scan and nothing else. +raised = time() +kinship = nothing +cache_key = cache_dir === nothing ? nothing : + joinpath(cache_dir, string(hash((base64encode(join(marker_names)), string(size(g)))), ".kin")) +if cache_key !== nothing && isfile(cache_key) + kinship = deserialize(cache_key) +end +if kinship === nothing + kinship = calcKinship(g) + if cache_key !== nothing + try + serialize(cache_key, kinship) + catch + # A cache that cannot be written is not a reason to fail the scan. + end + end +end +kinship_seconds = time() - raised options = Dict{Symbol, Any}(:reml => reml, :assumption => assumption, :method => method, :permutation_test => nperms > 0, :nperms => nperms, :rndseed => seed) -result = try - if covar === nothing - scan(y, g, kinship; options...) - else - scan(y, g, covar, kinship; options...) +# BulkLMM's single-trait `scan()` is serial -- only `bulkscan()` (many traits) +# and its helpers use Julia threads -- so one trait would leave every core but +# one idle except for BLAS. Each marker's LOD is independent given the null +# variance components (which depend on y and K only, not on the markers), so the +# matrix is split into blocks and the blocks are scanned in parallel. The answer +# is the same one the serial scan gives; only the wall clock changes. +nblocks = clamp(nblocks, 1, max(1, min(Threads.nthreads(), size(g, 2)))) +# BLAS threads: OpenBLAS defaults to every core in the machine (96 here), which +# on matrices this size is slower than a handful -- measured 10.5 s at 96 against +# 7.7 s at 4 for one 235x21056 scan. When the markers are split across Julia +# threads, each block gets one BLAS thread instead: oversubscribing both is what +# makes a "multicore" run look single-core. +BLAS.set_num_threads(nblocks > 1 ? 1 : min(4, Threads.nthreads())) +edges = round.(Int, range(0, size(g, 2); length = nblocks + 1)) +blocks = [(edges[i] + 1, edges[i + 1]) for i in 1:nblocks if edges[i + 1] > edges[i]] +results = Vector{Any}(undef, length(blocks)) +try + Threads.@threads for index in eachindex(blocks) + (first, last) = blocks[index] + piece = g[:, first:last] + results[index] = covar === nothing ? scan(y, piece, kinship; options...) : + scan(y, piece, covar, kinship; options...) end catch error println(stderr, "Error: ", sprint(showerror, error)) exit(2) end +result = ( + lod = vcat((r.lod for r in results)...), + b = vcat((r.b for r in results)...), + h2_null = results[1].h2_null, + sigma2_e = results[1].sigma2_e, +) +# Genome-wide thresholds need the maximum over *all* markers in each +# permutation, so combine the blocks' permutation matrices by taking their +# per-permutation maxima before the quantiles. +if nperms > 0 && all(r -> hasproperty(r, :L_perms), results) + maxima = reduce((left, right) -> max.(left, right), + (vec(maximum(r.L_perms; dims = 1)) for r in results)) + result = merge(result, (L_perms = reshape(maxima, 1, :),)) +end +scan_seconds = time() - raised +println(stderr, "bulklmm: julia threads ", Threads.nthreads(), ", blocks ", length(blocks), + ", BLAS threads ", BLAS.get_num_threads(), " | read ", round(read_seconds, digits = 1), + " s, scan ", round(scan_seconds, digits = 1), " s") h2 = hasproperty(result, :h2_null) ? result.h2_null : NaN beta = hasproperty(result, :b) ? result.b : fill(NaN, length(marker_names)) diff --git a/scripts/plot-bulklmm.py b/scripts/plot-bulklmm.py new file mode 100755 index 0000000..51c24f8 --- /dev/null +++ b/scripts/plot-bulklmm.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""LOD plot of a GN4 BulkLMM result, as one SVG panel. + +Input is the JSON response of POST /gn4/compute/bulklmm (saved as-is). The +rows are positional -- marker, chr, pos (Mb), lod, beta, h2 -- so this reads +`meta.columns` to find them. Thresholds come from the reply's own `notes` +("alpha 0.05: LOD 3.81"), which is where the endpoint puts them, so posting a +response back draws the picture a peak has to be judged against. + + python3 plot-bulklmm.py scan.json -o scan.svg --trait BXD::16627 + +Standard library only; writes plain SVG (light/dark aware). +""" +import argparse, json, math, re, sys + +def chr_key(c): + return (0, int(c), "") if c.isdigit() else (1, 0, c) + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("result", help="saved /compute/bulklmm response (JSON)") + ap.add_argument("-o", "--out", default="bulklmm_lod.svg") + ap.add_argument("--trait", default="", help="trait label for the title") + ap.add_argument("--title", default=None, help="override the title text") + a = ap.parse_args() + + doc = json.load(open(a.result)) + if "data" not in doc: + sys.exit(f"{a.result}: no 'data' member (error response?): {str(doc)[:200]}") + columns = (doc.get("meta") or {}).get("columns") or ["marker", "chr", "pos", "lod", "beta", "h2"] + rows = [r for r in doc["data"] if isinstance(r, list) and len(r) >= 4] + if not rows: + sys.exit(f"{a.result}: no rows to plot") + idx = {name: i for i, name in enumerate(columns)} + name_i, chr_i, pos_i, lod_i = idx["marker"], idx["chr"], idx["pos"], idx["lod"] + + # Thresholds: the endpoint writes them into `notes` as "alpha 0.05: LOD 3.81". + notes = " ".join(doc.get("notes", [])) + thresholds = [] + for match in re.finditer(r"alpha ([0-9.]+): LOD ([0-9]+(?:\.[0-9]+)?)", notes): + alpha, lod = float(match.group(1)), float(match.group(2)) + if abs(alpha - 0.05) < 1e-9: + thresholds.append((lod, "genome-wide (alpha 0.05)", "6 3")) + elif abs(alpha - 0.01) < 1e-9: + thresholds.append((lod, "alpha 0.01", "3 3")) + seed = re.search(r"seed (\d+)", notes) + h2 = re.search(r"heritability h2 = ([\d.eE+-]+)", notes) + nperm = re.search(r"from (\d+) rotations", notes) + + by = {} + for row in rows: + try: + lod = float(row[lod_i]) + except (TypeError, ValueError): + continue + by.setdefault(str(row[chr_i]), []).append((float(row[pos_i]), lod)) + if not by: + sys.exit(f"{a.result}: no numeric LODs") + order = sorted(by, key=chr_key) + off, tot, gap = {}, 0.0, 8 + for c in order: + by[c].sort() + off[c] = tot + tot += by[c][-1][0] + gap + + top = max(rows, key=lambda r: r[lod_i]) + top_lod = float(top[lod_i]) + ymax = max(2, math.ceil(max([top_lod] + [t[0] for t in thresholds]) + 0.5)) + W, H, L, R, T, B = 1000, 430, 62, 20, 64, 52 + pw, ph = W - L - R, H - T - B + X = lambda c, p: L + (off[c] + p) / tot * pw + Y = lambda v: T + ph - min(v, ymax) / ymax * ph + + label = a.trait or "BulkLMM scan" + title = a.title or f"{label}: LOD along the genome" + sub = "BulkLMM LMM, global kinship" + if h2: + sub += f", h2 {float(h2.group(1)):.3f}" + sub += (f", thresholds from {nperm.group(1)} rotations (seed {seed.group(1)})" + if nperm and seed else ", no permutation threshold") + + parts = [ + f'<svg xmlns="http://www.w3.org/2000/svg" width="{W}" height="{H}" ' + f'viewBox="0 0 {W} {H}" font-family="sans-serif" role="img">', + "<style>" + ".bg{fill:#fff}.fg{fill:#333}.grid{stroke:#e8e8e8}.axis{stroke:#999}" + ".pt{fill:#2b6cb0}.ptalt{fill:#4299e1}.thr{stroke:#c53030;stroke-dasharray:6 3}" + ".thr2{stroke:#dd6b20;stroke-dasharray:3 3}.lab{font-size:11px;fill:#555}" + ".ttl{font-size:15px;font-weight:bold;fill:#111}.sub{font-size:11px;fill:#666}" + "@media (prefers-color-scheme: dark){.bg{fill:#111}.fg{fill:#eee}" + ".grid{stroke:#2a2a2a}.pt{fill:#63b3ed}.ptalt{fill:#90cdf4}" + ".thr{stroke:#fc8181}.thr2{stroke:#f6ad55}.lab{fill:#bbb}.ttl{fill:#fff}.sub{fill:#999}}" + "</style>", + f'<rect class="bg" width="{W}" height="{H}"/>', + f'<text class="ttl" x="{L}" y="28">{title}</text>', + f'<text class="sub" x="{L}" y="46">{sub}</text>', + ] + for step in range(0, ymax + 1, max(1, ymax // 5)): + y = Y(step) + parts.append(f'<line class="grid" x1="{L}" y1="{y:.1f}" x2="{L + pw}" y2="{y:.1f}"/>') + parts.append(f'<text class="lab" x="{L - 8}" y="{y + 4:.1f}" text-anchor="end">{step}</text>') + parts.append(f'<line class="axis" x1="{L}" y1="{T}" x2="{L}" y2="{T + ph}"/>') + for index, c in enumerate(order): + colour = "pt" if index % 2 == 0 else "ptalt" + for pos, lod in by[c]: + parts.append(f'<circle class="{colour}" cx="{X(c, pos):.1f}" cy="{Y(lod):.1f}" r="1.6"/>') + centre = L + (off[c] + by[c][-1][0] / 2) / tot * pw + parts.append(f'<text class="lab" x="{centre:.1f}" y="{T + ph + 16}" ' + f'text-anchor="middle">{c}</text>') + for value, label_text, dash in thresholds: + y = Y(value) + parts.append(f'<line class="thr" x1="{L}" y1="{y:.1f}" x2="{L + pw}" y2="{y:.1f}" ' + f'stroke-dasharray="{dash}"/>') + parts.append(f'<text class="lab" x="{L + pw - 4}" y="{y - 4:.1f}" ' + f'text-anchor="end">{label_text}: LOD {value:.2f}</text>') + parts.append(f'<text class="lab" x="{L - 44}" y="{T - 10}">LOD</text>') + parts.append( + f'<desc>The highest peak is LOD {top_lod:.2f} on chromosome {top[chr_i]} at ' + f'{float(top[pos_i]):.1f} Mb ({top[name_i]}).' + + ("" if not thresholds else + " Genome-wide threshold (alpha 0.05) LOD " + + f"{[t[0] for t in thresholds if 'alpha 0.05' in t[1]][0]:.2f}.") + + "</desc>") + parts.append("</svg>") + open(a.out, "w").write("\n".join(parts)) + print(f"BulkLMM scan. The highest peak is LOD {top_lod:.2f} on chromosome {top[chr_i]} " + f"at {float(top[pos_i]):.1f} Mb ({top[name_i]}).") + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) |
