#!/usr/bin/env julia # Genome scan with BulkLMM.jl: a linear mixed model per marker, with kinship. # # The counterpart of scripts/qtl2_scan.R for the GEMMA side: same trait values, # same markers, a global kinship matrix, and -- the reason it exists -- # permutation thresholds that account for relatedness (BulkLMM rotates by the # eigen decomposition of K instead of shuffling phenotypes), which GEMMA cannot # do here and R/qtl does with the wrong null. # # Inputs are plain files this API writes: # --pheno Sampletrait, one row per individual with a value # --geno CSV matrix, rows = individuals, columns = markers, entries are # allele probabilities (0, 0.5, 1) -- our BIMBAM dosages # --markers CSV: marker,chr,pos (Mb) # --covar Samplec1c2..., optional # # Output is a CSV, one row per marker: # marker,chr,pos,lod,beta,se,h2 # and, with --perm-out, a CSV of permutation thresholds: alpha,lod # # Usage: # julia --threads 4 scripts/bulklmm_scan.jl --pheno p.tsv --geno g.csv \ # --markers m.csv --out scan.csv [--nperm 1000 --perm-out t.csv] using BulkLMM using CSV using DataFrames using DelimitedFiles using Serialization using Statistics using Base64 using LinearAlgebra function fail(message) println(stderr, "Error: ", message) exit(1) end function argument(name, default=nothing) index = findfirst(==(name), ARGS) if index === nothing return default end index < length(ARGS) || fail("$name needs a value") return ARGS[index + 1] end flag(name) = name in ARGS pheno_file = argument("--pheno") geno_file = argument("--geno") markers_file = argument("--markers") out_file = argument("--out") covar_file = argument("--covar") perm_out = argument("--perm-out") nperms = parse(Int, argument("--nperm", "0")) nblocks = parse(Int, argument("--blocks", string(Threads.nthreads()))) cache_dir = argument("--cache-dir") seed = parse(Int, argument("--seed", "0")) reml = flag("--reml") assumption = argument("--assumption", "null") method = argument("--method", "qr") (pheno_file === nothing || geno_file === nothing || markers_file === nothing || out_file === nothing) && fail("--pheno, --geno, --markers and --out are required") # ---- inputs ----------------------------------------------------------- # The phenotype file is this API's shape: a header and one row per individual # with a value; `x` (or an empty cell) is missing. pheno_lines = filter(line -> !isempty(strip(line)), split(read(pheno_file, String), '\n')) header = split(pheno_lines[1], '\t') length(header) >= 2 || fail("$(pheno_file): needs a header and at least one trait column") individuals = String[] values = Float64[] for line in pheno_lines[2:end] fields = split(line, '\t') length(fields) >= 2 || continue raw = strip(fields[2]) (isempty(raw) || raw in ("x", "X", "NA", "na")) && continue parsed = tryparse(Float64, raw) parsed === nothing && continue push!(individuals, String(strip(fields[1]))) push!(values, parsed) end length(individuals) >= 3 || fail("$(pheno_file): fewer than three individuals with a value") y = values # The genotype matrix: rows are individuals, columns markers, entries allele # probabilities (0, 0.5, 1). Parsed by hand -- `CSV.jl` into a DataFrame is an # order of magnitude slower for this shape -- and cached under --cache-dir, # because the matrix and the kinship depend only on the panel. function read_matrix(geno_file, cache_dir) tag = cache_dir === nothing ? nothing : joinpath(cache_dir, string(hash((normpath(geno_file), filesize(geno_file), mtime(geno_file))), ".matrix")) if tag !== nothing && isfile(tag) return deserialize(tag) end rows = split(strip(read(geno_file, String)), '\n') header = split(rows[1], ',') geno_individuals = String[strip(name) for name in header[2:end]] marker_names = String[] pieces = Vector{Float64}[] for row in rows[2:end] isempty(strip(row)) && continue fields = split(row, ',') line = Vector{Float64}(undef, length(geno_individuals)) for j in 1:length(geno_individuals) cell = strip(fields[j + 1]) line[j] = (isempty(cell) || cell == "NA") ? NaN : something(tryparse(Float64, cell), NaN) end push!(marker_names, String(fields[1])) push!(pieces, line) end xs = permutedims(reduce(hcat, pieces)) if tag !== nothing try mkpath(cache_dir) serialize(tag, (marker_names, geno_individuals, xs)) catch end end return (marker_names, geno_individuals, xs) end raised = time() marker_names, geno_individuals, g = read_matrix(geno_file, cache_dir) g = permutedims(g) # BulkLMM wants individuals x markers read_seconds = time() - raised # Keep only the individuals that have both a value and genotypes, in the same # order, and fill a marker's missing probability with its column mean. position = Dict(name => index for (index, name) in enumerate(geno_individuals)) keep_value = [i for (i, name) in enumerate(individuals) if haskey(position, name)] length(keep_value) >= 3 || fail("fewer than three phenotyped individuals are in the genotype matrix") rows_kept = [position[individuals[i]] for i in keep_value] y = y[keep_value] g = g[rows_kept, :] for column in 1:size(g, 2) column_values = @view g[:, column] if any(isnan, column_values) average = mean(filter(!isnan, column_values)) column_values[isnan.(column_values)] .= average end end # Marker positions, in the matrix's column order. marker_map = Dict{String, Tuple{String, Float64}}() for (index, line) in enumerate(split(read(markers_file, String), '\n')) isempty(strip(line)) && continue fields = split(strip(line), ',') length(fields) >= 3 || continue index == 1 && fields[1] == "marker" && continue marker_map[String(fields[1])] = (String(fields[2]), something(tryparse(Float64, fields[3]), 0.0)) end # Covariates, aligned onto the kept individuals. covar = nothing if covar_file !== nothing lines = filter(line -> !isempty(strip(line)), split(read(covar_file, String), '\n')) cov_header = split(lines[1], '\t') cov_individuals = String[strip(split(line, '\t')[1]) for line in lines[2:end]] position_cov = Dict(name => index for (index, name) in enumerate(cov_individuals)) matrix = zeros(Float64, length(keep_value), length(cov_header) - 1) for (row, i) in enumerate(eachindex(keep_value)) name = individuals[i] haskey(position_cov, name) || continue fields = split(lines[position_cov[name] + 1], '\t') for column in 2:length(cov_header) raw = strip(fields[column]) matrix[row, column - 1] = (isempty(raw) || raw in ("x", "NA")) ? 0.0 : something(tryparse(Float64, raw), 0.0) end end covar = matrix end println(stderr, "bulklmm: ", length(y), " individuals with a value, ", length(marker_names), " markers, covariates: ", covar === nothing ? "none" : string(size(covar, 2))) # ---- the scan --------------------------------------------------------- # Kinship depends only on the cross, so it is cached beside the matrix: a second # trait on the same panel should pay the scan and nothing else. raised = time() kinship = nothing cache_key = cache_dir === nothing ? nothing : joinpath(cache_dir, string(hash((base64encode(join(marker_names)), string(size(g)))), ".kin")) if cache_key !== nothing && isfile(cache_key) kinship = deserialize(cache_key) end if kinship === nothing kinship = calcKinship(g) if cache_key !== nothing try serialize(cache_key, kinship) catch # A cache that cannot be written is not a reason to fail the scan. end end end kinship_seconds = time() - raised options = Dict{Symbol, Any}(:reml => reml, :assumption => assumption, :method => method, :permutation_test => nperms > 0, :nperms => nperms, :rndseed => seed) # BulkLMM's single-trait `scan()` is serial -- only `bulkscan()` (many traits) # and its helpers use Julia threads -- so one trait would leave every core but # one idle except for BLAS. Each marker's LOD is independent given the null # variance components (which depend on y and K only, not on the markers), so the # matrix is split into blocks and the blocks are scanned in parallel. The answer # is the same one the serial scan gives; only the wall clock changes. nblocks = clamp(nblocks, 1, max(1, min(Threads.nthreads(), size(g, 2)))) # BLAS threads: OpenBLAS defaults to every core in the machine (96 here), which # on matrices this size is slower than a handful -- measured 10.5 s at 96 against # 7.7 s at 4 for one 235x21056 scan. When the markers are split across Julia # threads, each block gets one BLAS thread instead: oversubscribing both is what # makes a "multicore" run look single-core. BLAS.set_num_threads(nblocks > 1 ? 1 : min(4, Threads.nthreads())) edges = round.(Int, range(0, size(g, 2); length = nblocks + 1)) blocks = [(edges[i] + 1, edges[i + 1]) for i in 1:nblocks if edges[i + 1] > edges[i]] results = Vector{Any}(undef, length(blocks)) try Threads.@threads for index in eachindex(blocks) (first, last) = blocks[index] piece = g[:, first:last] results[index] = covar === nothing ? scan(y, piece, kinship; options...) : scan(y, piece, covar, kinship; options...) end catch error println(stderr, "Error: ", sprint(showerror, error)) exit(2) end result = ( lod = vcat((r.lod for r in results)...), b = vcat((r.b for r in results)...), h2_null = results[1].h2_null, sigma2_e = results[1].sigma2_e, ) # Genome-wide thresholds need the maximum over *all* markers in each # permutation, so combine the blocks' permutation matrices by taking their # per-permutation maxima before the quantiles. if nperms > 0 && all(r -> hasproperty(r, :L_perms), results) maxima = reduce((left, right) -> max.(left, right), (vec(maximum(r.L_perms; dims = 1)) for r in results)) result = merge(result, (L_perms = reshape(maxima, 1, :),)) end h2 = hasproperty(result, :h2_null) ? result.h2_null : NaN sigma2_e = hasproperty(result, :sigma2_e) ? result.sigma2_e : NaN scan_seconds = time() - raised # The variance components, so a caller can see whether the kinship term did # anything: h2 estimate = sigma2_g / (sigma2_g + sigma2_e) under the null model. println(stderr, "bulklmm: variance components under the null: h2 ", round(h2, digits = 6), ", sigma2_e ", round(sigma2_e, digits = 6), ", n ", length(y)) println(stderr, "bulklmm: julia threads ", Threads.nthreads(), ", blocks ", length(blocks), ", BLAS threads ", BLAS.get_num_threads(), " | read ", round(read_seconds, digits = 1), " s, scan ", round(scan_seconds, digits = 1), " s") beta = hasproperty(result, :b) ? result.b : fill(NaN, length(marker_names)) se = hasproperty(result, :SE) ? result.SE : nothing open(out_file, "w") do handle println(handle, "marker,chr,pos,lod,beta,se,h2") for (index, marker) in enumerate(marker_names) chr, pos = get(marker_map, marker, ("", 0.0)) b = length(beta) >= index ? beta[index] : NaN s = se === nothing ? "NA" : string(se[index]) println(handle, join((marker, chr, string(pos), string(result.lod[index]), string(b), s, string(h2)), ",")) end end if perm_out !== nothing && nperms > 0 && hasproperty(result, :L_perms) alphas = [0.10, 0.05, 0.01] found = get_thresholds(result.L_perms, alphas) thresholds = hasproperty(found, :thrs) ? collect(found.thrs) : collect(found) open(perm_out, "w") do handle println(handle, "alpha,lod") for (alpha, lod) in zip(alphas, thresholds) println(handle, join((string(alpha), string(lod)), ",")) end end println(stderr, "bulklmm: permutation thresholds ", join(round.(thresholds, digits = 3), " / ")) end println(stderr, "bulklmm: wrote ", out_file)