//! Genome scans with BulkLMM.jl: a linear mixed model per marker, with //! permutation thresholds that account for relatedness. //! //! Why this engine sits beside `/compute/gemma`: GEMMA's LOCO scan is the //! reference for a biallelic panel, but its p-values have no threshold we can //! compute -- a permutation repeats the whole scan, so `/compute/gemma` refuses //! `nperm` on purpose. R/qtl's `nperm` is the usual stand-in and it is the //! wrong null (it shuffles phenotypes as if individuals were unrelated). //! BulkLMM rotates by the eigen decomposition of the kinship matrix and //! reshuffles *that*, so its thresholds belong to the model that produced the //! LODs -- and 200 permutations cost seconds, not a second scan each. //! //! The scan itself is a global-kinship LMM, not a LOCO one. Its `h2_null` says //! how much relatedness it found; when that is ~0 the model degrades to a fixed //! effects fit, which is why its LODs can match R/qtl's on a panel where GEMMA //! differs. //! //! The work happens in `scripts/bulklmm_scan.jl`; this module only prepares //! its inputs (a genotype matrix of allele dosages, marker positions, the //! phenotype and covariates), runs it, and reads the CSV back. use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use serde_json::Value; /// Columns of a finished scan, in order. pub const COLUMNS: [&str; 6] = ["marker", "chr", "pos", "lod", "beta", "h2"]; /// Why a run failed. #[derive(Debug)] pub enum BulklmmError { /// Reading or writing one of the files this module prepares. Io(String), /// Julia or the driver failed; carries their own words. CommandFailed(String), } impl std::fmt::Display for BulklmmError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { BulklmmError::Io(what) | BulklmmError::CommandFailed(what) => write!(f, "{what}"), } } } /// Why Julia stopped, in its own words. /// /// Julia writes the sentence that matters on a line starting `ERROR:`, with /// `Stacktrace:` and frame lines under it, and — for a missing package — a /// `- Run \`import Pkg; ...\`` hint after that. Taking the *last* line /// instead reports `in expression starting at scripts/bulklmm_scan.jl:26`, /// which names neither the package nor the reason: the same defect as /// gemma-wrapper's Ruby backtrace, in our own error handling. So: the first /// `ERROR:` line, plus its hint line when there is one, and the last line /// only as a fallback for a driver that failed without one. fn julia_complaint(stderr: &str, stdout: &str) -> String { let lines: Vec<&str> = stderr .lines() .chain(stdout.lines()) .map(str::trim) .filter(|line| !line.is_empty()) .collect(); if let Some(index) = lines.iter().position(|line| { let lowered = line.to_ascii_lowercase(); lowered.starts_with("error") || lowered.starts_with("loaderror") }) { let mut complaint = lines[index].to_string(); if let Some(hint) = lines.get(index + 1).filter(|line| line.starts_with("- ")) { complaint.push(' '); complaint.push_str(hint.trim_start_matches("- ").trim()); } return complaint; } lines .last() .map(|line| (*line).to_string()) .unwrap_or_else(|| "BulkLMM failed without saying why".to_string()) } fn io_err(path: &Path, error: std::io::Error) -> BulklmmError { BulklmmError::Io(format!("{}: {error}", path.display())) } /// A finished scan: one row per marker, plus permutation thresholds. pub struct Output { pub rows: Value, pub markers: usize, /// `alpha -> LOD` from the rotations, when permutations were requested. pub thresholds: BTreeMap, /// What BulkLMM estimated under the null: heritability, and its variance. pub h2_null: Option, /// Residual variance under the null, when the driver printed it. pub sigma2_e: Option, pub command: String, pub summary: String, } /// Convert a BIMBAM pair into what the driver reads: a matrix of allele /// dosages with the individuals named in its header, and a marker map. /// /// BIMBAM carries no individual names -- the columns are positions in the /// panel -- so the names come from the caller (`individuals`, in the same /// order as the prepared file's columns). pub fn write_inputs( bimbam_geno: &Path, bimbam_snps: &Path, individuals: &[String], geno_csv: &Path, markers_csv: &Path, ) -> Result { let text = std::fs::read_to_string(bimbam_geno).map_err(|e| io_err(bimbam_geno, e))?; let mut out = String::from("marker,"); out.push_str(&individuals.join(",")); out.push('\n'); let mut markers = 0usize; for line in text.lines() { if line.trim().is_empty() || line.starts_with('#') { continue; } // `marker, X, Y, dosage, dosage, ...` let fields: Vec<&str> = line.split(',').map(str::trim).collect(); if fields.len() < 4 { continue; } // Only the individuals the caller kept, in their order. if fields.len() - 3 < individuals.len() { continue; } out.push_str(fields[0]); for value in fields[3..3 + individuals.len()].iter() { out.push(','); // BIMBAM spells missing as NA; the driver reads an empty cell. if matches!(*value, "NA" | "na" | "nan" | "-9") { continue; } out.push_str(value); } out.push('\n'); markers += 1; } std::fs::write(geno_csv, out).map_err(|e| io_err(geno_csv, e))?; // The SNP map is `marker, position in bp, chromosome`; the driver wants Mb. let snps = std::fs::read_to_string(bimbam_snps).map_err(|e| io_err(bimbam_snps, e))?; let mut map = String::from("marker,chr,pos\n"); for line in snps.lines() { let fields: Vec<&str> = line.split(',').map(str::trim).collect(); if fields.len() < 3 { continue; } let megabases = fields[1].parse::().map(|bp| bp / 1e6).unwrap_or(0.0); map.push_str(&format!("{},{},{megabases}\n", fields[0], fields[2])); } std::fs::write(markers_csv, map).map_err(|e| io_err(markers_csv, e))?; Ok(markers) } /// Read the driver's CSV back, and the threshold file when there is one. fn parse_output(contents: &str, thresholds: Option<&str>) -> (Value, usize, BTreeMap, Option) { let mut rows = Vec::new(); let mut h2 = None; for (index, line) in contents.lines().enumerate() { let line = line.trim_end_matches('\r'); if line.trim().is_empty() || (index == 0 && line.starts_with("marker")) { continue; } let fields: Vec<&str> = line.split(',').collect(); if fields.len() < 6 { continue; } // "NaN" parses as an f64 in Rust; a marker the model could not fit is // not a LOD of zero and must not be reported as one. let lod = fields[3].parse::().ok().filter(|lod| lod.is_finite()); let beta = fields[4].parse::().ok().filter(|beta| beta.is_finite()); let h2_value = fields[6].parse::().ok().filter(|h2| h2.is_finite()); if h2.is_none() { h2 = h2_value; } let Some(lod) = lod else { continue }; rows.push(serde_json::json!([ fields[0], fields[1], fields[2].parse::().unwrap_or(0.0), lod, beta, h2_value, ])); } let mut map = BTreeMap::new(); if let Some(text) = thresholds { for (index, line) in text.lines().enumerate() { let line = line.trim(); if line.is_empty() || (index == 0 && line.starts_with("alpha")) { continue; } let fields: Vec<&str> = line.split(',').collect(); if fields.len() < 2 { continue; } if let Ok(lod) = fields[1].parse::() { map.insert(fields[0].to_string(), lod); } } } let count = rows.len(); (Value::Array(rows), count, map, h2) } /// Run one scan. #[allow(clippy::too_many_arguments)] pub async fn compute( julia: &str, project: &str, depot: &str, script: &str, work_dir: &Path, genotype_csv: &Path, markers_csv: &Path, pheno_path: &Path, covar_path: Option<&Path>, nperm: u32, seed: u32, reml: bool, assumption: &str, threads: u32, blocks: u32, ) -> Result { let unique = format!( "{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_nanos()) .unwrap_or(0) ); let result_path = work_dir.join(format!("{unique}-bulklmm.csv")); let perm_path = work_dir.join(format!("{unique}-bulklmm-perm.csv")); let threads_str = threads.to_string(); let blocks_str = blocks.to_string(); let threads_for_env = threads.to_string(); let nperm_str = nperm.to_string(); let seed_str = seed.to_string(); let mut args: Vec = vec![ "--threads".to_string(), threads_str, "--startup-file=no".to_string(), // `--project=`, not `--project `: Julia 1.8 takes the // space-separated form as a program argument and then silently runs // nothing. format!("--project={project}"), script.to_string(), "--pheno".to_string(), pheno_path.to_string_lossy().into_owned(), "--geno".to_string(), genotype_csv.to_string_lossy().into_owned(), "--markers".to_string(), markers_csv.to_string_lossy().into_owned(), "--out".to_string(), result_path.to_string_lossy().into_owned(), "--assumption".to_string(), assumption.to_string(), "--blocks".to_string(), blocks_str, "--seed".to_string(), seed_str, ]; if nperm > 0 { args.push("--nperm".to_string()); args.push(nperm_str); args.push("--perm-out".to_string()); args.push(perm_path.to_string_lossy().into_owned()); } if reml { args.push("--reml".to_string()); } if let Some(covar) = covar_path { args.push("--covar".to_string()); args.push(covar.to_string_lossy().into_owned()); } let borrowed: Vec<&str> = args.iter().map(String::as_str).collect(); let command = format!("JULIA_DEPOT_PATH={depot} {julia} {}", borrowed.join(" ")); let output = crate::streaming::run_streaming_with_env( julia, &borrowed, None, // Offline: a request must never download a package. The depot is // prepared once, deliberately, and it is small (79 packages, 139 MB, // no GPU/CUDA stack -- BulkLMM's closure is CSV, DataFrames, // Distributions, Optim and what those need). &[ ("JULIA_DEPOT_PATH", depot), ("JULIA_PKG_OFFLINE", "true"), ("JULIA_NUM_THREADS", threads_for_env.as_str()), ], ) .await .map_err(|error| BulklmmError::Io(format!("{julia}: {error}")))?; if !output.status.success() { return Err(BulklmmError::CommandFailed(julia_complaint( &String::from_utf8_lossy(&output.stderr), &String::from_utf8_lossy(&output.stdout), ))); } let contents = std::fs::read_to_string(&result_path).map_err(|e| io_err(&result_path, e))?; let perm_contents = if nperm > 0 { std::fs::read_to_string(&perm_path).ok() } else { None }; let (rows, markers, thresholds, h2) = parse_output(&contents, perm_contents.as_deref()); // Every line the driver printed about the run, minus the "wrote" one: the // threads/blocks/read/scan breakdown and the thresholds are two separate // lines and both belong in the reply. let summary = String::from_utf8_lossy(&output.stderr) .lines() .map(str::trim) .filter(|line| line.starts_with("bulklmm:") && !line.starts_with("bulklmm: wrote")) .map(|line| line.trim_start_matches("bulklmm: ")) .collect::>() .join("; "); // "h2 0.000000, sigma2_e 0.094253, n 35" from the driver's own line. let components = String::from_utf8_lossy(&output.stderr) .lines() .find(|line| line.contains("variance components under the null")) .map(str::to_string) .unwrap_or_default(); let pick = |key: &str| -> Option { let rest = components.split(key).nth(1)?; rest.trim_start() .trim_start_matches('=') .trim_start() .split(',') .next()? .trim() .parse::() .ok() }; let sigma2_e = pick("sigma2_e"); Ok(Output { rows, markers, thresholds, h2_null: h2, sigma2_e, command, summary, }) } /// Where a run's scratch files live. pub fn work_dir(cache_dir: &str) -> Result { let dir = Path::new(cache_dir); std::fs::create_dir_all(dir).map_err(|e| io_err(dir, e))?; Ok(dir.to_path_buf()) } #[cfg(test)] mod tests { use super::*; #[test] fn a_bimbam_pair_becomes_the_matrix_the_driver_reads() { let dir = std::env::temp_dir().join(format!("gn4-bulklmm-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let geno = dir.join("pair_geno.txt"); let snps = dir.join("pair_snps.txt"); std::fs::write( &geno, "m1, X, Y, 1, 0.5, 0\nm2, X, Y, 0, NA, 1\n", ) .unwrap(); std::fs::write(&snps, "m1, 3001490, 1\nm2, 5653007, 2\n").unwrap(); let individuals = vec!["BXD1".to_string(), "BXD2".to_string(), "BXD3".to_string()]; let geno_csv = dir.join("geno.csv"); let markers_csv = dir.join("markers.csv"); let markers = write_inputs(&geno, &snps, &individuals, &geno_csv, &markers_csv).unwrap(); assert_eq!(markers, 2); let matrix = std::fs::read_to_string(&geno_csv).unwrap(); assert_eq!(matrix.lines().next().unwrap(), "marker,BXD1,BXD2,BXD3"); // A missing dosage is an empty cell, not "NA". assert_eq!(matrix.lines().nth(2).unwrap(), "m2,0,,1"); let map = std::fs::read_to_string(&markers_csv).unwrap(); assert!(map.contains("m2,2,5.653007"), "{map}"); std::fs::remove_dir_all(&dir).ok(); } #[test] fn the_scan_csv_and_the_thresholds_are_read_back() { let contents = "marker,chr,pos,lod,beta,se,h2\n\ m1,1,3.0,9.59,0.0034,NA,0.0\n\ m2,1,4.0,NaN,NaN,NA,0.0\n"; let thresholds = "alpha,lod\n0.1,3.501\n0.05,3.959\n"; let (rows, count, map, h2) = parse_output(contents, Some(thresholds)); assert_eq!(count, 1, "a NaN LOD row is dropped, not reported as zero"); let row = &rows.as_array().unwrap()[0]; assert_eq!(row[0], serde_json::json!("m1")); assert_eq!(row[3], serde_json::json!(9.59)); assert_eq!(map["0.05"], 3.959); assert_eq!(h2, Some(0.0)); } /// The failure that prompted this: a Julia project without CSV. The last /// line names a line number and nothing else; the `ERROR:` line names the /// package, and the hint says what to run. #[test] fn the_complaint_is_julias_error_line_and_its_hint() { let stderr = "ERROR: LoadError: ArgumentError: Package CSV not found in current path.\n\ - Run `import Pkg; Pkg.add(\"CSV\")` to install the CSV package.\n\ Stacktrace:\n\ [1] macro expansion\n\ @ ./loading.jl:1166 [inlined]\n\ in expression starting at scripts/bulklmm_scan.jl:26\n"; let complaint = julia_complaint(stderr, ""); assert!(complaint.starts_with("ERROR: LoadError: ArgumentError: Package CSV not found"), "{complaint}"); assert!(complaint.contains("Pkg.add"), "{complaint}"); } #[test] fn a_driver_that_fails_silently_says_so() { assert_eq!( julia_complaint("", ""), "BulkLMM failed without saying why" ); } #[test] fn without_an_error_line_the_last_line_is_better_than_nothing() { assert_eq!( julia_complaint("something went sideways\n", ""), "something went sideways" ); } }