//! Core logic for POST /compute/gemma: compose and run gemma-wrapper //! against a genotype file, a SNP annotation file and trait values, and //! parse the resulting GEMMA association output into QTL mapping rows. //! //! Mirrors gn3/computations/gemma.py and gn3/api/gemma.py //! (GeneNetwork3), but synchronously: axum/tokio can await an external //! process directly, so there is no need for GeneNetwork3's Redis job //! queue. The actual invocation also differs from GeneNetwork3's in two //! respects, both found by running the real gemma-wrapper/GEMMA binaries //! against the BXD fixtures rather than trusting GeneNetwork3's own //! (entirely @unittest.skip'd) integration tests -- see `compute_gwa` //! and `compute` below. Progress-streaming (`run_id`/`status`) is //! shared infrastructure; see `crate::streaming`. use std::io; use std::path::Path; use std::process::Output; use serde_json::Value; use crate::streaming::{self, RunLogError}; /// One phenotype value per strain (genotype-file order), or `None` for /// a missing ("NA") sample. pub type TraitValues = Vec>; #[derive(Debug)] pub enum GemmaError { /// Failed to spawn or run a child process. /// A filesystem operation failed (writing the phenotype file, /// reading GEMMA's output, ...) -- not the same thing as a subprocess /// that would not start, which is `Spawn`. Io(io::Error), /// A subprocess could not be started at all (the binary is missing, /// or not executable). Names it, because "failed to run /// gemma-wrapper" was printed for a missing `ccwl` too. Spawn { command: String, error: io::Error }, /// gemma-wrapper exited non-zero; carries its stderr. CommandFailed { step: &'static str, stderr: String }, /// gemma-wrapper's own --json stdout record was not valid JSON, or /// reported an error (its own `errno` field was non-zero). WrapperError { step: &'static str, detail: String }, /// The real GEMMA `.assoc.txt` file gemma-wrapper pointed to could /// not be found or read. AssocFile(io::Error), /// gemma-wrapper's GWA output archive did not list any files. NoAssocFiles, /// `run_id` already has a log file (a run with that id is in /// progress, or finished and was never polled/cleaned up). Caught /// before any subprocess runs, so this never silently truncates /// another run's in-progress log. RunIdInUse(String), /// `ravanan` (see `compute_ravanan`) exited non-zero; carries its /// stderr. RavananFailed(String), /// `ravanan`'s own stdout (the CWL outputs record) was not valid /// JSON, or didn't carry the output this code expects. RavananOutput(String), /// The compiled workflow and this build disagree about what the /// workflow takes: the `gemma-loco.scm` in the deployment's checkout /// is not the one this binary was written against. Caught before /// ravanan runs, because cwltool's own message for this ("Type /// mismatch for input X; expected File but got null") says nothing /// about why -- and the why is "upgrade the checkout and the binary /// together". WorkflowMismatch(String), } impl std::fmt::Display for GemmaError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { GemmaError::Io(e) => write!(f, "I/O error: {e}"), GemmaError::Spawn { command, error } => { write!(f, "failed to run {command}: {error}") } GemmaError::CommandFailed { step, stderr } => { write!(f, "gemma-wrapper ({step}) failed: {stderr}") } GemmaError::WrapperError { step, detail } => { write!(f, "gemma-wrapper ({step}) reported an error: {detail}") } GemmaError::AssocFile(e) => write!(f, "could not read GEMMA association output: {e}"), GemmaError::NoAssocFiles => { write!( f, "gemma-wrapper's response listed no association output files" ) } GemmaError::RunIdInUse(id) => { write!(f, "run id '{id}' is already in use; pick a different id") } GemmaError::RavananFailed(stderr) => write!(f, "ravanan failed: {stderr}"), GemmaError::RavananOutput(detail) => { write!(f, "could not read ravanan's output: {detail}") } GemmaError::WorkflowMismatch(detail) => write!( f, "the workflow source and this build disagree about the workflow's inputs: {detail}. \ cwl/gemma/gemma-loco.scm comes from the deployment's checkout, not from the \ binary, so put both from the same commit (or set GEMMA_RAVANAN_WORKFLOW to the \ matching file)." ), } } } impl From for GemmaError { fn from(e: RunLogError) -> Self { match e { RunLogError::AlreadyInUse(id) => GemmaError::RunIdInUse(id), RunLogError::Io(e) => GemmaError::Io(e), } } } /// The marker with the smallest p_lrt in GEMMA's own positional rows (the /// last column), if there is one: the GWAS peak, which is what a PheWAS /// wants to be centred on. pub fn peak_marker(rows: &serde_json::Value) -> Option { rows.as_array()? .iter() .filter_map(|row| { let row = row.as_array()?; let p = row.last()?.as_f64()?; let name = row.get(1)?.as_str()?.to_string(); Some((p, name)) }) .min_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)) .map(|(_, name)| name) } /// Write `values` as a GEMMA/BIMBAM phenotype file: one value per line, /// "NA" for a missing sample. Mirrors /// gn3.computations.gemma.generate_pheno_txt_file, minus its hash-based /// cache filename (the caller picks the path; see `compute`). pub fn write_pheno_file(path: &Path, values: &TraitValues) -> io::Result<()> { let mut contents = String::new(); for value in values { match value { Some(v) => contents.push_str(&v.to_string()), None => contents.push_str("NA"), } contents.push('\n'); } std::fs::write(path, contents) } /// The same file with the trait's *pattern* but constant values: what the /// kinship step is given, so that K is a function of the animal set alone. /// /// GEMMA reads `-p` in a `-gk` run only to know which individuals are /// analysed; the matrix is computed from the genotypes. Verified by running /// `-gk` on a real trait and on its constant-valued twin: the `.cXX.txt` files /// are byte-identical. pub fn write_constant_pheno_file(path: &Path, values: &TraitValues) -> io::Result<()> { let constant: TraitValues = values .iter() .map(|value| value.map(|_| 1.0)) .collect(); write_pheno_file(path, &constant) } /// The one line worth quoting from a failed gemma-wrapper run. /// /// The wrapper prints GEMMA's own diagnostics in the middle of its /// per-chromosome chatter and then bails out with a banner that says /// nothing ("gemma-wrapper bailed out with pid ... exit 1"). What a caller /// needs is GEMMA's sentence -- `ERROR: Enforce failed for N>0 ...` -- so /// that a panel GEMMA cannot fit is not mistaken for a broken API. fn gemma_complaint(stdout: &str, stderr: &str) -> String { let interesting = |line: &&str| { let line = line.trim(); // The wrapper's own banner and Ruby backtrace are not the answer. if line.contains("gemma-wrapper") || line.contains(".rb:") { return false; } line.contains("ERROR") || line.contains("error!") || line.starts_with("****") || line.contains("Enforce failed") || line.contains("not enough") }; let last = stdout .lines() .chain(stderr.lines()) .filter(interesting) .next_back() .map(str::trim) .unwrap_or_default(); if last.is_empty() { // Nothing GEMMA said: hand back the tail, which at least names the // wrapper's own complaint. let tail: Vec<&str> = stderr .lines() .chain(stdout.lines()) .filter(|line| !line.trim().is_empty()) .collect(); return tail .iter() .rev() .take(2) .rev() .cloned() .collect::>() .join(" / "); } last.to_string() } fn parse_wrapper_output(step: &'static str, output: Output) -> Result { if !output.status.success() { let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); return Err(GemmaError::CommandFailed { step, stderr: gemma_complaint(&stdout, &stderr), }); } let stdout = String::from_utf8_lossy(&output.stdout); let record: Value = serde_json::from_str(&stdout).map_err(|e| GemmaError::WrapperError { step, detail: format!("could not parse gemma-wrapper's --json output: {e}"), })?; if record.get("errno").and_then(Value::as_i64).unwrap_or(0) != 0 { return Err(GemmaError::WrapperError { step, detail: record .get("error") .and_then(Value::as_str) .unwrap_or("gemma-wrapper reported a non-zero errno") .to_string(), }); } Ok(record) } /// Run gemma-wrapper's `--json` K-matrix step and return its stdout /// record (parsed JSON). /// /// `--loco` (bare, no chromosome list) is required here: confirmed by /// actually running gemma-wrapper against the BXD fixtures, a plain /// (non-LOCO) `-gk` only ever leaves its kinship matrix inside a /// `.tar.xz` archive in `--cache-dir`, with no loose file at the path /// the non-LOCO branch of gemma-wrapper's own GWA step later assumes -- /// that combination errors out (`gemma: ... open file ... CheckParam`). /// `--loco` is the combination gemma-wrapper's own USAGE banner /// documents and the one whose GWA-side code path actually unpacks the /// archive it needs (see `compute_gwa` and `compute`). Passing it with /// no chromosome list makes gemma-wrapper derive the full chromosome /// list from the SNP annotation file itself. async fn compute_kinship( gemma_wrapper_cmd: &str, cache_dir: &str, genotype_file: &str, pheno_file: &str, snps_file: &str, log_path: Option<&Path>, ) -> Result { let args = kinship_args(cache_dir, genotype_file, pheno_file, snps_file); let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path) .await .map_err(|error| GemmaError::Spawn { command: gemma_wrapper_cmd.to_string(), error, })?; parse_wrapper_output("kinship", output) } /// Run gemma-wrapper's `--json` GWA (association) step, given the /// K-step's JSON record written to `k_json_path` (gemma-wrapper's /// `--input` takes a file path, not stdin), and return its stdout /// record. `--loco` is required here for the same reason as in /// `compute_kinship`. /// /// Note: unlike GeneNetwork3's `generate_gemma_cmd` /// (gn3/computations/gemma.py), which always appends a literal "-gk" to /// the forwarded GEMMA args regardless of whether an LMM run was /// requested, this passes `-lmm` without `-gk`, matching gemma-wrapper's /// own documented usage (its USAGE banner in bin/gemma-wrapper). /// gemma-wrapper decides K-only vs. GWA purely by whether "-gk" is /// present in those args (`DO_COMPUTE_KINSHIP = /// gemma_args.include?("-gk")`); passing both flags together, as /// GeneNetwork3 does, makes every one of its calls a K-only run, and the /// `-lmm` value is silently ignored. That GeneNetwork3 code path has no /// non-skipped test covering it (tests/integration/test_gemma.py is /// `@unittest.skip`), so it is not knowingly replicated here. async fn compute_gwa( gemma_wrapper_cmd: &str, cache_dir: &str, genotype_file: &str, pheno_file: &str, snps_file: &str, lmm: u8, k_json_path: &str, covariate_file: Option<&str>, log_path: Option<&Path>, ) -> Result { let lmm_str = lmm.to_string(); let args = gwa_args( cache_dir, genotype_file, pheno_file, snps_file, covariate_file, &lmm_str, k_json_path, ); let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path) .await .map_err(|error| GemmaError::Spawn { command: gemma_wrapper_cmd.to_string(), error, })?; parse_wrapper_output("gwa", output) } /// The association output basenames gemma-wrapper's GWA-step JSON /// record points at: `record["files"]` is a list of /// `[chr, log_basename, data_basename]`, one entry per chromosome in a /// `--loco` run (see `compute`). fn assoc_file_basenames(record: &Value) -> Vec<&str> { record .get("files") .and_then(Value::as_array) .map(|files| { files .iter() .filter_map(|entry| entry.as_array()?.get(2)?.as_str()) .collect() }) .unwrap_or_default() } /// The per-chromosome `.log.txt` basenames gemma-wrapper recorded for a /// step, taken from the second element of every `files` entry. fn log_file_basenames(record: &Value) -> Vec<&str> { record .get("files") .and_then(Value::as_array) .map(|files| { files .iter() .filter_map(|entry| entry.as_array()?.get(1)?.as_str()) .collect() }) .unwrap_or_default() } /// GEMMA's own variance components for one chromosome, as `-pve` writes /// them: /// /// ```text /// ## pve estimate in the null model = 0.612 /// ## se(pve) in the null model = 0.031 /// ## vg estimate in the null model = 1.4 /// ## ve estimate in the null model = 0.9 /// ``` /// /// All four are optional individually: GEMMA prints `-nan` for `se(pve)` /// when the estimate sits on a boundary (`h2 = 0` or `1`), and older /// builds print nothing at all without `-pve`. #[derive(Debug, Clone, PartialEq)] pub struct VarianceComponents { pub pve: Option, pub se_pve: Option, pub vg: Option, pub ve: Option, } fn parse_variance_components(log: &str) -> VarianceComponents { fn value_after<'a>(log: &'a str, marker: &str) -> Option<&'a str> { log.lines() .find_map(|line| line.trim().strip_prefix(marker)) .map(|rest| rest.trim_start_matches([' ', '=']).trim()) } fn number(text: Option<&str>) -> Option { // GEMMA prints `-nan` (and `nan`) on a boundary, and Rust parses both // as a NaN f64 -- so the finite check is what rejects them. A boundary // estimate has no standard error to report, and `None` says so; a NaN // in the reply would be worse than nothing. text.and_then(|t| t.parse::().ok()) .filter(|value| value.is_finite()) } VarianceComponents { pve: number(value_after(log, "## pve estimate in the null model")), se_pve: number(value_after(log, "## se(pve) in the null model")), vg: number(value_after(log, "## vg estimate in the null model")), ve: number(value_after(log, "## ve estimate in the null model")), } } /// Read the variance components gemma-wrapper left beside the archive, one /// per chromosome. Missing logs (an older wrapper, a cache hit from before /// the logs were kept) yield an empty vector rather than an error: the /// association scan is the result, these are commentary on it. fn read_variance_components( cache_dir: &str, record: &Value, ) -> Vec<(Option, VarianceComponents)> { let chromosomes: Vec> = record .get("files") .and_then(Value::as_array) .map(|files| { files .iter() .map(|entry| entry.as_array().and_then(|e| e.first()).and_then(|c| { // `null` for the whole-genome (non-LOCO) case. match c { Value::String(s) => Some(s.clone()), Value::Number(n) => Some(n.to_string()), _ => None, } })) .collect() }) .unwrap_or_default(); log_file_basenames(record) .into_iter() .enumerate() .filter_map(|(index, basename)| { let path = Path::new(cache_dir).join(basename); let log = std::fs::read_to_string(&path).ok()?; let components = parse_variance_components(&log); if components.pve.is_none() && components.vg.is_none() && components.ve.is_none() { return None; } Some((chromosomes.get(index).cloned().flatten(), components)) }) .collect() } /// Four significant figures, scientific when the magnitude needs it. /// /// Variance components are not all O(1): a null fit with a near-zero /// residual prints `vg = 3.4e-05` and `ve = 3.4e-10`, and rounding those to /// four decimals reports 0.0000 for both -- which would read as "GEMMA found /// no variance" when what it found was that the split is 1.0. GEMMA's own /// logs use `%g` for the same reason. fn format_component(value: f64) -> String { let magnitude = value.abs(); if magnitude == 0.0 { "0".to_string() } else if (1e-3..1e4).contains(&magnitude) { format!("{value:.4}") } else { format!("{value:.3e}") } } /// Variance components read from a directory holding GEMMA `.log.txt` files /// -- an extracted GWA archive, say. Each is labelled by the chromosome in its /// own command line (`gemma ... -loco 7 ...`), falling back to the file name /// gemma-wrapper gave it (`..assoc.txt.log.txt`). /// /// Reading the archive is what makes this work whichever gemma-wrapper ran the /// job: the logs have always been inside it, and only the *copy* beside it /// arrived in 5eb53fa. The ravanan path extracts each chromosome's archive /// anyway to read its `.assoc.txt`, so they cost nothing to look at. fn read_variance_components_from_logs(dir: &Path) -> Vec<(Option, VarianceComponents)> { let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new(); }; let mut found: Vec<(Option, VarianceComponents)> = Vec::new(); for entry in entries.filter_map(Result::ok) { let path = entry.path(); if !path.to_string_lossy().ends_with(".log.txt") { continue; } if let Some(components) = variance_components_from_log(&path) { found.push(components); } } sort_by_chromosome(&mut found); found } /// The same, for log paths the workflow hands back directly: since the /// workflow calls `gemma` itself, each job's `.log.txt` is a CWL output rather /// than a file packed into an archive. fn read_variance_components_from_log_files( paths: &[&str], ) -> Vec<(Option, VarianceComponents)> { let mut found: Vec<(Option, VarianceComponents)> = paths .iter() .filter_map(|path| variance_components_from_log(Path::new(path))) .collect(); sort_by_chromosome(&mut found); found } /// One GEMMA `.log.txt`, labelled by the chromosome in its own command line /// (`gemma ... -loco 7 ...`) with the file name as a fallback. fn variance_components_from_log(path: &Path) -> Option<(Option, VarianceComponents)> { let log = std::fs::read_to_string(path).ok()?; let components = parse_variance_components(&log); if components.pve.is_none() && components.vg.is_none() && components.ve.is_none() { return None; } let chromosome = log .lines() .find_map(|line| { let mut words = line.split_whitespace(); while let Some(word) = words.next() { if word == "-loco" { return words.next().map(str::to_string); } } None }) .or_else(|| { let name = path.file_name()?.to_string_lossy().into_owned(); let stem = name.strip_suffix(".log.txt").unwrap_or(&name); // gemma-wrapper's `..assoc.txt.log.txt`, or this // workflow's own `-gwa.log.txt`. if let Some(rest) = stem.strip_suffix(".assoc.txt") { return rest.split('.').nth(1).map(str::to_string); } stem.split('-').next().map(str::to_string) }); Some((chromosome, components)) } /// A stable order, so the note reads chr1..chrX rather than in whatever order /// the filesystem listed the files. fn sort_by_chromosome(found: &mut [(Option, VarianceComponents)]) { found.sort_by_key(|(chr, _)| { chr.as_deref() .map(|chr| (chr.parse::().unwrap_or(u32::MAX), chr.to_string())) }); } /// One note carrying every chromosome's variance components. /// /// All of them, not just the range: GEMMA's per-chromosome `.log.txt` files /// live on the server, so a caller that is told only "0.31 to 0.95" cannot /// ask which chromosome is which -- and under `--loco` that is the whole /// question, since each value belongs to a different kinship matrix. The /// values are written compactly (four significant figures each) so twenty /// chromosomes stay one readable sentence, with the two extremes carrying /// `vg`/`ve` as well. /// /// `pve` is the proportion of variance the kinship explains in the *null* /// model -- the same quantity as BulkLMM's `h2`, per chromosome here. fn variance_components_note(components: &[(Option, VarianceComponents)]) -> Option { if components.is_empty() { return None; } let label = |chr: &Option| { chr.clone() .map(|chr| format!("chr{chr}")) .unwrap_or_else(|| "whole genome".to_string()) }; let scored: Vec<(&Option, &VarianceComponents)> = components .iter() .filter(|(_, c)| c.pve.is_some()) .map(|(chr, c)| (chr, c)) .collect(); if scored.is_empty() { return None; } let per_chromosome: Vec = scored .iter() .map(|(chr, c)| { let se = c .se_pve .map(|se| format!("±{}", format_component(se))) .unwrap_or_default(); format!("{} {}{se}", label(chr), format_component(c.pve.unwrap())) }) .collect(); let describe = |chr: &Option, c: &VarianceComponents| { let pve = format_component(c.pve.unwrap()); let se = c .se_pve .map(|se| format!(" ± {}", format_component(se))) .unwrap_or_default(); match (c.vg, c.ve) { (Some(vg), Some(ve)) => format!( "{} pve {pve}{se} (vg {}, ve {})", label(chr), format_component(vg), format_component(ve) ), _ => format!("{} pve {pve}{se}", label(chr)), } }; let note = if scored.len() == 1 { format!( "GEMMA -pve, null model: {}. This is the proportion of variance the kinship explains \ -- the same quantity as BulkLMM's h2 -- for the null fit, so it does not depend on any \ marker.", describe(scored[0].0, scored[0].1) ) } else { let lowest = scored .iter() .min_by(|a, b| a.1.pve.unwrap().total_cmp(&b.1.pve.unwrap())) .unwrap(); let highest = scored .iter() .max_by(|a, b| a.1.pve.unwrap().total_cmp(&b.1.pve.unwrap())) .unwrap(); format!( "GEMMA -pve, null model, per chromosome (LOCO): {}. That is pve {} to {} over {} \ chromosomes; lowest {}, highest {}. Each value is the proportion of variance that one \ chromosome's kinship explains -- BulkLMM's h2 computed per chromosome instead of \ genome-wide -- so they are not expected to agree with each other or to add up. The \ same numbers are in the run log and in GEMMA's own .log.txt beside the archive.", per_chromosome.join(", "), format_component(lowest.1.pve.unwrap()), format_component(highest.1.pve.unwrap()), scored.len(), describe(lowest.0, lowest.1), describe(highest.0, highest.1), ) }; Some(note) } /// The same table as the note, as log lines, so a `?id=` run's /// `/compute/gemma/status` stream carries it too -- the caller watching a long /// run sees the variance components as soon as they exist, rather than having /// to keep the finished reply. fn variance_components_log_lines( components: &[(Option, VarianceComponents)], ) -> Vec { components .iter() .filter(|(_, c)| c.pve.is_some()) .map(|(chr, c)| { let mut line = format!( "variance: {} pve={}", chr.clone() .map(|chr| format!("chr{chr}")) .unwrap_or_else(|| "genome".to_string()), c.pve.unwrap() ); if let Some(se) = c.se_pve { line.push_str(&format!(" se_pve={se}")); } if let Some(vg) = c.vg { line.push_str(&format!(" vg={vg}")); } if let Some(ve) = c.ve { line.push_str(&format!(" ve={ve}")); } line }) .collect() } /// Parse a GEMMA `.assoc.txt` file (tab-separated, header row first) /// into its column names and one row per marker, each row a /// `Value::Array` in that same column order (not keyed by column name: /// columns vary with `-lmm` -- stock 1/2/3 run one test, 4 runs all /// three; this build's `9`, GeneNetwork's convention, was observed to /// carry `p_lrt` -- so callers needing a name match it up against the /// returned column list themselves). A value that parses as a number is /// kept numeric; "NA" and anything else stays a string. fn parse_assoc_file(contents: &str) -> (Vec, Vec>) { let mut lines = contents.lines(); let Some(header) = lines.next() else { return (Vec::new(), Vec::new()); }; let columns: Vec = header.split('\t').map(str::to_string).collect(); let rows: Vec> = lines .filter(|line| !line.is_empty()) .map(|line| { line.split('\t') .map(|value| { value .parse::() .map(|n| serde_json::json!(n)) .unwrap_or_else(|_| Value::String(value.to_string())) }) .collect() }) .collect(); with_effect_size(columns, rows) } /// Add the effect size GeneNetwork's own displays pair with the score. /// /// GEMMA already reports `beta` (its per-copy effect) and `se`, but in /// GEMMA's own vocabulary and in GEMMA's own column order, where the /// score is the last column and the effect is four columns earlier. /// GeneNetwork's tables and CSV pair the two as **`additive` = /// `−beta/2`** (GN2's `marker_regression/gemma_mapping.py`): halved, /// because half the difference between the two homozygotes is the usual /// additive effect, and negated, because GN2's BIMBAM conversion codes /// `@mat` as 1 so GEMMA's beta is per *maternal* copy while GeneNetwork's /// published sign convention is anchored on the other allele. /// /// The column goes **immediately before the score**, so a row reads /// `… beta, se, l_mle, additive, p_lrt` -- next to the score, without /// moving anything GEMMA wrote and without breaking "the last column is /// the p-value", which `peak_marker` and the plotter both rely on. /// /// `beta` and `se` stay: `se` is what makes the effect interpretable, and /// they are GEMMA's own numbers. Nothing is added when `beta` is absent /// (a non-LMM run) -- the columns are then exactly GEMMA's. fn with_effect_size( mut columns: Vec, mut rows: Vec>, ) -> (Vec, Vec>) { let Some(beta_at) = columns.iter().position(|name| name == "beta") else { return (columns, rows); }; // Before the score: the last column, which this module and the // plotters treat as the p-value. let score_at = columns.len().saturating_sub(1); if score_at == 0 || !columns[score_at].starts_with('p') || score_at <= beta_at { return (columns, rows); } for row in rows.iter_mut() { let additive = match row.get(beta_at).and_then(Value::as_f64) { Some(beta) => serde_json::json!(-beta / 2.0), None => Value::Null, }; row.insert(score_at, additive); } columns.insert(score_at, "additive".to_string()); (columns, rows) } /// Extract `archive` (a `.tar.xz`, relative to `cache_dir`) into /// `dest_dir`. Confirmed necessary by actually running the real /// binaries: a `--loco` GWA run's `.assoc.txt` files (one per /// chromosome) only ever end up inside this archive, never as loose /// files in `cache_dir` -- unlike the kinship step's `.cXX.txt` files, /// which do also appear loose (gemma-wrapper itself is inconsistent /// about this between the two steps). async fn extract_archive(cache_dir: &str, archive: &str, dest_dir: &str) -> Result<(), GemmaError> { let archive_path = Path::new(cache_dir).join(archive); let output = streaming::run_streaming( "tar", &["xJf", &archive_path.to_string_lossy(), "-C", dest_dir], None, ) .await .map_err(GemmaError::Io)?; if !output.status.success() { return Err(GemmaError::CommandFailed { step: "extract-archive", stderr: String::from_utf8_lossy(&output.stderr).into_owned(), }); } Ok(()) } /// Read and parse every `basenames` file under `extract_dir` (one per /// `--loco` chromosome), returning the column names (from the first /// file; every chromosome's `.assoc.txt` shares the same header) and /// every file's rows concatenated, each row a `Value::Array`. fn parse_assoc_files(extract_dir: &Path, basenames: Vec<&str>) -> Result<(Vec, Vec), GemmaError> { let mut columns = Vec::new(); let mut rows = Vec::new(); for basename in basenames { let contents = std::fs::read_to_string(extract_dir.join(basename)) .map_err(GemmaError::AssocFile)?; let (file_columns, file_rows) = parse_assoc_file(&contents); if columns.is_empty() { columns = file_columns; } rows.extend(file_rows.into_iter().map(Value::Array)); } Ok((columns, rows)) } fn nanos_since_epoch() -> u128 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_nanos()) .unwrap_or(0) } /// The mapped QTL rows (each a `Value::Array` in `columns`' order, not /// a keyed object -- see `parse_assoc_file`), plus the exact /// gemma-wrapper invocations used to produce them (see `compute`'s /// `notes` below), so a caller can see precisely what was run without /// reading server logs. pub struct GemmaOutput { pub columns: Vec, pub rows: Value, pub notes: Vec, } /// The gemma-wrapper arguments for the kinship step, shared between the /// actual invocation (`compute_kinship`) and the note describing it /// (`compute`), so the two can never drift apart. fn kinship_args<'a>( cache_dir: &'a str, genotype_file: &'a str, pheno_file: &'a str, snps_file: &'a str, ) -> Vec<&'a str> { vec![ "--json", "--cache-dir", cache_dir, "--loco", "--", "-g", genotype_file, "-p", pheno_file, "-a", snps_file, "-gk", ] } /// The gemma-wrapper arguments for the GWA step, same pairing as /// `kinship_args`. fn gwa_args<'a>( cache_dir: &'a str, genotype_file: &'a str, pheno_file: &'a str, snps_file: &'a str, covariate_file: Option<&'a str>, lmm_str: &'a str, k_json_path: &'a str, ) -> Vec<&'a str> { let mut args = vec![ "--json", "--cache-dir", cache_dir, "--loco", "--input", k_json_path, "--", "-g", genotype_file, "-p", pheno_file, "-a", snps_file, ]; // Covariates reach only the GWA step: -gk (kinship) takes none, and // GEMMA's own key for the K cache drops the phenotype pair, so the // kinship computed once is still shared across covariate sets. if let Some(covariate_file) = covariate_file { args.push("-c"); args.push(covariate_file); } args.push("-lmm"); args.push(lmm_str); // Ask GEMMA for its own variance components in the null model. It writes // them into the per-run `.log.txt`, which gemma-wrapper keeps beside the // archive; `-pve` is accepted alongside `-lmm 9` (verified against GEMMA // 0.98.6 on a BIMBAM fixture). It changes nothing in `.assoc.txt`, so it // costs four log lines and no recomputation of the scan. args.push("-pve"); args } /// Run the full gemma-wrapper flow -- compute kinship, then run the LMM /// association test, both `--loco` -- and return the mapped QTL as a /// JSON array of rows (each a JSON object), concatenated across every /// chromosome's `.assoc.txt` from the real GEMMA output, plus the two /// gemma-wrapper invocations used (`GemmaOutput::notes`). The rows are /// one `Value::Array`, not a `Vec`: utoipa (for the OpenAPI /// schema) implements `ToSchema` for `serde_json::Value` itself, but /// `Vec` needs `Value: ComposeSchema`, which it does /// not implement in this utoipa version. pub async fn compute( gemma_wrapper_cmd: &str, cache_dir: &str, genotype_file: &str, snps_file: &str, trait_values: &TraitValues, covariates: &[crate::covariates::Covariate], lmm: u8, run_id: Option<&str>, ) -> Result { std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?; let unique = format!("{}-{}", std::process::id(), nanos_since_epoch()); let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt")); write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?; let pheno_file = pheno_path.to_string_lossy().into_owned(); // GEMMA's own covariate file: positional over the panel, numeric, with // `NA` for a missing value -- which it treats like a missing phenotype // and drops the individual for (see crate::covariates). let covariate_path = Path::new(cache_dir).join(format!("{unique}-covariates.txt")); let mut notes: Vec = Vec::new(); let covariate_file = if covariates.is_empty() { None } else { let written = crate::covariates::write_gemma_covariates_file( &covariate_path, covariates, &trait_values.iter().map(Option::is_some).collect::>(), ) .map_err(GemmaError::Io)?; for (name, absent) in &written.missing { notes.push(format!( "Missing covariate values: {name} has none for {absent} individual(s), which \ GEMMA drops, as it drops a missing phenotype." )); } if !written.columns.is_empty() { // Say exactly what GEMMA was handed: the dummies are ours to // name, and a reader comparing two runs wants to see them. notes.push(format!( "GEMMA fits {} covariate column(s): {}.", written.columns.len(), written.columns.join(", ") )); } if !written.dropped.is_empty() { notes.push(format!( "Covariate column(s) left out because they do not vary among the individuals \ fitted: {}.", written.dropped.join(", ") )); } if written.columns.is_empty() { notes.push( "No covariate varies among the individuals fitted, so GEMMA ran without one." .to_string(), ); None } else { Some(covariate_path.to_string_lossy().into_owned()) } }; // One log file for the whole request (both steps); see // crate::streaming::create_run_log for why this can't silently // collide with another run's. let log_path = streaming::create_run_log(cache_dir, run_id)?; // Everything after the log exists reports its outcome into the log: // success ends with `COMPLETION_MARKER`, any error with // `FAILURE_MARKER: ` -- so a `status` poller can never // mistake a failed run for a still-running one (a healthy GEMMA // run is also quiet for long stretches). let outcome = async { let kinship_note = format!( "Kinship: {gemma_wrapper_cmd} {}", kinship_args(cache_dir, genotype_file, &pheno_file, snps_file).join(" ") ); let k_record = compute_kinship( gemma_wrapper_cmd, cache_dir, genotype_file, &pheno_file, snps_file, log_path.as_deref(), ) .await?; let k_json_path = Path::new(cache_dir).join(format!("{unique}-k.json")); std::fs::write(&k_json_path, k_record.to_string()).map_err(GemmaError::Io)?; let lmm_str = lmm.to_string(); let gwa_note = format!( "GWA: {gemma_wrapper_cmd} {}", gwa_args( cache_dir, genotype_file, &pheno_file, snps_file, covariate_file.as_deref(), &lmm_str, &k_json_path.to_string_lossy(), ) .join(" ") ); let gwa_record = compute_gwa( gemma_wrapper_cmd, cache_dir, genotype_file, &pheno_file, snps_file, lmm, &k_json_path.to_string_lossy(), covariate_file.as_deref(), log_path.as_deref(), ) .await?; let basenames = assoc_file_basenames(&gwa_record); if basenames.is_empty() { return Err(GemmaError::NoAssocFiles); } let archive = gwa_record .get("archive") .and_then(Value::as_str) .ok_or_else(|| GemmaError::WrapperError { step: "gwa", detail: "gemma-wrapper's response did not list an output archive".to_string(), })?; let extract_dir = Path::new(cache_dir).join(format!("{unique}-gwa-extract")); std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?; extract_archive(cache_dir, archive, &extract_dir.to_string_lossy()).await?; let (columns, rows) = parse_assoc_files(&extract_dir, basenames)?; let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless let components = read_variance_components(cache_dir, &gwa_record); let mut output_notes = vec![kinship_note, gwa_note]; output_notes.extend(notes); if let Some(note) = variance_components_note(&components) { output_notes.push(note); } streaming::append_lines(&log_path, &variance_components_log_lines(&components)); Ok(GemmaOutput { columns, rows: Value::Array(rows), notes: output_notes, }) } .await; match outcome { Ok(output) => { streaming::mark_complete(&log_path); Ok(output) } Err(e) => { streaming::mark_failed(&log_path, &e); Err(e) } } } /// Read a scattered `File[]` output's list of `path`s from ravanan's /// stdout (the CWL outputs record for a scattered output: /// `{"": [{"class": "File", "path": ..., ...}, ...], ...}`, /// one entry per `cwl/gemma/gemma-loco.scm` chromosome). fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result, GemmaError> { let files = outputs .get(id) .and_then(Value::as_array) .ok_or_else(|| GemmaError::RavananOutput(format!("no \"{id}\" array in ravanan's output")))?; files .iter() .map(|file| { file.get("path").and_then(Value::as_str).ok_or_else(|| { GemmaError::RavananOutput(format!("no .path on one of \"{id}\"'s entries")) }) }) .collect() } /// The input names a compiled CWL workflow declares, and which of them have /// no default (so they must be supplied). /// /// `ccwl compile` emits **CWL as YAML** (its only other target is dot), so this /// reads that shape rather than JSON: a top-level `inputs:` key, then one /// two-space-indented `name:` per input, with that input's own fields (such as /// `default:`) indented further. Deliberately not a YAML parser and not a new /// dependency -- ccwl's output is machine-generated and unvaried, and a /// `serde_yaml` in the tree would mean regenerating the crate list in /// `guix.scm` for one small check. Anything it does not understand surfaces as /// "declares no inputs", which is reported as a mismatch rather than ignored. fn workflow_inputs(compiled_path: &Path) -> Result<(Vec, Vec), GemmaError> { let text = std::fs::read_to_string(compiled_path).map_err(GemmaError::Io)?; let mut lines = text.lines().peekable(); // Find the top-level `inputs:` block. let mut in_inputs = false; for line in lines.by_ref() { if line.trim_end() == "inputs:" { in_inputs = true; break; } } if !in_inputs { return Err(GemmaError::WorkflowMismatch( "the compiled workflow has no top-level `inputs:` block; is \ GEMMA_RAVANAN_WORKFLOW a ccwl source?" .to_string(), )); } let mut required = Vec::new(); let mut declared = Vec::new(); let mut current: Option<(String, bool)> = None; for line in lines { let indented = line.len() - line.trim_start_matches(' ').len(); if indented == 0 { break; // e.g. `outputs:` -- the inputs block has ended. } let trimmed = line.trim(); if trimmed.is_empty() || trimmed.starts_with('#') { continue; } if indented == 2 { if let Some((name, has_default)) = current.take() { declared.push(name.clone()); if !has_default { required.push(name); } } if let Some(name) = trimmed.strip_suffix(':') { current = Some((name.trim().to_string(), false)); } } else if trimmed.starts_with("default:") { if let Some((_, has_default)) = current.as_mut() { *has_default = true; } } } if let Some((name, has_default)) = current.take() { declared.push(name.clone()); if !has_default { required.push(name); } } if declared.is_empty() { return Err(GemmaError::WorkflowMismatch( "the compiled workflow declares no inputs".to_string(), )); } declared.sort(); required.sort(); Ok((declared, required)) } /// Refuse to invoke ravanan when the workflow and this build disagree, naming /// the mismatch rather than letting cwltool report a `null` input. fn check_workflow_inputs(compiled_path: &Path, supplied: &[&str]) -> Result<(), GemmaError> { let (declared, required) = workflow_inputs(compiled_path)?; let missing: Vec<&str> = required .iter() .filter(|name| !supplied.contains(&name.as_str())) .map(String::as_str) .collect(); let unknown: Vec<&str> = supplied .iter() .copied() .filter(|name| !declared.iter().any(|declared| declared == name)) .collect(); if missing.is_empty() && unknown.is_empty() { return Ok(()); } let mut detail = String::new(); if !missing.is_empty() { detail.push_str(&format!( "the workflow needs {} and this build does not supply it; ", missing.iter().map(|n| format!("`{n}'")).collect::>().join(", ") )); } if !unknown.is_empty() { detail.push_str(&format!( "this build supplies {} which the workflow does not declare; ", unknown.iter().map(|n| format!("`{n}'")).collect::>().join(", ") )); } detail.push_str(&format!( "the workflow declares [{}]", declared.join(", ") )); Err(GemmaError::WorkflowMismatch(detail)) } /// The distinct chromosome names in a SNP annotation file's contents /// (whitespace-separated: rs id, position, chromosome), in first-seen /// order, used here to build `compute_ravanan`'s `chromosomes` scatter /// input: GEMMA's own `-loco ` takes a chromosome name, so the /// annotation file decides the scatter (CWL's /// `scatter` needs a concrete array at invocation time, which can't be /// discovered by a step inside the workflow itself without another /// round of plumbing -- see `cwl/gemma/gemma-loco.scm`'s own doc /// comment on this input). /// /// Takes the contents, not the path, so the caller reads the file once. /// A stable identity for one request's GEMMA inputs, used to name the files /// ravanan keys its work reuse on: the phenotype file, the inputs YAML and the /// compiled CWL. Identical inputs therefore produce identical paths and a /// repeat request is answered from ravanan's store instead of recomputing /// every chromosome. /// /// The genotype file is identified by its path and size rather than its /// contents: hashing a genome-sized file on every request would cost more than /// the reuse saves, and the path is stable in practice (it is the caller's own /// file, interned verbatim into ravanan's store). The SNP file is hashed, /// because the chromosome list -- and therefore the whole scatter -- comes /// from it. fn sha16(material: &[u8]) -> String { crate::indexer::sha256::hex_digest(material)[..16].to_string() } /// A key for the compiled workflow: the ccwl source is the only input, so the /// same source compiles to the same file and ravanan's (script, inputs) pair /// stays stable across calls. fn workflow_key(workflow_scm_path: &str) -> String { match std::fs::read(workflow_scm_path) { Ok(contents) => sha16(&contents), Err(_) => sha16(workflow_scm_path.as_bytes()), } } /// A key for the *kinship* step's inputs: genotype, SNP annotation, `lmm`, and /// the pattern of which individuals have a value -- deliberately not the /// values themselves. /// /// GEMMA's `-gk` reads the phenotype file only to learn which individuals are /// analysed; the matrix itself comes from the genotypes. Keying K on the /// animal set is what lets two traits over the same animals share all twenty /// K matrices, instead of storing a fresh 11 MB scatter per trait. It is also /// stricter than gemma-wrapper's own key, which ignores the phenotype /// altogether and would therefore reuse a K computed for a different set of /// animals. fn kinship_input_key( genotype_file: &str, snps_file: &str, trait_values: &TraitValues, lmm: u8, ) -> String { let mut material = Vec::new(); material.extend_from_slice(genotype_file.as_bytes()); material.push(0); if let Ok(metadata) = std::fs::metadata(genotype_file) { material.extend_from_slice(&metadata.len().to_le_bytes()); } material.push(0); if let Ok(snps) = std::fs::read(snps_file) { material.extend_from_slice(&snps); } material.push(0); material.extend_from_slice(lmm.to_string().as_bytes()); material.push(0); for value in trait_values { material.extend_from_slice(match value { Some(_) => b"1", None => b"NA", }); material.push(b','); } sha16(&material) } /// A key for one trait's own inputs: the kinship key plus the values, so a /// different trait gets its own phenotype file while sharing the kinship /// step's. fn trait_input_key(kinship_key: &str, trait_values: &TraitValues) -> String { let mut material = Vec::new(); material.extend_from_slice(kinship_key.as_bytes()); material.push(0); for value in trait_values { match value { Some(value) => material.extend_from_slice(&value.to_bits().to_le_bytes()), None => material.extend_from_slice(b"NA"), } material.push(b','); } sha16(&material) } fn chromosomes_from(contents: &str) -> Vec { let mut seen = std::collections::HashSet::new(); let mut chromosomes = Vec::new(); for line in contents.lines() { if let Some(chr) = line.split_whitespace().nth(2) { if seen.insert(chr.to_string()) { chromosomes.push(chr.to_string()); } } } chromosomes } /// The same GEMMA `--loco` kinship-then-GWA computation as `compute`, /// but run via `ravanan` (https://github.com/vepiteski/ravanan) driving /// a CWL workflow compiled from the ccwl (https://ccwl.systemreboot.net/) /// source at `workflow_scm_path` (see `cwl/gemma/gemma-loco.scm`). /// /// This runner invokes **gemma itself**, one invocation per /// chromosome, relying on `-loco ` being native to 0.98.6 for both /// `-gk` and `-lmm` -- where `compute` still drives gemma-wrapper. That /// is deliberate: it is what makes the two endpoints comparable, the /// same computation through two stacks, and the day the numbers stop /// agreeing the difference is visible rather than assumed away. /// Backs `POST /compute/gemma/ravanan`. pub async fn compute_ravanan( ravanan_cmd: &str, batch: &crate::config::RavananBatch, ccwl_cmd: &str, workflow_scm_path: &str, manifest_path: &str, cache_dir: &str, scratch_dir: &str, genotype_file: &str, snps_file: &str, trait_values: &TraitValues, lmm: u8, run_id: Option<&str>, ) -> Result { std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?; // The workflow's inputs are named by their *content*, not by a pid and a // timestamp: ravanan reuses a step's result when the (script, inputs) pair // is unchanged, and that pair is built from these paths. Unique names made // every request cold, which cost the whole computation again for a repeat // of the same request -- 14.7 s versus 1.6 s on the BXD fixture. // // Two keys, not one: the kinship step is keyed on the *individual set* // only (see `kinship_input_key`), so a second trait over the same animals // reuses all twenty K matrices, and only the GWA step's inputs change. let kinship_key = kinship_input_key(genotype_file, snps_file, trait_values, lmm); let trait_key = trait_input_key(&kinship_key, trait_values); let workflow = workflow_key(workflow_scm_path); // A constant-valued phenotype file with the trait's missing-data pattern: // what the kinship step gets, so its files are the same for every trait // over the same animals. K depends on the genotypes and the individual // set, never on the values -- verified byte-for-byte against a real trait. let kinship_pheno_path = Path::new(cache_dir).join(format!("{kinship_key}-kin-pheno.txt")); write_constant_pheno_file(&kinship_pheno_path, trait_values).map_err(GemmaError::Io)?; let pheno_path = Path::new(cache_dir).join(format!("{trait_key}-pheno.txt")); // Written aside and renamed, so a concurrent request with the same key // never reads a half-written file. let pheno_tmp = Path::new(cache_dir).join(format!("{trait_key}-pheno.tmp-{}", std::process::id())); write_pheno_file(&pheno_tmp, trait_values).map_err(GemmaError::Io)?; std::fs::rename(&pheno_tmp, &pheno_path).map_err(GemmaError::Io)?; // The run log stays unique per call: it is the thing a caller streams by // run_id, so two calls must never share one. let log_path = streaming::create_run_log(cache_dir, run_id)?; // gemma-loco.scm is the maintained source; the CWL ravanan actually // runs is a generated, throwaway artifact compiled fresh on every // call (cheap: ccwl compile is pure macro expansion, no Guix build // involved) and never checked into the repo -- see the doc comment // on Config::gemma_ravanan_workflow. let compiled_workflow_path = Path::new(cache_dir).join(format!("{workflow}-gemma-loco.cwl")); // Compile to a per-process path and rename: the compiled CWL is // deterministic, so two concurrent identical requests would otherwise // race on the same file. let compiled_tmp = Path::new(cache_dir).join(format!("{workflow}-gemma-loco.tmp-{}.cwl", std::process::id())); let ccwl_output_arg = format!("--output={}", compiled_tmp.display()); let ccwl_args = vec!["compile", ccwl_output_arg.as_str(), workflow_scm_path]; let ccwl_output = streaming::run_streaming(ccwl_cmd, &ccwl_args, None) .await .map_err(|error| GemmaError::Spawn { command: ccwl_cmd.to_string(), error, })?; if !ccwl_output.status.success() { return Err(GemmaError::CommandFailed { step: "ccwl compile", stderr: String::from_utf8_lossy(&ccwl_output.stderr).into_owned(), }); } std::fs::rename(&compiled_tmp, &compiled_workflow_path).map_err(GemmaError::Io)?; // Before anything is staged for ravanan: does the workflow in this // deployment's checkout take what this build is about to send it? check_workflow_inputs( &compiled_workflow_path, &[ "genotype_file", "pheno_file", "kinship_pheno_file", "snps_file", "lmm", "chromosomes", ], )?; let compiled_workflow_arg = compiled_workflow_path.to_string_lossy().into_owned(); let store_dir = Path::new(cache_dir).join("ravanan-store"); // ravanan's --store needs these three subdirectories to already // exist; it does not create them itself. for subdir in ["files", "logs", "data"] { std::fs::create_dir_all(store_dir.join(subdir)).map_err(GemmaError::Io)?; } let snps_contents = std::fs::read_to_string(snps_file).map_err(GemmaError::Io)?; let chromosomes = chromosomes_from(&snps_contents); if chromosomes.is_empty() { return Err(GemmaError::NoAssocFiles); } let chromosomes_yaml = chromosomes .iter() .map(|c| format!("\"{c}\"")) .collect::>() .join(", "); let inputs_yaml_path = Path::new(cache_dir).join(format!("{trait_key}-ravanan-inputs.yaml")); let inputs_tmp = Path::new(cache_dir).join(format!("{trait_key}-ravanan-inputs.tmp-{}.yaml", std::process::id())); std::fs::write( &inputs_tmp, format!( "genotype_file: {{class: File, path: {genotype_file}}}\n\ pheno_file: {{class: File, path: {}}}\n\ kinship_pheno_file: {{class: File, path: {}}}\n\ snps_file: {{class: File, path: {snps_file}}}\n\ lmm: \"{lmm}\"\n\ chromosomes: [{chromosomes_yaml}]\n", pheno_path.display(), kinship_pheno_path.display(), ), ) .map_err(GemmaError::Io)?; std::fs::rename(&inputs_tmp, &inputs_yaml_path).map_err(GemmaError::Io)?; let store_arg = format!("--store={}", store_dir.display()); let manifest_arg = format!("--guix-manifest={manifest_path}"); let inputs_yaml_arg = inputs_yaml_path.to_string_lossy().into_owned(); // No --output-directory: this code reads the scattered outputs' // paths straight out of ravanan's own --store via its stdout record, // never through a flattened copy, and the workflow names each // chromosome's files `-gwa.*` itself. // Backend switches: nothing for the default local runner, and // --batch-system=slurm-api plus endpoint/JWT/partition/scratch for a // cluster (see docs/ravanan-slurm.md). `scratch` is the cache dir // unless configured otherwise: with slurm, ravanan stages each job's // scratch there, and the compute nodes have to see the same path. let scratch = if scratch_dir.is_empty() { cache_dir } else { scratch_dir }; let mut args = vec![ store_arg.as_str(), manifest_arg.as_str(), compiled_workflow_arg.as_str(), inputs_yaml_arg.as_str(), ]; let batch_args = batch.ravanan_args(scratch); args.extend(batch_args.iter().map(String::as_str)); let ravanan_note = format!( "ravanan ({ravanan_cmd}, {} backend): {}", match batch { crate::config::RavananBatch::SingleMachine => "single-machine", crate::config::RavananBatch::SlurmApi { .. } => "slurm-api", }, args.join(" ") ); let chromosomes_note = format!( "Scattered gemma itself over {} chromosome(s) via CWL scatter, one \ invocation per chromosome: {}", chromosomes.len(), chromosomes.join(", "), ); let output = streaming::run_streaming(ravanan_cmd, &args, log_path.as_deref()) .await .map_err(|error| GemmaError::Spawn { command: ravanan_cmd.to_string(), error, })?; if !output.status.success() { return Err(GemmaError::RavananFailed( String::from_utf8_lossy(&output.stderr).into_owned(), )); } let outputs: Value = serde_json::from_str(&String::from_utf8_lossy(&output.stdout)) .map_err(|e| GemmaError::RavananOutput(format!("could not parse ravanan's stdout: {e}")))?; let assoc_paths = ravanan_output_paths(&outputs, "assoc")?; let gwa_log_paths = ravanan_output_paths(&outputs, "gwa_log")?; if assoc_paths.is_empty() { return Err(GemmaError::NoAssocFiles); } // One .assoc.txt and one .log.txt per chromosome, straight from the // workflow's own outputs: gemma writes named files (`-o -gwa`), so // there is no wrapper JSON record to parse and no archive to unpack. let mut columns = Vec::new(); let mut rows = Vec::new(); for assoc_path in &assoc_paths { let contents = std::fs::read_to_string(assoc_path).map_err(GemmaError::AssocFile)?; let (file_columns, file_rows) = parse_assoc_file(&contents); if columns.is_empty() { columns = file_columns; } rows.extend(file_rows.into_iter().map(Value::Array)); } // `-pve` puts the variance components in each job's .log.txt, which the // workflow returns as a first-class output. let components = read_variance_components_from_log_files(&gwa_log_paths); streaming::mark_complete(&log_path); // Ravanan's reuse lives in this directory (--store), so whether the cache // survives a container restart -- and whether a job inside the container // can even see the interned inputs -- depends entirely on where it is. // The default is under /tmp, which is neither durable nor visible to a // container that only bind-mounts one host path; say so on the first run // rather than letting an operator discover it as a 45-minute rebuild. let mut ephemeral_note: Vec = Vec::new(); if cache_dir.starts_with("/tmp/") || cache_dir == "/tmp" { ephemeral_note.push(format!( "ravanan's cache is under {cache_dir}: that is not durable (a reboot drops it) \ and a Slurm container can only see paths under its bind-mounted share, so jobs \ may not find the interned inputs. Set RAVANAN_CACHE_DIR to a durable path the \ nodes can see -- e.g. /gn4-ravanan-cache." )); } let mut notes = vec![ ravanan_note, chromosomes_note, format!("workflow: {workflow_scm_path} (compiled to {compiled_workflow_arg})"), ]; notes.extend(ephemeral_note); if let Some(note) = variance_components_note(&components) { notes.push(note); } streaming::append_lines(&log_path, &variance_components_log_lines(&components)); Ok(GemmaOutput { columns, rows: Value::Array(rows), notes, }) } #[cfg(test)] mod tests { use super::*; /// A stale `gemma-loco.scm` in the deployment's checkout is a deployment /// mistake with an inscrutable symptom (cwltool: "expected File but got /// null"); the check names it instead. The fixture is the shape /// `ccwl compile` actually emits: YAML, two-space-indented input names. #[test] fn a_workflow_from_another_commit_is_refused_by_name() { let path = std::env::temp_dir().join(format!("gn4-wf-{}.cwl", std::process::id())); let older = "cwlVersion: v1.2\nclass: Workflow\ninputs:\n genotype_file:\n type: File\n pheno_file:\n type: File\n snps_file:\n type: File\n lmm:\n type: string\n chromosomes:\n type: string[]\noutputs:\n assoc:\n type: File\n"; std::fs::write(&path, older).unwrap(); let error = check_workflow_inputs( &path, &[ "genotype_file", "pheno_file", "kinship_pheno_file", "snps_file", "lmm", "chromosomes", ], ) .expect_err("must refuse"); let message = error.to_string(); assert!( message.contains("kinship_pheno_file") && message.contains("does not declare"), "{message}" ); assert!(message.contains("same commit"), "says what to do: {message}"); assert!(matches!(error, GemmaError::WorkflowMismatch(_))); // A defaulted input is declared but not required. let current = "cwlVersion: v1.2\nclass: Workflow\ninputs:\n genotype_file:\n type: File\n pheno_file:\n type: File\n kinship_pheno_file:\n type: File\n snps_file:\n type: File\n lmm:\n type: string\n default: \"9\"\n chromosomes:\n type: string[]\noutputs: {}\n"; std::fs::write(&path, current).unwrap(); let (declared, required) = workflow_inputs(&path).expect("parses"); assert!(declared.contains(&"kinship_pheno_file".to_string()), "{declared:?}"); assert!(!required.contains(&"lmm".to_string()), "defaulted: {required:?}"); check_workflow_inputs( &path, &[ "genotype_file", "pheno_file", "kinship_pheno_file", "snps_file", "lmm", "chromosomes", ], ) .expect("matching workflow"); let _ = std::fs::remove_file(&path); } #[test] fn a_missing_binary_names_itself_and_a_file_error_does_not_blame_gemma() { let spawn = GemmaError::Spawn { command: "ccwl".to_string(), error: std::io::Error::from(std::io::ErrorKind::NotFound), }; assert_eq!( spawn.to_string(), "failed to run ccwl: entity not found", "the message must name the command that would not start" ); assert!(!spawn.to_string().contains("gemma-wrapper"), "{spawn}"); let io = GemmaError::Io(std::io::Error::from(std::io::ErrorKind::PermissionDenied)); assert_eq!(io.to_string(), "I/O error: permission denied"); } #[test] fn the_peak_marker_is_the_smallest_p_value() { let rows = serde_json::json!([ [1.0, "rs1", 3.0, 0.0, "X", "Y", 0.2, 1.0, 2.0, 3.0, 0.5], [1.0, "rs2", 9.0, 0.0, "X", "Y", 0.2, 1.0, 2.0, 3.0, 0.001], [1.0, "rs3", 12.0, 0.0, "X", "Y", 0.2, 1.0, 2.0, 3.0, 0.2] ]); assert_eq!(peak_marker(&rows).as_deref(), Some("rs2")); assert_eq!(peak_marker(&serde_json::json!([])), None); assert_eq!(peak_marker(&serde_json::json!({})), None); } #[test] fn write_pheno_file_writes_one_value_per_line_and_na_for_missing() { let path = std::env::temp_dir().join(format!("gn4-gemma-test-{}", std::process::id())); write_pheno_file(&path, &vec![Some(9.944), None, Some(8.572)]).unwrap(); let contents = std::fs::read_to_string(&path).unwrap(); std::fs::remove_file(&path).unwrap(); assert_eq!(contents, "9.944\nNA\n8.572\n"); } #[test] fn parse_assoc_file_parses_header_and_numeric_rows() { let contents = "chr\trs\tps\tp_wald\n1\trs001\t3010274\t0.05\n1\trs002\t3492195\tNA\n"; let (columns, rows) = parse_assoc_file(contents); assert_eq!(columns, vec!["chr", "rs", "ps", "p_wald"]); assert_eq!(rows.len(), 2); assert_eq!(rows[0], vec![ serde_json::json!(1.0), serde_json::json!("rs001"), serde_json::json!(3010274.0), serde_json::json!(0.05), ]); assert_eq!(rows[1][3], serde_json::json!("NA")); } #[test] fn parse_assoc_file_empty_contents_yields_no_rows() { let (columns, rows) = parse_assoc_file(""); assert_eq!(columns.len(), 0); assert_eq!(rows.len(), 0); } #[test] fn assoc_file_basenames_reads_the_third_element_of_every_file_entry() { let record = serde_json::json!({ "files": [["1", "a.log.txt", "a.assoc.txt"], ["2", "b.log.txt", "b.assoc.txt"]] }); assert_eq!( assoc_file_basenames(&record), vec!["a.assoc.txt", "b.assoc.txt"] ); } #[test] fn assoc_file_basenames_missing_files_yields_empty() { assert_eq!( assoc_file_basenames(&serde_json::json!({})), Vec::<&str>::new() ); } #[test] fn variance_components_are_read_from_gemmas_pve_block() { let log = "## Command Line Input = gemma -lmm 9 -pve\n\ total computation time = 12.5 s\n\ ## pve estimate in the null model = 0.61702\n\ ## se(pve) in the null model = 0.0306283\n\ ## vg estimate in the null model = 1.4402\n\ ## ve estimate in the null model = 0.8941\n"; let components = parse_variance_components(log); assert!((components.pve.unwrap() - 0.61702).abs() < 1e-12); assert!((components.se_pve.unwrap() - 0.0306283).abs() < 1e-12); assert!((components.vg.unwrap() - 1.4402).abs() < 1e-12); assert!((components.ve.unwrap() - 0.8941).abs() < 1e-12); } /// GEMMA prints `-nan` for the standard error when the estimate sits on a /// boundary. That is not a number to report, and reporting it as 0 would /// be a lie: `None` says "not available". #[test] fn a_boundary_estimate_reports_no_standard_error() { let log = "## pve estimate in the null model = 1.62846e-06\n\ ## se(pve) in the null model = -nan\n\ ## vg estimate in the null model = 0\n\ ## ve estimate in the null model = 0\n"; let components = parse_variance_components(log); assert!(components.pve.is_some()); assert_eq!(components.se_pve, None); } /// The case that made this necessary: GEMMA reported vg = 3.4e-05 and /// ve = 3.4e-10 on a boundary null fit, and four decimals says 0.0000 for /// both. #[test] fn tiny_variance_components_are_not_rounded_to_zero() { assert_eq!(format_component(3.41066e-05), "3.411e-5"); assert_eq!(format_component(3.41066e-10), "3.411e-10"); assert_eq!(format_component(0.0), "0"); assert_eq!(format_component(0.61702), "0.6170"); assert_eq!(format_component(1.0), "1.0000"); } #[test] fn a_log_without_the_block_yields_nothing() { let components = parse_variance_components("total computation time = 1 s\n"); assert_eq!( components, VarianceComponents { pve: None, se_pve: None, vg: None, ve: None } ); } /// Every chromosome's value is in the note: GEMMA's per-chromosome logs /// live on the server, so a caller told only "0.40 to 0.62" cannot ask /// which chromosome is which. #[test] fn every_chromosome_appears_in_the_note() { let components = vec![ ( Some("1".to_string()), VarianceComponents { pve: Some(0.4), se_pve: Some(0.02), vg: Some(1.0), ve: Some(1.5), }, ), ( Some("2".to_string()), VarianceComponents { pve: Some(0.62), se_pve: None, vg: None, ve: None, }, ), ( Some("3".to_string()), VarianceComponents { pve: Some(0.51), se_pve: Some(0.01), vg: Some(2.0), ve: Some(1.0), }, ), ]; let note = variance_components_note(&components).unwrap(); assert!(note.contains("chr1 0.4000±0.0200"), "{note}"); assert!(note.contains("chr2 0.6200"), "{note}"); assert!(note.contains("chr3 0.5100±0.0100"), "{note}"); assert!(note.contains("pve 0.4000 to 0.6200 over 3 chromosomes"), "{note}"); // The extremes also say what they are made of. assert!(note.contains("lowest chr1 pve 0.4000 ± 0.0200 (vg 1.0000, ve 1.5000)"), "{note}"); assert!(note.contains("highest chr2 pve 0.6200"), "{note}"); assert!(note.contains("LOCO"), "{note}"); } /// One chromosome (a non-LOCO run, or a panel with one chromosome) /// reports the single value rather than a range of one. #[test] fn a_single_chromosome_reports_one_value() { let components = vec![( Some("1".to_string()), VarianceComponents { pve: Some(0.61702), se_pve: Some(0.0306), vg: Some(1.4402), ve: Some(0.8941), }, )]; let note = variance_components_note(&components).unwrap(); assert!(note.contains("chr1 pve 0.6170 ± 0.0306"), "{note}"); assert!(!note.contains(" to "), "{note}"); } /// The ravanan path reads each chromosome's log out of its extracted GWA /// archive, so the labelling has to work from the log itself. #[test] fn components_are_read_out_of_an_extracted_archive() { let dir = std::env::temp_dir().join(format!("gn4-pve-logs-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); // chr7: labelled by its own command line. std::fs::write( dir.join("abcdef.7.assoc.txt.log.txt"), "## Command Line Input = gemma -loco 7 -lmm 9 -pve -o abcdef.7.assoc.txt\n\ ## pve estimate in the null model = 0.954938\n\ ## se(pve) in the null model = 0.357032\n\ ## vg estimate in the null model = 76.0933\n\ ## ve estimate in the null model = 0.86612\n", ) .unwrap(); // chrX: no -loco on its command line, so the file name has to say. std::fs::write( dir.join("abcdef.X.assoc.txt.log.txt"), "## pve estimate in the null model = 0.856335\n", ) .unwrap(); // chr1: no pve block at all -- a log from a run without -pve. std::fs::write( dir.join("abcdef.1.assoc.txt.log.txt"), "total computation time = 1 s\n", ) .unwrap(); let components = read_variance_components_from_logs(&dir); let labels: Vec> = components.iter().map(|(chr, _)| chr.as_deref()).collect(); // Numeric order, chrX last, and the log without the block dropped. assert_eq!(labels, vec![Some("7"), Some("X")], "{labels:?}"); assert!((components[0].1.pve.unwrap() - 0.954938).abs() < 1e-12); assert!((components[0].1.se_pve.unwrap() - 0.357032).abs() < 1e-12); assert!((components[1].1.pve.unwrap() - 0.856335).abs() < 1e-12); std::fs::remove_dir_all(&dir).unwrap(); } #[test] fn a_directory_without_logs_yields_nothing() { assert!(read_variance_components_from_logs(Path::new("/nonexistent-gn4-dir")).is_empty()); } /// The log lines carry the unrounded numbers, because that is what a /// caller comparing engines wants to compute with, and because a /// `?id=` run is watched rather than returned. #[test] fn the_log_lines_carry_every_chromosome_unrounded() { let components = vec![( Some("7".to_string()), VarianceComponents { pve: Some(0.9549), se_pve: Some(0.357), vg: Some(76.0933), ve: Some(0.8661), }, )]; let lines = variance_components_log_lines(&components); assert_eq!(lines.len(), 1); assert!(lines[0].contains("variance: chr7 pve=0.9549"), "{}", lines[0]); assert!(lines[0].contains("se_pve=0.357"), "{}", lines[0]); assert!(lines[0].contains("vg=76.0933"), "{}", lines[0]); assert!(lines[0].contains("ve=0.8661"), "{}", lines[0]); } #[test] fn no_logs_yields_no_note() { assert_eq!(variance_components_note(&[]), None); } #[test] fn log_basenames_read_the_second_element_of_every_file_entry() { let record = serde_json::json!({ "files": [["1", "a.log.txt", "a.assoc.txt"], ["2", "b.log.txt", "b.assoc.txt"]] }); assert_eq!(log_file_basenames(&record), vec!["a.log.txt", "b.log.txt"]); } #[test] fn parse_wrapper_output_rejects_non_zero_errno() { use std::os::unix::process::ExitStatusExt; let output = Output { status: std::process::ExitStatus::from_raw(0), stdout: br#"{"errno": 1, "error": "boom"}"#.to_vec(), stderr: Vec::new(), }; let err = parse_wrapper_output("kinship", output).unwrap_err(); assert!(matches!( err, GemmaError::WrapperError { step: "kinship", .. } )); assert_eq!( err.to_string(), "gemma-wrapper (kinship) reported an error: boom" ); } #[test] fn kinship_args_is_loco_with_gk_and_no_lmm() { let args = kinship_args("/cache", "geno.txt.gz", "pheno.txt", "snps.txt"); assert_eq!( args, vec![ "--json", "--cache-dir", "/cache", "--loco", "--", "-g", "geno.txt.gz", "-p", "pheno.txt", "-a", "snps.txt", "-gk" ] ); } #[test] fn gwa_args_is_loco_with_lmm_and_no_gk() { let args = gwa_args( "/cache", "geno.txt.gz", "pheno.txt", "snps.txt", None, "9", "/cache/k.json", ); assert_eq!( args, vec![ "--json", "--cache-dir", "/cache", "--loco", "--input", "/cache/k.json", "--", "-g", "geno.txt.gz", "-p", "pheno.txt", "-a", "snps.txt", "-lmm", "9", "-pve" ] ); assert!( !args.contains(&"-gk"), "the GWA step must never carry -gk alongside -lmm" ); } } #[cfg(test)] mod effect_size_tests { use super::*; fn header() -> &'static str { "chr\trs\tps\tn_miss\tallele1\tallele0\taf\tbeta\tse\tl_mle\tp_lrt\n" } /// The column GeneNetwork pairs with the score, in GeneNetwork's /// convention: `additive = -beta/2`, placed immediately before the /// p-value column so nothing else moves. #[test] fn the_additive_effect_sits_just_before_the_score() { let contents = format!( "{}1\trs1\t1000\t0\tX\tY\t0.3\t2.0\t0.5\t1.0\t0.01\n\ 1\trs2\t2000\t0\tX\tY\t0.2\t-1.5\t0.4\t1.0\t0.2\n", header() ); let (columns, rows) = parse_assoc_file(&contents); assert_eq!( columns, vec![ "chr", "rs", "ps", "n_miss", "allele1", "allele0", "af", "beta", "se", "l_mle", "additive", "p_lrt" ] ); // GEMMA's own numbers are untouched, in place... assert_eq!(rows[0][7], serde_json::json!(2.0), "beta stays at index 7"); assert_eq!(rows[0][8], serde_json::json!(0.5), "se stays at index 8"); // ... and the derived column is next to the score, with the sign // and halving GN2 uses. assert_eq!(rows[0][10], serde_json::json!(-1.0), "additive = -beta/2"); assert_eq!(rows[1][10], serde_json::json!(0.75), "sign follows beta"); assert_eq!(rows[0][11], serde_json::json!(0.01), "p_lrt is still last"); } /// Peak detection reads the last column; that must survive the /// insertion, or every GEMMA peak link would silently point at an /// effect size instead of a p-value. #[test] fn the_peak_is_still_the_smallest_last_column() { let contents = format!( "{}1\trs1\t1000\t0\tX\tY\t0.3\t2.0\t0.5\t1.0\t0.01\n\ 1\trs2\t2000\t0\tX\tY\t0.2\t9.0\t0.4\t1.0\t1e-06\n", header() ); let (_, rows) = parse_assoc_file(&contents); let rows = Value::Array(rows.into_iter().map(Value::Array).collect()); assert_eq!(peak_marker(&rows).as_deref(), Some("rs2")); } /// A table without `beta` (a non-LMM run) is left exactly as GEMMA /// wrote it. #[test] fn a_table_without_beta_is_untouched() { let contents = "chr\trs\tps\tn_miss\tn_obs\tallele1\tallele0\taf\tscore\n\ 1\trs1\t1000\t0\t8\tX\tY\t0.3\t0.5\n"; let (columns, rows) = parse_assoc_file(contents); assert_eq!(columns, vec!["chr", "rs", "ps", "n_miss", "n_obs", "allele1", "allele0", "af", "score"]); assert_eq!(rows[0].len(), 9); } /// A missing beta in one row is a null effect, not a shifted row. #[test] fn a_row_without_beta_gets_a_null_effect() { let contents = format!( "{}1\trs1\t1000\t0\tX\tY\t0.3\tNA\t0.5\t1.0\t0.01\n", header() ); let (_, rows) = parse_assoc_file(&contents); assert_eq!(rows[0].len(), 12); assert_eq!(rows[0][10], Value::Null); } } #[cfg(test)] mod complaint_tests { use super::gemma_complaint; /// The wrapper's banner says nothing; GEMMA's own line says everything. #[test] fn the_complaint_is_gemmas_own_line_not_the_wrappers_banner() { let stdout = "\ Executing: gemma -gk 1 ... ## number of analyzed SNPs = 0 ERROR: Enforce failed for N>0 in src/fastblas.cpp at line 194 in fast_cblas_dgemm FATAL ERROR: gemma-wrapper bailed out with pid 42 exit 130 "; let got = gemma_complaint(stdout, ""); assert_eq!( got, "ERROR: Enforce failed for N>0 in src/fastblas.cpp at line 194 in fast_cblas_dgemm" ); } #[test] fn a_silent_failure_still_names_something() { let got = gemma_complaint("", "gemma-wrapper bailed out with pid 7 exit 1\n"); assert!(got.contains("bailed out"), "{got}"); } }