//! Core logic for POST /compute/gemma: compose and run gemma-wrapper //! against a genotype file, a SNP annotation file and trait values, and //! parse the resulting GEMMA association output into QTL mapping rows. //! //! Mirrors gn3/computations/gemma.py and gn3/api/gemma.py //! (GeneNetwork3), but synchronously: axum/tokio can await an external //! process directly, so there is no need for GeneNetwork3's Redis job //! queue. The actual invocation also differs from GeneNetwork3's in two //! respects, both found by running the real gemma-wrapper/GEMMA binaries //! against the BXD fixtures rather than trusting GeneNetwork3's own //! (entirely @unittest.skip'd) integration tests -- see `compute_gwa` //! and `compute` below. Progress-streaming (`run_id`/`status`) is //! shared infrastructure; see `crate::streaming`. use std::io; use std::path::Path; use std::process::Output; use serde_json::Value; use crate::streaming::{self, RunLogError}; /// One phenotype value per strain (genotype-file order), or `None` for /// a missing ("NA") sample. pub type TraitValues = Vec>; #[derive(Debug)] pub enum GemmaError { /// Failed to spawn or run a child process. Io(io::Error), /// gemma-wrapper exited non-zero; carries its stderr. CommandFailed { step: &'static str, stderr: String }, /// gemma-wrapper's own --json stdout record was not valid JSON, or /// reported an error (its own `errno` field was non-zero). WrapperError { step: &'static str, detail: String }, /// The real GEMMA `.assoc.txt` file gemma-wrapper pointed to could /// not be found or read. AssocFile(io::Error), /// gemma-wrapper's GWA output archive did not list any files. NoAssocFiles, /// `run_id` already has a log file (a run with that id is in /// progress, or finished and was never polled/cleaned up). Caught /// before any subprocess runs, so this never silently truncates /// another run's in-progress log. RunIdInUse(String), /// `ravanan` (see `compute_ravanan`) exited non-zero; carries its /// stderr. RavananFailed(String), /// `ravanan`'s own stdout (the CWL outputs record) was not valid /// JSON, or didn't carry the output this code expects. RavananOutput(String), } impl std::fmt::Display for GemmaError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { GemmaError::Io(e) => write!(f, "failed to run gemma-wrapper: {e}"), GemmaError::CommandFailed { step, stderr } => { write!(f, "gemma-wrapper ({step}) failed: {stderr}") } GemmaError::WrapperError { step, detail } => { write!(f, "gemma-wrapper ({step}) reported an error: {detail}") } GemmaError::AssocFile(e) => write!(f, "could not read GEMMA association output: {e}"), GemmaError::NoAssocFiles => { write!(f, "gemma-wrapper's response listed no association output files") } GemmaError::RunIdInUse(id) => { write!(f, "run id '{id}' is already in use; pick a different id") } GemmaError::RavananFailed(stderr) => write!(f, "ravanan failed: {stderr}"), GemmaError::RavananOutput(detail) => { write!(f, "could not read ravanan's output: {detail}") } } } } impl From for GemmaError { fn from(e: RunLogError) -> Self { match e { RunLogError::AlreadyInUse(id) => GemmaError::RunIdInUse(id), RunLogError::Io(e) => GemmaError::Io(e), } } } /// Write `values` as a GEMMA/BIMBAM phenotype file: one value per line, /// "NA" for a missing sample. Mirrors /// gn3.computations.gemma.generate_pheno_txt_file, minus its hash-based /// cache filename (the caller picks the path; see `compute`). pub fn write_pheno_file(path: &Path, values: &TraitValues) -> io::Result<()> { let mut contents = String::new(); for value in values { match value { Some(v) => contents.push_str(&v.to_string()), None => contents.push_str("NA"), } contents.push('\n'); } std::fs::write(path, contents) } fn parse_wrapper_output(step: &'static str, output: Output) -> Result { if !output.status.success() { return Err(GemmaError::CommandFailed { step, stderr: String::from_utf8_lossy(&output.stderr).into_owned(), }); } let stdout = String::from_utf8_lossy(&output.stdout); let record: Value = serde_json::from_str(&stdout).map_err(|e| GemmaError::WrapperError { step, detail: format!("could not parse gemma-wrapper's --json output: {e}"), })?; if record.get("errno").and_then(Value::as_i64).unwrap_or(0) != 0 { return Err(GemmaError::WrapperError { step, detail: record .get("error") .and_then(Value::as_str) .unwrap_or("gemma-wrapper reported a non-zero errno") .to_string(), }); } Ok(record) } /// Run gemma-wrapper's `--json` K-matrix step and return its stdout /// record (parsed JSON). /// /// `--loco` (bare, no chromosome list) is required here: confirmed by /// actually running gemma-wrapper against the BXD fixtures, a plain /// (non-LOCO) `-gk` only ever leaves its kinship matrix inside a /// `.tar.xz` archive in `--cache-dir`, with no loose file at the path /// the non-LOCO branch of gemma-wrapper's own GWA step later assumes -- /// that combination errors out (`gemma: ... open file ... CheckParam`). /// `--loco` is the combination gemma-wrapper's own USAGE banner /// documents and the one whose GWA-side code path actually unpacks the /// archive it needs (see `compute_gwa` and `compute`). Passing it with /// no chromosome list makes gemma-wrapper derive the full chromosome /// list from the SNP annotation file itself. async fn compute_kinship( gemma_wrapper_cmd: &str, cache_dir: &str, genotype_file: &str, pheno_file: &str, snps_file: &str, log_path: Option<&Path>, ) -> Result { let args = kinship_args(cache_dir, genotype_file, pheno_file, snps_file); let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path, &[]) .await .map_err(GemmaError::Io)?; parse_wrapper_output("kinship", output) } /// Run gemma-wrapper's `--json` GWA (association) step, given the /// K-step's JSON record written to `k_json_path` (gemma-wrapper's /// `--input` takes a file path, not stdin), and return its stdout /// record. `--loco` is required here for the same reason as in /// `compute_kinship`. /// /// Note: unlike GeneNetwork3's `generate_gemma_cmd` /// (gn3/computations/gemma.py), which always appends a literal "-gk" to /// the forwarded GEMMA args regardless of whether an LMM run was /// requested, this passes `-lmm` without `-gk`, matching gemma-wrapper's /// own documented usage (its USAGE banner in bin/gemma-wrapper). /// gemma-wrapper decides K-only vs. GWA purely by whether "-gk" is /// present in those args (`DO_COMPUTE_KINSHIP = /// gemma_args.include?("-gk")`); passing both flags together, as /// GeneNetwork3 does, makes every one of its calls a K-only run, and the /// `-lmm` value is silently ignored. That GeneNetwork3 code path has no /// non-skipped test covering it (tests/integration/test_gemma.py is /// `@unittest.skip`), so it is not knowingly replicated here. async fn compute_gwa( gemma_wrapper_cmd: &str, cache_dir: &str, genotype_file: &str, pheno_file: &str, snps_file: &str, lmm: u8, k_json_path: &str, log_path: Option<&Path>, ) -> Result { let lmm_str = lmm.to_string(); let args = gwa_args( cache_dir, genotype_file, pheno_file, snps_file, &lmm_str, k_json_path, ); let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path, &[]) .await .map_err(GemmaError::Io)?; parse_wrapper_output("gwa", output) } /// The association output basenames gemma-wrapper's GWA-step JSON /// record points at: `record["files"]` is a list of /// `[chr, log_basename, data_basename]`, one entry per chromosome in a /// `--loco` run (see `compute`). fn assoc_file_basenames(record: &Value) -> Vec<&str> { record .get("files") .and_then(Value::as_array) .map(|files| { files .iter() .filter_map(|entry| entry.as_array()?.get(2)?.as_str()) .collect() }) .unwrap_or_default() } /// Parse a GEMMA `.assoc.txt` file (tab-separated, header row first) /// into its column names and one row per marker, each row a /// `Value::Array` in that same column order (not keyed by column name: /// columns vary with `-lmm` -- stock 1/2/3 run one test, 4 runs all /// three; this build's `9`, GeneNetwork's convention, was observed to /// carry `p_lrt` -- so callers needing a name match it up against the /// returned column list themselves). A value that parses as a number is /// kept numeric; "NA" and anything else stays a string. fn parse_assoc_file(contents: &str) -> (Vec, Vec>) { let mut lines = contents.lines(); let Some(header) = lines.next() else { return (Vec::new(), Vec::new()); }; let columns: Vec = header.split('\t').map(str::to_string).collect(); let rows = lines .filter(|line| !line.is_empty()) .map(|line| { line.split('\t') .map(|value| { value .parse::() .map(|n| serde_json::json!(n)) .unwrap_or_else(|_| Value::String(value.to_string())) }) .collect() }) .collect(); (columns, rows) } /// Extract `archive` (a `.tar.xz`, relative to `cache_dir`) into /// `dest_dir`. Confirmed necessary by actually running the real /// binaries: a `--loco` GWA run's `.assoc.txt` files (one per /// chromosome) only ever end up inside this archive, never as loose /// files in `cache_dir` -- unlike the kinship step's `.cXX.txt` files, /// which do also appear loose (gemma-wrapper itself is inconsistent /// about this between the two steps). async fn extract_archive(cache_dir: &str, archive: &str, dest_dir: &str) -> Result<(), GemmaError> { let archive_path = Path::new(cache_dir).join(archive); let output = streaming::run_streaming( "tar", &["xJf", &archive_path.to_string_lossy(), "-C", dest_dir], None, &[], ) .await .map_err(GemmaError::Io)?; if !output.status.success() { return Err(GemmaError::CommandFailed { step: "extract-archive", stderr: String::from_utf8_lossy(&output.stderr).into_owned(), }); } Ok(()) } /// Read and parse every `basenames` file under `extract_dir` (one per /// `--loco` chromosome), returning the column names (from the first /// file; every chromosome's `.assoc.txt` shares the same header) and /// every file's rows concatenated, each row a `Value::Array`. fn parse_assoc_files(extract_dir: &Path, basenames: Vec<&str>) -> Result<(Vec, Vec), GemmaError> { let mut columns = Vec::new(); let mut rows = Vec::new(); for basename in basenames { let contents = std::fs::read_to_string(extract_dir.join(basename)) .map_err(GemmaError::AssocFile)?; let (file_columns, file_rows) = parse_assoc_file(&contents); if columns.is_empty() { columns = file_columns; } rows.extend(file_rows.into_iter().map(Value::Array)); } Ok((columns, rows)) } fn nanos_since_epoch() -> u128 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_nanos()) .unwrap_or(0) } /// The mapped QTL rows (each a `Value::Array` in `columns`' order, not /// a keyed object -- see `parse_assoc_file`), plus the exact /// gemma-wrapper invocations used to produce them (see `compute`'s /// `notes` below), so a caller can see precisely what was run without /// reading server logs. pub struct GemmaOutput { pub columns: Vec, pub rows: Value, pub notes: Vec, } /// The gemma-wrapper arguments for the kinship step, shared between the /// actual invocation (`compute_kinship`) and the note describing it /// (`compute`), so the two can never drift apart. fn kinship_args<'a>( cache_dir: &'a str, genotype_file: &'a str, pheno_file: &'a str, snps_file: &'a str, ) -> Vec<&'a str> { vec![ "--json", "--cache-dir", cache_dir, "--loco", "--", "-g", genotype_file, "-p", pheno_file, "-a", snps_file, "-gk", ] } /// The gemma-wrapper arguments for the GWA step, same pairing as /// `kinship_args`. fn gwa_args<'a>( cache_dir: &'a str, genotype_file: &'a str, pheno_file: &'a str, snps_file: &'a str, lmm_str: &'a str, k_json_path: &'a str, ) -> Vec<&'a str> { vec![ "--json", "--cache-dir", cache_dir, "--loco", "--input", k_json_path, "--", "-g", genotype_file, "-p", pheno_file, "-a", snps_file, "-lmm", lmm_str, ] } /// Run the full gemma-wrapper flow -- compute kinship, then run the LMM /// association test, both `--loco` -- and return the mapped QTL as a /// JSON array of rows (each a JSON object), concatenated across every /// chromosome's `.assoc.txt` from the real GEMMA output, plus the two /// gemma-wrapper invocations used (`GemmaOutput::notes`). The rows are /// one `Value::Array`, not a `Vec`: utoipa (for the OpenAPI /// schema) implements `ToSchema` for `serde_json::Value` itself, but /// `Vec` needs `Value: ComposeSchema`, which it does /// not implement in this utoipa version. pub async fn compute( gemma_wrapper_cmd: &str, cache_dir: &str, genotype_file: &str, snps_file: &str, trait_values: &TraitValues, lmm: u8, run_id: Option<&str>, ) -> Result { std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?; let unique = format!("{}-{}", std::process::id(), nanos_since_epoch()); let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt")); write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?; let pheno_file = pheno_path.to_string_lossy().into_owned(); // One log file for the whole request (both steps); see // crate::streaming::create_run_log for why this can't silently // collide with another run's. let log_path = streaming::create_run_log(cache_dir, run_id)?; let kinship_note = format!( "Kinship: {gemma_wrapper_cmd} {}", kinship_args(cache_dir, genotype_file, &pheno_file, snps_file).join(" ") ); let k_record = compute_kinship( gemma_wrapper_cmd, cache_dir, genotype_file, &pheno_file, snps_file, log_path.as_deref(), ) .await?; let k_json_path = Path::new(cache_dir).join(format!("{unique}-k.json")); std::fs::write(&k_json_path, k_record.to_string()).map_err(GemmaError::Io)?; let lmm_str = lmm.to_string(); let gwa_note = format!( "GWA: {gemma_wrapper_cmd} {}", gwa_args( cache_dir, genotype_file, &pheno_file, snps_file, &lmm_str, &k_json_path.to_string_lossy(), ) .join(" ") ); let gwa_record = compute_gwa( gemma_wrapper_cmd, cache_dir, genotype_file, &pheno_file, snps_file, lmm, &k_json_path.to_string_lossy(), log_path.as_deref(), ) .await?; let basenames = assoc_file_basenames(&gwa_record); if basenames.is_empty() { return Err(GemmaError::NoAssocFiles); } let archive = gwa_record .get("archive") .and_then(Value::as_str) .ok_or_else(|| GemmaError::WrapperError { step: "gwa", detail: "gemma-wrapper's response did not list an output archive".to_string(), })?; let extract_dir = Path::new(cache_dir).join(format!("{unique}-gwa-extract")); std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?; extract_archive(cache_dir, archive, &extract_dir.to_string_lossy()).await?; let (columns, rows) = parse_assoc_files(&extract_dir, basenames)?; let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless streaming::mark_complete(&log_path); Ok(GemmaOutput { columns, rows: Value::Array(rows), notes: vec![kinship_note, gwa_note], }) } /// Read a scattered `File[]` output's list of `path`s from ravanan's /// stdout (the CWL outputs record for a scattered output: /// `{"": [{"class": "File", "path": ..., ...}, ...], ...}`, /// one entry per `cwl/gemma/gemma-loco.scm` chromosome). fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result, GemmaError> { let files = outputs .get(id) .and_then(Value::as_array) .ok_or_else(|| GemmaError::RavananOutput(format!("no \"{id}\" array in ravanan's output")))?; files .iter() .map(|file| { file.get("path").and_then(Value::as_str).ok_or_else(|| { GemmaError::RavananOutput(format!("no .path on one of \"{id}\"'s entries")) }) }) .collect() } /// The distinct chromosome names in `snps_file` (whitespace-separated: /// rs id, position, chromosome), in first-seen order -- reproduces /// gemma-wrapper's own `get_chromosomes`, used here to build /// `compute_ravanan`'s `chromosomes` scatter input (CWL's `scatter` /// needs a concrete array at invocation time, which can't be /// discovered by a step inside the workflow itself without another /// round of plumbing -- see `cwl/gemma/gemma-loco.scm`'s own doc /// comment on this input). fn chromosomes_in(snps_file: &str) -> Result, GemmaError> { let contents = std::fs::read_to_string(snps_file).map_err(GemmaError::Io)?; let mut seen = std::collections::HashSet::new(); let mut chromosomes = Vec::new(); for line in contents.lines() { if let Some(chr) = line.split_whitespace().nth(2) { if seen.insert(chr.to_string()) { chromosomes.push(chr.to_string()); } } } Ok(chromosomes) } /// The same GEMMA `--loco` kinship-then-GWA computation as `compute`, /// but run via `ravanan` (https://github.com/vepiteski/ravanan) driving /// a CWL workflow compiled from the ccwl (https://ccwl.systemreboot.net/) /// source at `workflow_scm_path` (see `cwl/gemma/gemma-loco.scm`) /// instead of this module's own direct subprocess calls: one /// gemma-wrapper invocation per chromosome, scattered by ravanan /// itself (`--chromosomes`/`--no-parallel`, see /// `cwl/gemma/gemma-loco.scm`'s `kinship` and `gwa` commands) rather /// than gemma-wrapper's own internal GNU parallel fan-out -- an alternative /// runner for the same computation, so the two can be compared. /// Backs `POST /compute/gemma/ravanan`. pub async fn compute_ravanan( ravanan_cmd: &str, ccwl_cmd: &str, workflow_scm_path: &str, manifest_path: &str, cache_dir: &str, genotype_file: &str, snps_file: &str, trait_values: &TraitValues, lmm: u8, run_id: Option<&str>, ) -> Result { std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?; let unique = format!("{}-{}", std::process::id(), nanos_since_epoch()); let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt")); write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?; let log_path = streaming::create_run_log(cache_dir, run_id)?; // gemma-loco.scm is the maintained source; the CWL ravanan actually // runs is a generated, throwaway artifact compiled fresh on every // call (cheap: ccwl compile is pure macro expansion, no Guix build // involved) and never checked into the repo -- see the doc comment // on Config::gemma_ravanan_workflow. let compiled_workflow_path = Path::new(cache_dir).join(format!("{unique}-gemma-loco.cwl")); let ccwl_output_arg = format!("--output={}", compiled_workflow_path.display()); let ccwl_args = vec!["compile", ccwl_output_arg.as_str(), workflow_scm_path]; let ccwl_output = streaming::run_streaming(ccwl_cmd, &ccwl_args, None, &[]) .await .map_err(GemmaError::Io)?; if !ccwl_output.status.success() { return Err(GemmaError::CommandFailed { step: "ccwl compile", stderr: String::from_utf8_lossy(&ccwl_output.stderr).into_owned(), }); } let compiled_workflow_arg = compiled_workflow_path.to_string_lossy().into_owned(); let store_dir = Path::new(cache_dir).join("ravanan-store"); // ravanan's --store needs these three subdirectories to already // exist; it does not create them itself. for subdir in ["files", "logs", "data"] { std::fs::create_dir_all(store_dir.join(subdir)).map_err(GemmaError::Io)?; } let chromosomes = chromosomes_in(snps_file)?; if chromosomes.is_empty() { return Err(GemmaError::NoAssocFiles); } let chromosomes_yaml = chromosomes .iter() .map(|c| format!("\"{c}\"")) .collect::>() .join(", "); // One persistent cache directory per chromosome, reused across // requests, so gemma-wrapper's content-hash cache actually gets hit // on repeated identical requests -- unlike "." (each job's own // hermetic CWL working directory under ravanan's --store, fresh // every run), which never accumulates a cache at all. Keyed by // chromosome, not shared, because gemma-wrapper's cache-hit hash // does not vary with --chromosomes: a shared directory would let // one chromosome's job silently reuse another's cached result. let wrapper_cache_dir = Path::new(cache_dir).join("gemma-wrapper-cache"); let mut chr_cache_dirs = Vec::new(); for chr in &chromosomes { let chr_cache_dir = wrapper_cache_dir.join(format!("chr-{chr}")); std::fs::create_dir_all(&chr_cache_dir).map_err(GemmaError::Io)?; chr_cache_dirs.push(chr_cache_dir); } let cache_dirs_yaml = chr_cache_dirs .iter() .map(|d| format!("\"{}\"", d.display())) .collect::>() .join(", "); let inputs_yaml_path = Path::new(cache_dir).join(format!("{unique}-ravanan-inputs.yaml")); std::fs::write( &inputs_yaml_path, format!( "genotype_file: {{class: File, path: {genotype_file}}}\n\ pheno_file: {{class: File, path: {}}}\n\ snps_file: {{class: File, path: {snps_file}}}\n\ lmm: \"{lmm}\"\n\ chromosomes: [{chromosomes_yaml}]\n\ cache_dirs: [{cache_dirs_yaml}]\n", pheno_path.display(), ), ) .map_err(GemmaError::Io)?; let store_arg = format!("--store={}", store_dir.display()); let manifest_arg = format!("--guix-manifest={manifest_path}"); let inputs_yaml_arg = inputs_yaml_path.to_string_lossy().into_owned(); // No --output-directory: the gwa step's per-chromosome archive // outputs all share gemma-wrapper's own content-hash basename (its // hash doesn't vary with --chromosomes), and ravanan's // --output-directory step symlinks every scattered output into one // flat directory by basename, which collides across chromosomes -- // a ravanan/gemma-wrapper interaction bug, not something fixable // on either side alone. Harmless to skip: it's optional, and this // code reads paths straight out of ravanan's own --store via its // stdout output record, never through --output-directory. let args = vec![ store_arg.as_str(), manifest_arg.as_str(), compiled_workflow_arg.as_str(), inputs_yaml_arg.as_str(), ]; let ravanan_note = format!("ravanan: {ravanan_cmd} {}", args.join(" ")); let chromosomes_note = format!( "Scattered gemma-wrapper over {} chromosome(s) via CWL scatter \ (--chromosomes/--no-parallel per job, so GNU parallel is never \ invoked), each against its own persistent, chromosome-keyed \ --cache-dir under {}: {}", chromosomes.len(), wrapper_cache_dir.display(), chromosomes.join(", "), ); let output = streaming::run_streaming(ravanan_cmd, &args, log_path.as_deref(), &[]) .await .map_err(GemmaError::Io)?; if !output.status.success() { return Err(GemmaError::RavananFailed( String::from_utf8_lossy(&output.stderr).into_owned(), )); } let outputs: Value = serde_json::from_str(&String::from_utf8_lossy(&output.stdout)) .map_err(|e| GemmaError::RavananOutput(format!("could not parse ravanan's stdout: {e}")))?; let gwa_json_paths = ravanan_output_paths(&outputs, "gwa_json")?; let archive_paths = ravanan_output_paths(&outputs, "archive")?; if gwa_json_paths.len() != archive_paths.len() { return Err(GemmaError::RavananOutput( "\"gwa_json\" and \"archive\" arrays have different lengths".to_string(), )); } // One archive per chromosome now (gwa.cwl's scatter), not one // shared archive as compute() and this function's own kinship step // get -- each gets its own extract dir and its basenames' rows are // concatenated across all of them. let mut columns = Vec::new(); let mut rows = Vec::new(); for (i, (gwa_json_path, archive_path)) in gwa_json_paths.iter().zip(archive_paths.iter()).enumerate() { let gwa_record: Value = serde_json::from_str( &std::fs::read_to_string(gwa_json_path).map_err(GemmaError::AssocFile)?, ) .map_err(|e| GemmaError::WrapperError { step: "gwa", detail: format!("could not parse gemma-wrapper's --json output: {e}"), })?; let basenames = assoc_file_basenames(&gwa_record); if basenames.is_empty() { return Err(GemmaError::NoAssocFiles); } let extract_dir = Path::new(cache_dir).join(format!("{unique}-ravanan-extract-{i}")); std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?; let tar_output = streaming::run_streaming( "tar", &["xJf", archive_path, "-C", &extract_dir.to_string_lossy()], None, &[], ) .await .map_err(GemmaError::Io)?; if !tar_output.status.success() { return Err(GemmaError::CommandFailed { step: "extract-archive", stderr: String::from_utf8_lossy(&tar_output.stderr).into_owned(), }); } let (file_columns, file_rows) = parse_assoc_files(&extract_dir, basenames)?; if columns.is_empty() { columns = file_columns; } rows.extend(file_rows); let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless } streaming::mark_complete(&log_path); Ok(GemmaOutput { columns, rows: Value::Array(rows), notes: vec![ ravanan_note, chromosomes_note, format!("workflow: {workflow_scm_path} (compiled to {compiled_workflow_arg})"), ], }) } #[cfg(test)] mod tests { use super::*; #[test] fn write_pheno_file_writes_one_value_per_line_and_na_for_missing() { let path = std::env::temp_dir().join(format!("gn4-gemma-test-{}", std::process::id())); write_pheno_file(&path, &vec![Some(9.944), None, Some(8.572)]).unwrap(); let contents = std::fs::read_to_string(&path).unwrap(); std::fs::remove_file(&path).unwrap(); assert_eq!(contents, "9.944\nNA\n8.572\n"); } #[test] fn parse_assoc_file_parses_header_and_numeric_rows() { let contents = "chr\trs\tps\tp_wald\n1\trs001\t3010274\t0.05\n1\trs002\t3492195\tNA\n"; let (columns, rows) = parse_assoc_file(contents); assert_eq!(columns, vec!["chr", "rs", "ps", "p_wald"]); assert_eq!(rows.len(), 2); assert_eq!(rows[0], vec![ serde_json::json!(1.0), serde_json::json!("rs001"), serde_json::json!(3010274.0), serde_json::json!(0.05), ]); assert_eq!(rows[1][3], serde_json::json!("NA")); } #[test] fn parse_assoc_file_empty_contents_yields_no_rows() { let (columns, rows) = parse_assoc_file(""); assert_eq!(columns.len(), 0); assert_eq!(rows.len(), 0); } #[test] fn assoc_file_basenames_reads_the_third_element_of_every_file_entry() { let record = serde_json::json!({ "files": [["1", "a.log.txt", "a.assoc.txt"], ["2", "b.log.txt", "b.assoc.txt"]] }); assert_eq!(assoc_file_basenames(&record), vec!["a.assoc.txt", "b.assoc.txt"]); } #[test] fn assoc_file_basenames_missing_files_yields_empty() { assert_eq!(assoc_file_basenames(&serde_json::json!({})), Vec::<&str>::new()); } #[test] fn parse_wrapper_output_rejects_non_zero_errno() { use std::os::unix::process::ExitStatusExt; let output = Output { status: std::process::ExitStatus::from_raw(0), stdout: br#"{"errno": 1, "error": "boom"}"#.to_vec(), stderr: Vec::new(), }; let err = parse_wrapper_output("kinship", output).unwrap_err(); assert!(matches!(err, GemmaError::WrapperError { step: "kinship", .. })); assert_eq!(err.to_string(), "gemma-wrapper (kinship) reported an error: boom"); } #[test] fn kinship_args_is_loco_with_gk_and_no_lmm() { let args = kinship_args("/cache", "geno.txt.gz", "pheno.txt", "snps.txt"); assert_eq!( args, vec![ "--json", "--cache-dir", "/cache", "--loco", "--", "-g", "geno.txt.gz", "-p", "pheno.txt", "-a", "snps.txt", "-gk" ] ); } #[test] fn gwa_args_is_loco_with_lmm_and_no_gk() { let args = gwa_args("/cache", "geno.txt.gz", "pheno.txt", "snps.txt", "9", "/cache/k.json"); assert_eq!( args, vec![ "--json", "--cache-dir", "/cache", "--loco", "--input", "/cache/k.json", "--", "-g", "geno.txt.gz", "-p", "pheno.txt", "-a", "snps.txt", "-lmm", "9" ] ); assert!(!args.contains(&"-gk"), "the GWA step must never carry -gk alongside -lmm"); } }