diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-14 04:14:07 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-14 04:14:07 +0000 |
| commit | 0e017416628d72ae9814b8225d17f62b28f855ee (patch) | |
| tree | a073e1f8dd009b548cab0cb0bfc95904c3b3d77f /src | |
| parent | f8b34ca82b40f9eda65fa3abab580a0ca8e66f49 (diff) | |
| download | genenetwork4-0e017416628d72ae9814b8225d17f62b28f855ee.tar.gz | |
ravanan+gemma: call gemma itself, not gemma-wrapper
gemma-wrapper was doing two things ravanan now does or GEMMA already does: the parallel fan-out (ravanan's CWL scatter replaced it long ago, which is why every job passed --no-parallel) and LOCO orchestration -- gemma 0.98.6 documents `-loco [chr]` natively for both -gk and -lmm and takes the annotation file it needs. What was left was a content-hash K cache and a --json record with results packed into .tar.xz archives. So cwl/gemma/gemma-loco.scm now runs two gemma invocations per chromosome: `-gk -loco <chr> -o <chr>-kin`, then `-k <that matrix> -lmm <n> -loco <chr> -pve -o <chr>-gwa`, with `-outdir .` for flat, deterministic names. The kinship matrix travels from one step to the other as a CWL File output, and each job's .assoc.txt and .log.txt are the workflow's outputs -- so compute_ravanan reads files instead of parsing a wrapper JSON record and unpacking archives, and the pve note comes straight out of the jobs' logs. Measured on tests/test_data/BXD (chr 1 and chr 2): identical rows -- 636/636 and 583/583 on beta, se, l_mle, p_lrt. One chromosome end to end: 496 ms (gemma) vs 1140 ms (wrapper), 2.3x per job. The wrapper also needed Ruby and pfff on PATH, and named its outputs with the extension doubled (...1.assoc.txt.assoc.txt). /compute/gemma keeps gemma-wrapper, deliberately: that is what makes the two endpoints comparable rather than two copies of one stack. scripts/gemma-compare.py runs the same body through both and reports row-level differences and timings. The gemma-wrapper content-hash K cache is the one thing given up; a repeat request recomputes (271 ms of kinship on this fixture).
Diffstat (limited to 'src')
| -rw-r--r-- | src/gemma.rs | 260 |
1 files changed, 88 insertions, 172 deletions
diff --git a/src/gemma.rs b/src/gemma.rs index a41cc7b..ca9c522 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -436,39 +436,67 @@ fn read_variance_components_from_logs(dir: &Path) -> Vec<(Option<String>, Varian if !path.to_string_lossy().ends_with(".log.txt") { continue; } - let Ok(log) = std::fs::read_to_string(&path) else { - continue; - }; - let components = parse_variance_components(&log); - if components.pve.is_none() && components.vg.is_none() && components.ve.is_none() { - continue; + if let Some(components) = variance_components_from_log(&path) { + found.push(components); } - let chromosome = log - .lines() - .find_map(|line| { - let mut words = line.split_whitespace(); - while let Some(word) = words.next() { - if word == "-loco" { - return words.next().map(str::to_string); - } - } - None - }) - .or_else(|| { - // <hash>.<chr>.assoc.txt.log.txt - let name = path.file_name()?.to_string_lossy().into_owned(); - let after_hash = name.split('.').nth(1)?; - Some(after_hash.to_string()) - }); - found.push((chromosome, components)); } - // A stable order, so the note reads chr1..chrX rather than in whatever - // order the filesystem listed them. + sort_by_chromosome(&mut found); + found +} + +/// The same, for log paths the workflow hands back directly: since the +/// workflow calls `gemma` itself, each job's `.log.txt` is a CWL output rather +/// than a file packed into an archive. +fn read_variance_components_from_log_files( + paths: &[&str], +) -> Vec<(Option<String>, VarianceComponents)> { + let mut found: Vec<(Option<String>, VarianceComponents)> = paths + .iter() + .filter_map(|path| variance_components_from_log(Path::new(path))) + .collect(); + sort_by_chromosome(&mut found); + found +} + +/// One GEMMA `.log.txt`, labelled by the chromosome in its own command line +/// (`gemma ... -loco 7 ...`) with the file name as a fallback. +fn variance_components_from_log(path: &Path) -> Option<(Option<String>, VarianceComponents)> { + let log = std::fs::read_to_string(path).ok()?; + let components = parse_variance_components(&log); + if components.pve.is_none() && components.vg.is_none() && components.ve.is_none() { + return None; + } + let chromosome = log + .lines() + .find_map(|line| { + let mut words = line.split_whitespace(); + while let Some(word) = words.next() { + if word == "-loco" { + return words.next().map(str::to_string); + } + } + None + }) + .or_else(|| { + let name = path.file_name()?.to_string_lossy().into_owned(); + let stem = name.strip_suffix(".log.txt").unwrap_or(&name); + // gemma-wrapper's `<hash>.<chr>.assoc.txt.log.txt`, or this + // workflow's own `<chr>-gwa.log.txt`. + if let Some(rest) = stem.strip_suffix(".assoc.txt") { + return rest.split('.').nth(1).map(str::to_string); + } + stem.split('-').next().map(str::to_string) + }); + Some((chromosome, components)) +} + +/// A stable order, so the note reads chr1..chrX rather than in whatever order +/// the filesystem listed the files. +fn sort_by_chromosome(found: &mut [(Option<String>, VarianceComponents)]) { found.sort_by_key(|(chr, _)| { chr.as_deref() .map(|chr| (chr.parse::<u32>().unwrap_or(u32::MAX), chr.to_string())) }); - found } /// One note carrying every chromosome's variance components. @@ -991,15 +1019,15 @@ fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str> /// The distinct chromosome names in a SNP annotation file's contents /// (whitespace-separated: rs id, position, chromosome), in first-seen -/// order -- reproduces gemma-wrapper's own `get_chromosomes`, used here -/// to build `compute_ravanan`'s `chromosomes` scatter input (CWL's +/// order, used here to build `compute_ravanan`'s `chromosomes` scatter +/// input: GEMMA's own `-loco <chr>` takes a chromosome name, so the +/// annotation file decides the scatter (CWL's /// `scatter` needs a concrete array at invocation time, which can't be /// discovered by a step inside the workflow itself without another /// round of plumbing -- see `cwl/gemma/gemma-loco.scm`'s own doc /// comment on this input). /// -/// Takes the contents, not the path: the caller also hashes them (see -/// [`gemma_wrapper_input_key`]) and should not read the file twice. +/// Takes the contents, not the path, so the caller reads the file once. fn chromosomes_from(contents: &str) -> Vec<String> { let mut seen = std::collections::HashSet::new(); let mut chromosomes = Vec::new(); @@ -1013,51 +1041,17 @@ fn chromosomes_from(contents: &str) -> Vec<String> { chromosomes } -/// A stable identity for one request's GEMMA inputs, used to give each -/// distinct input set its own gemma-wrapper cache root (see -/// `compute_ravanan`). -/// -/// The genotype file is identified by its path and size rather than its -/// contents: hashing a genome-sized file on every request would cost -/// more than the cache saves, and the path is stable in practice (it is -/// the caller's own file, interned verbatim into ravanan's store). -fn gemma_wrapper_input_key( - genotype_file: &str, - snps_contents: &str, - trait_values: &TraitValues, - lmm: u8, -) -> String { - let mut material = Vec::new(); - material.extend_from_slice(genotype_file.as_bytes()); - material.push(0); - if let Ok(metadata) = std::fs::metadata(genotype_file) { - material.extend_from_slice(&metadata.len().to_le_bytes()); - } - material.push(0); - material.extend_from_slice(snps_contents.as_bytes()); - material.push(0); - material.extend_from_slice(lmm.to_string().as_bytes()); - material.push(0); - for value in trait_values { - match value { - Some(value) => material.extend_from_slice(&value.to_bits().to_le_bytes()), - None => material.extend_from_slice(b"NA"), - } - material.push(b','); - } - crate::indexer::sha256::hex_digest(&material)[..16].to_string() -} - /// The same GEMMA `--loco` kinship-then-GWA computation as `compute`, /// but run via `ravanan` (https://github.com/vepiteski/ravanan) driving /// a CWL workflow compiled from the ccwl (https://ccwl.systemreboot.net/) -/// source at `workflow_scm_path` (see `cwl/gemma/gemma-loco.scm`) -/// instead of this module's own direct subprocess calls: one -/// gemma-wrapper invocation per chromosome, scattered by ravanan -/// itself (`--chromosomes`/`--no-parallel`, see -/// `cwl/gemma/gemma-loco.scm`'s `kinship` and `gwa` commands) rather -/// than gemma-wrapper's own internal GNU parallel fan-out -- an alternative -/// runner for the same computation, so the two can be compared. +/// source at `workflow_scm_path` (see `cwl/gemma/gemma-loco.scm`). +/// +/// This runner invokes **gemma itself**, one invocation per +/// chromosome, relying on `-loco <chr>` being native to 0.98.6 for both +/// `-gk` and `-lmm` -- where `compute` still drives gemma-wrapper. That +/// is deliberate: it is what makes the two endpoints comparable, the +/// same computation through two stacks, and the day the numbers stop +/// agreeing the difference is visible rather than assumed away. /// Backs `POST /compute/gemma/ravanan`. pub async fn compute_ravanan( ravanan_cmd: &str, @@ -1121,39 +1115,6 @@ pub async fn compute_ravanan( .collect::<Vec<_>>() .join(", "); - // A persistent cache directory per (input set, chromosome), reused - // across requests, so gemma-wrapper's content-hash cache actually - // gets hit on repeated identical requests -- unlike "." (each job's - // own hermetic CWL working directory under ravanan's --store, fresh - // every run), which never accumulates a cache at all. Keyed by - // chromosome as well as the input set, because gemma-wrapper's - // cache-hit hash does not vary with --chromosomes: a directory - // shared between chromosomes would let one chromosome's job - // silently reuse another's cached result. - // - // Keyed by the input set too, because gemma-wrapper names its - // archives by a *content* hash: two different input sets (another - // trait, or another snps/genotype file) sharing a `chr-N` directory - // leave two `*-gemma-cXX.tar.xz` files there, and the CWL's - // single-File glob output binding (`$(inputs.cache_dir)/*-gemma-cXX.tar.xz`) - // then matches more than one path and the step fails -- with an - // error that ravanan's `user-error` silently drops in a job script. - let input_key = gemma_wrapper_input_key(genotype_file, &snps_contents, trait_values, lmm); - let wrapper_cache_dir = Path::new(cache_dir) - .join("gemma-wrapper-cache") - .join(&input_key); - let mut chr_cache_dirs = Vec::new(); - for chr in &chromosomes { - let chr_cache_dir = wrapper_cache_dir.join(format!("chr-{chr}")); - std::fs::create_dir_all(&chr_cache_dir).map_err(GemmaError::Io)?; - chr_cache_dirs.push(chr_cache_dir); - } - let cache_dirs_yaml = chr_cache_dirs - .iter() - .map(|d| format!("\"{}\"", d.display())) - .collect::<Vec<_>>() - .join(", "); - let inputs_yaml_path = Path::new(cache_dir).join(format!("{unique}-ravanan-inputs.yaml")); std::fs::write( &inputs_yaml_path, @@ -1162,8 +1123,7 @@ pub async fn compute_ravanan( pheno_file: {{class: File, path: {}}}\n\ snps_file: {{class: File, path: {snps_file}}}\n\ lmm: \"{lmm}\"\n\ - chromosomes: [{chromosomes_yaml}]\n\ - cache_dirs: [{cache_dirs_yaml}]\n", + chromosomes: [{chromosomes_yaml}]\n", pheno_path.display(), ), ) @@ -1172,15 +1132,10 @@ pub async fn compute_ravanan( let store_arg = format!("--store={}", store_dir.display()); let manifest_arg = format!("--guix-manifest={manifest_path}"); let inputs_yaml_arg = inputs_yaml_path.to_string_lossy().into_owned(); - // No --output-directory: the gwa step's per-chromosome archive - // outputs all share gemma-wrapper's own content-hash basename (its - // hash doesn't vary with --chromosomes), and ravanan's - // --output-directory step symlinks every scattered output into one - // flat directory by basename, which collides across chromosomes -- - // a ravanan/gemma-wrapper interaction bug, not something fixable - // on either side alone. Harmless to skip: it's optional, and this - // code reads paths straight out of ravanan's own --store via its - // stdout output record, never through --output-directory. + // No --output-directory: this code reads the scattered outputs' + // paths straight out of ravanan's own --store via its stdout record, + // never through a flattened copy, and the workflow names each + // chromosome's files `<chr>-gwa.*` itself. // Backend switches: nothing for the default local runner, and // --batch-system=slurm-api plus endpoint/JWT/partition/scratch for a // cluster (see docs/ravanan-slurm.md). `scratch` is the cache dir @@ -1208,12 +1163,9 @@ pub async fn compute_ravanan( args.join(" ") ); let chromosomes_note = format!( - "Scattered gemma-wrapper over {} chromosome(s) via CWL scatter \ - (--chromosomes/--no-parallel per job, so GNU parallel is never \ - invoked), each against its own persistent, chromosome-keyed \ - --cache-dir under {}: {}", + "Scattered gemma itself over {} chromosome(s) via CWL scatter, one \ + invocation per chromosome: {}", chromosomes.len(), - wrapper_cache_dir.display(), chromosomes.join(", "), ); @@ -1230,64 +1182,28 @@ pub async fn compute_ravanan( } let outputs: Value = serde_json::from_str(&String::from_utf8_lossy(&output.stdout)) .map_err(|e| GemmaError::RavananOutput(format!("could not parse ravanan's stdout: {e}")))?; - let gwa_json_paths = ravanan_output_paths(&outputs, "gwa_json")?; - let archive_paths = ravanan_output_paths(&outputs, "archive")?; - if gwa_json_paths.len() != archive_paths.len() { - return Err(GemmaError::RavananOutput( - "\"gwa_json\" and \"archive\" arrays have different lengths".to_string(), - )); + let assoc_paths = ravanan_output_paths(&outputs, "assoc")?; + let gwa_log_paths = ravanan_output_paths(&outputs, "gwa_log")?; + if assoc_paths.is_empty() { + return Err(GemmaError::NoAssocFiles); } - // One archive per chromosome now (gwa.cwl's scatter), not one - // shared archive as compute() and this function's own kinship step - // get -- each gets its own extract dir and its basenames' rows are - // concatenated across all of them. + // One .assoc.txt and one .log.txt per chromosome, straight from the + // workflow's own outputs: gemma writes named files (`-o <chr>-gwa`), so + // there is no wrapper JSON record to parse and no archive to unpack. let mut columns = Vec::new(); let mut rows = Vec::new(); - let mut components: Vec<(Option<String>, VarianceComponents)> = Vec::new(); - for (i, (gwa_json_path, archive_path)) in gwa_json_paths.iter().zip(archive_paths.iter()).enumerate() { - let gwa_record: Value = serde_json::from_str( - &std::fs::read_to_string(gwa_json_path).map_err(GemmaError::AssocFile)?, - ) - .map_err(|e| GemmaError::WrapperError { - step: "gwa", - detail: format!("could not parse gemma-wrapper's --json output: {e}"), - })?; - let basenames = assoc_file_basenames(&gwa_record); - if basenames.is_empty() { - return Err(GemmaError::NoAssocFiles); - } - - let extract_dir = Path::new(cache_dir).join(format!("{unique}-ravanan-extract-{i}")); - std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?; - let tar_output = streaming::run_streaming( - "tar", - &["xJf", archive_path, "-C", &extract_dir.to_string_lossy()], - None, - ) - .await - .map_err(GemmaError::Io)?; - if !tar_output.status.success() { - return Err(GemmaError::CommandFailed { - step: "extract-archive", - stderr: String::from_utf8_lossy(&tar_output.stderr).into_owned(), - }); - } - - let (file_columns, file_rows) = parse_assoc_files(&extract_dir, basenames)?; + for assoc_path in &assoc_paths { + let contents = std::fs::read_to_string(assoc_path).map_err(GemmaError::AssocFile)?; + let (file_columns, file_rows) = parse_assoc_file(&contents); if columns.is_empty() { columns = file_columns; } - rows.extend(file_rows); - // The archive holds this chromosome's GEMMA `.log.txt`, which is where - // `-pve` writes the variance components. Read them from there rather - // than from beside the archive: the copy beside it is a 5eb53fa - // feature, and a ravanan job's gemma-wrapper comes from the job's own - // Guix manifest, which need not be that new. (The extract dir is - // removed below, so this has to happen first.) - components.extend(read_variance_components_from_logs(&extract_dir)); - let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless + rows.extend(file_rows.into_iter().map(Value::Array)); } + // `-pve` puts the variance components in each job's .log.txt, which the + // workflow returns as a first-class output. + let components = read_variance_components_from_log_files(&gwa_log_paths); streaming::mark_complete(&log_path); |
