about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-11-14 04:14:07 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-11-14 04:14:07 +0000
commit0e017416628d72ae9814b8225d17f62b28f855ee (patch)
treea073e1f8dd009b548cab0cb0bfc95904c3b3d77f /src
parentf8b34ca82b40f9eda65fa3abab580a0ca8e66f49 (diff)
downloadgenenetwork4-0e017416628d72ae9814b8225d17f62b28f855ee.tar.gz
ravanan+gemma: call gemma itself, not gemma-wrapper
gemma-wrapper was doing two things ravanan now does or GEMMA already
does: the parallel fan-out (ravanan's CWL scatter replaced it long ago,
which is why every job passed --no-parallel) and LOCO orchestration --
gemma 0.98.6 documents `-loco [chr]` natively for both -gk and -lmm and
takes the annotation file it needs. What was left was a content-hash K
cache and a --json record with results packed into .tar.xz archives.

So cwl/gemma/gemma-loco.scm now runs two gemma invocations per
chromosome: `-gk -loco <chr> -o <chr>-kin`, then `-k <that matrix>
-lmm <n> -loco <chr> -pve -o <chr>-gwa`, with `-outdir .` for flat,
deterministic names. The kinship matrix travels from one step to the
other as a CWL File output, and each job's .assoc.txt and .log.txt are
the workflow's outputs -- so compute_ravanan reads files instead of
parsing a wrapper JSON record and unpacking archives, and the pve note
comes straight out of the jobs' logs.

Measured on tests/test_data/BXD (chr 1 and chr 2): identical rows --
636/636 and 583/583 on beta, se, l_mle, p_lrt. One chromosome end to
end: 496 ms (gemma) vs 1140 ms (wrapper), 2.3x per job. The wrapper
also needed Ruby and pfff on PATH, and named its outputs with the
extension doubled (...1.assoc.txt.assoc.txt).

/compute/gemma keeps gemma-wrapper, deliberately: that is what makes
the two endpoints comparable rather than two copies of one stack.
scripts/gemma-compare.py runs the same body through both and reports
row-level differences and timings.

The gemma-wrapper content-hash K cache is the one thing given up;
a repeat request recomputes (271 ms of kinship on this fixture).
Diffstat (limited to 'src')
-rw-r--r--src/gemma.rs260
1 files changed, 88 insertions, 172 deletions
diff --git a/src/gemma.rs b/src/gemma.rs
index a41cc7b..ca9c522 100644
--- a/src/gemma.rs
+++ b/src/gemma.rs
@@ -436,39 +436,67 @@ fn read_variance_components_from_logs(dir: &Path) -> Vec<(Option<String>, Varian
         if !path.to_string_lossy().ends_with(".log.txt") {
             continue;
         }
-        let Ok(log) = std::fs::read_to_string(&path) else {
-            continue;
-        };
-        let components = parse_variance_components(&log);
-        if components.pve.is_none() && components.vg.is_none() && components.ve.is_none() {
-            continue;
+        if let Some(components) = variance_components_from_log(&path) {
+            found.push(components);
         }
-        let chromosome = log
-            .lines()
-            .find_map(|line| {
-                let mut words = line.split_whitespace();
-                while let Some(word) = words.next() {
-                    if word == "-loco" {
-                        return words.next().map(str::to_string);
-                    }
-                }
-                None
-            })
-            .or_else(|| {
-                // <hash>.<chr>.assoc.txt.log.txt
-                let name = path.file_name()?.to_string_lossy().into_owned();
-                let after_hash = name.split('.').nth(1)?;
-                Some(after_hash.to_string())
-            });
-        found.push((chromosome, components));
     }
-    // A stable order, so the note reads chr1..chrX rather than in whatever
-    // order the filesystem listed them.
+    sort_by_chromosome(&mut found);
+    found
+}
+
+/// The same, for log paths the workflow hands back directly: since the
+/// workflow calls `gemma` itself, each job's `.log.txt` is a CWL output rather
+/// than a file packed into an archive.
+fn read_variance_components_from_log_files(
+    paths: &[&str],
+) -> Vec<(Option<String>, VarianceComponents)> {
+    let mut found: Vec<(Option<String>, VarianceComponents)> = paths
+        .iter()
+        .filter_map(|path| variance_components_from_log(Path::new(path)))
+        .collect();
+    sort_by_chromosome(&mut found);
+    found
+}
+
+/// One GEMMA `.log.txt`, labelled by the chromosome in its own command line
+/// (`gemma ... -loco 7 ...`) with the file name as a fallback.
+fn variance_components_from_log(path: &Path) -> Option<(Option<String>, VarianceComponents)> {
+    let log = std::fs::read_to_string(path).ok()?;
+    let components = parse_variance_components(&log);
+    if components.pve.is_none() && components.vg.is_none() && components.ve.is_none() {
+        return None;
+    }
+    let chromosome = log
+        .lines()
+        .find_map(|line| {
+            let mut words = line.split_whitespace();
+            while let Some(word) = words.next() {
+                if word == "-loco" {
+                    return words.next().map(str::to_string);
+                }
+            }
+            None
+        })
+        .or_else(|| {
+            let name = path.file_name()?.to_string_lossy().into_owned();
+            let stem = name.strip_suffix(".log.txt").unwrap_or(&name);
+            // gemma-wrapper's `<hash>.<chr>.assoc.txt.log.txt`, or this
+            // workflow's own `<chr>-gwa.log.txt`.
+            if let Some(rest) = stem.strip_suffix(".assoc.txt") {
+                return rest.split('.').nth(1).map(str::to_string);
+            }
+            stem.split('-').next().map(str::to_string)
+        });
+    Some((chromosome, components))
+}
+
+/// A stable order, so the note reads chr1..chrX rather than in whatever order
+/// the filesystem listed the files.
+fn sort_by_chromosome(found: &mut [(Option<String>, VarianceComponents)]) {
     found.sort_by_key(|(chr, _)| {
         chr.as_deref()
             .map(|chr| (chr.parse::<u32>().unwrap_or(u32::MAX), chr.to_string()))
     });
-    found
 }
 
 /// One note carrying every chromosome's variance components.
@@ -991,15 +1019,15 @@ fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str>
 
 /// The distinct chromosome names in a SNP annotation file's contents
 /// (whitespace-separated: rs id, position, chromosome), in first-seen
-/// order -- reproduces gemma-wrapper's own `get_chromosomes`, used here
-/// to build `compute_ravanan`'s `chromosomes` scatter input (CWL's
+/// order, used here to build `compute_ravanan`'s `chromosomes` scatter
+/// input: GEMMA's own `-loco <chr>` takes a chromosome name, so the
+/// annotation file decides the scatter (CWL's
 /// `scatter` needs a concrete array at invocation time, which can't be
 /// discovered by a step inside the workflow itself without another
 /// round of plumbing -- see `cwl/gemma/gemma-loco.scm`'s own doc
 /// comment on this input).
 ///
-/// Takes the contents, not the path: the caller also hashes them (see
-/// [`gemma_wrapper_input_key`]) and should not read the file twice.
+/// Takes the contents, not the path, so the caller reads the file once.
 fn chromosomes_from(contents: &str) -> Vec<String> {
     let mut seen = std::collections::HashSet::new();
     let mut chromosomes = Vec::new();
@@ -1013,51 +1041,17 @@ fn chromosomes_from(contents: &str) -> Vec<String> {
     chromosomes
 }
 
-/// A stable identity for one request's GEMMA inputs, used to give each
-/// distinct input set its own gemma-wrapper cache root (see
-/// `compute_ravanan`).
-///
-/// The genotype file is identified by its path and size rather than its
-/// contents: hashing a genome-sized file on every request would cost
-/// more than the cache saves, and the path is stable in practice (it is
-/// the caller's own file, interned verbatim into ravanan's store).
-fn gemma_wrapper_input_key(
-    genotype_file: &str,
-    snps_contents: &str,
-    trait_values: &TraitValues,
-    lmm: u8,
-) -> String {
-    let mut material = Vec::new();
-    material.extend_from_slice(genotype_file.as_bytes());
-    material.push(0);
-    if let Ok(metadata) = std::fs::metadata(genotype_file) {
-        material.extend_from_slice(&metadata.len().to_le_bytes());
-    }
-    material.push(0);
-    material.extend_from_slice(snps_contents.as_bytes());
-    material.push(0);
-    material.extend_from_slice(lmm.to_string().as_bytes());
-    material.push(0);
-    for value in trait_values {
-        match value {
-            Some(value) => material.extend_from_slice(&value.to_bits().to_le_bytes()),
-            None => material.extend_from_slice(b"NA"),
-        }
-        material.push(b',');
-    }
-    crate::indexer::sha256::hex_digest(&material)[..16].to_string()
-}
-
 /// The same GEMMA `--loco` kinship-then-GWA computation as `compute`,
 /// but run via `ravanan` (https://github.com/vepiteski/ravanan) driving
 /// a CWL workflow compiled from the ccwl (https://ccwl.systemreboot.net/)
-/// source at `workflow_scm_path` (see `cwl/gemma/gemma-loco.scm`)
-/// instead of this module's own direct subprocess calls: one
-/// gemma-wrapper invocation per chromosome, scattered by ravanan
-/// itself (`--chromosomes`/`--no-parallel`, see
-/// `cwl/gemma/gemma-loco.scm`'s `kinship` and `gwa` commands) rather
-/// than gemma-wrapper's own internal GNU parallel fan-out -- an alternative
-/// runner for the same computation, so the two can be compared.
+/// source at `workflow_scm_path` (see `cwl/gemma/gemma-loco.scm`).
+///
+/// This runner invokes **gemma itself**, one invocation per
+/// chromosome, relying on `-loco <chr>` being native to 0.98.6 for both
+/// `-gk` and `-lmm` -- where `compute` still drives gemma-wrapper. That
+/// is deliberate: it is what makes the two endpoints comparable, the
+/// same computation through two stacks, and the day the numbers stop
+/// agreeing the difference is visible rather than assumed away.
 /// Backs `POST /compute/gemma/ravanan`.
 pub async fn compute_ravanan(
     ravanan_cmd: &str,
@@ -1121,39 +1115,6 @@ pub async fn compute_ravanan(
         .collect::<Vec<_>>()
         .join(", ");
 
-    // A persistent cache directory per (input set, chromosome), reused
-    // across requests, so gemma-wrapper's content-hash cache actually
-    // gets hit on repeated identical requests -- unlike "." (each job's
-    // own hermetic CWL working directory under ravanan's --store, fresh
-    // every run), which never accumulates a cache at all. Keyed by
-    // chromosome as well as the input set, because gemma-wrapper's
-    // cache-hit hash does not vary with --chromosomes: a directory
-    // shared between chromosomes would let one chromosome's job
-    // silently reuse another's cached result.
-    //
-    // Keyed by the input set too, because gemma-wrapper names its
-    // archives by a *content* hash: two different input sets (another
-    // trait, or another snps/genotype file) sharing a `chr-N` directory
-    // leave two `*-gemma-cXX.tar.xz` files there, and the CWL's
-    // single-File glob output binding (`$(inputs.cache_dir)/*-gemma-cXX.tar.xz`)
-    // then matches more than one path and the step fails -- with an
-    // error that ravanan's `user-error` silently drops in a job script.
-    let input_key = gemma_wrapper_input_key(genotype_file, &snps_contents, trait_values, lmm);
-    let wrapper_cache_dir = Path::new(cache_dir)
-        .join("gemma-wrapper-cache")
-        .join(&input_key);
-    let mut chr_cache_dirs = Vec::new();
-    for chr in &chromosomes {
-        let chr_cache_dir = wrapper_cache_dir.join(format!("chr-{chr}"));
-        std::fs::create_dir_all(&chr_cache_dir).map_err(GemmaError::Io)?;
-        chr_cache_dirs.push(chr_cache_dir);
-    }
-    let cache_dirs_yaml = chr_cache_dirs
-        .iter()
-        .map(|d| format!("\"{}\"", d.display()))
-        .collect::<Vec<_>>()
-        .join(", ");
-
     let inputs_yaml_path = Path::new(cache_dir).join(format!("{unique}-ravanan-inputs.yaml"));
     std::fs::write(
         &inputs_yaml_path,
@@ -1162,8 +1123,7 @@ pub async fn compute_ravanan(
              pheno_file: {{class: File, path: {}}}\n\
              snps_file: {{class: File, path: {snps_file}}}\n\
              lmm: \"{lmm}\"\n\
-             chromosomes: [{chromosomes_yaml}]\n\
-             cache_dirs: [{cache_dirs_yaml}]\n",
+             chromosomes: [{chromosomes_yaml}]\n",
             pheno_path.display(),
         ),
     )
@@ -1172,15 +1132,10 @@ pub async fn compute_ravanan(
     let store_arg = format!("--store={}", store_dir.display());
     let manifest_arg = format!("--guix-manifest={manifest_path}");
     let inputs_yaml_arg = inputs_yaml_path.to_string_lossy().into_owned();
-    // No --output-directory: the gwa step's per-chromosome archive
-    // outputs all share gemma-wrapper's own content-hash basename (its
-    // hash doesn't vary with --chromosomes), and ravanan's
-    // --output-directory step symlinks every scattered output into one
-    // flat directory by basename, which collides across chromosomes --
-    // a ravanan/gemma-wrapper interaction bug, not something fixable
-    // on either side alone. Harmless to skip: it's optional, and this
-    // code reads paths straight out of ravanan's own --store via its
-    // stdout output record, never through --output-directory.
+    // No --output-directory: this code reads the scattered outputs'
+    // paths straight out of ravanan's own --store via its stdout record,
+    // never through a flattened copy, and the workflow names each
+    // chromosome's files `<chr>-gwa.*` itself.
     // Backend switches: nothing for the default local runner, and
     // --batch-system=slurm-api plus endpoint/JWT/partition/scratch for a
     // cluster (see docs/ravanan-slurm.md). `scratch` is the cache dir
@@ -1208,12 +1163,9 @@ pub async fn compute_ravanan(
         args.join(" ")
     );
     let chromosomes_note = format!(
-        "Scattered gemma-wrapper over {} chromosome(s) via CWL scatter \
-         (--chromosomes/--no-parallel per job, so GNU parallel is never \
-         invoked), each against its own persistent, chromosome-keyed \
-         --cache-dir under {}: {}",
+        "Scattered gemma itself over {} chromosome(s) via CWL scatter, one \
+         invocation per chromosome: {}",
         chromosomes.len(),
-        wrapper_cache_dir.display(),
         chromosomes.join(", "),
     );
 
@@ -1230,64 +1182,28 @@ pub async fn compute_ravanan(
     }
     let outputs: Value = serde_json::from_str(&String::from_utf8_lossy(&output.stdout))
         .map_err(|e| GemmaError::RavananOutput(format!("could not parse ravanan's stdout: {e}")))?;
-    let gwa_json_paths = ravanan_output_paths(&outputs, "gwa_json")?;
-    let archive_paths = ravanan_output_paths(&outputs, "archive")?;
-    if gwa_json_paths.len() != archive_paths.len() {
-        return Err(GemmaError::RavananOutput(
-            "\"gwa_json\" and \"archive\" arrays have different lengths".to_string(),
-        ));
+    let assoc_paths = ravanan_output_paths(&outputs, "assoc")?;
+    let gwa_log_paths = ravanan_output_paths(&outputs, "gwa_log")?;
+    if assoc_paths.is_empty() {
+        return Err(GemmaError::NoAssocFiles);
     }
 
-    // One archive per chromosome now (gwa.cwl's scatter), not one
-    // shared archive as compute() and this function's own kinship step
-    // get -- each gets its own extract dir and its basenames' rows are
-    // concatenated across all of them.
+    // One .assoc.txt and one .log.txt per chromosome, straight from the
+    // workflow's own outputs: gemma writes named files (`-o <chr>-gwa`), so
+    // there is no wrapper JSON record to parse and no archive to unpack.
     let mut columns = Vec::new();
     let mut rows = Vec::new();
-    let mut components: Vec<(Option<String>, VarianceComponents)> = Vec::new();
-    for (i, (gwa_json_path, archive_path)) in gwa_json_paths.iter().zip(archive_paths.iter()).enumerate() {
-        let gwa_record: Value = serde_json::from_str(
-            &std::fs::read_to_string(gwa_json_path).map_err(GemmaError::AssocFile)?,
-        )
-        .map_err(|e| GemmaError::WrapperError {
-            step: "gwa",
-            detail: format!("could not parse gemma-wrapper's --json output: {e}"),
-        })?;
-        let basenames = assoc_file_basenames(&gwa_record);
-        if basenames.is_empty() {
-            return Err(GemmaError::NoAssocFiles);
-        }
-
-        let extract_dir = Path::new(cache_dir).join(format!("{unique}-ravanan-extract-{i}"));
-        std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?;
-        let tar_output = streaming::run_streaming(
-            "tar",
-            &["xJf", archive_path, "-C", &extract_dir.to_string_lossy()],
-            None,
-        )
-        .await
-        .map_err(GemmaError::Io)?;
-        if !tar_output.status.success() {
-            return Err(GemmaError::CommandFailed {
-                step: "extract-archive",
-                stderr: String::from_utf8_lossy(&tar_output.stderr).into_owned(),
-            });
-        }
-
-        let (file_columns, file_rows) = parse_assoc_files(&extract_dir, basenames)?;
+    for assoc_path in &assoc_paths {
+        let contents = std::fs::read_to_string(assoc_path).map_err(GemmaError::AssocFile)?;
+        let (file_columns, file_rows) = parse_assoc_file(&contents);
         if columns.is_empty() {
             columns = file_columns;
         }
-        rows.extend(file_rows);
-        // The archive holds this chromosome's GEMMA `.log.txt`, which is where
-        // `-pve` writes the variance components. Read them from there rather
-        // than from beside the archive: the copy beside it is a 5eb53fa
-        // feature, and a ravanan job's gemma-wrapper comes from the job's own
-        // Guix manifest, which need not be that new. (The extract dir is
-        // removed below, so this has to happen first.)
-        components.extend(read_variance_components_from_logs(&extract_dir));
-        let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless
+        rows.extend(file_rows.into_iter().map(Value::Array));
     }
+    // `-pve` puts the variance components in each job's .log.txt, which the
+    // workflow returns as a first-class output.
+    let components = read_variance_components_from_log_files(&gwa_log_paths);
 
     streaming::mark_complete(&log_path);