about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-11-14 13:57:41 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-11-14 13:57:41 +0000
commitdc40e89ce72b3213d227a1ee90df701aa17a1c7f (patch)
tree0665da1af8fdd3823f33c739c9572e9c8837e3f5 /src
parentf64a7084694fc28fc2069e39778a83716c1b858e (diff)
downloadgenenetwork4-dc40e89ce72b3213d227a1ee90df701aa17a1c7f.tar.gz
ravanan: key the kinship step on the animal set, so K is shared across traits
gemma -gk reads the phenotype file only to learn which individuals are
analysed -- the matrix comes from the genotypes -- so the workflow now
gives the kinship step a constant-valued pheno file carrying just the
trait's missing-data pattern, and gives gwa the real values.  Verified
byte-for-byte: -gk on a real trait and on its constant-valued twin
produce identical .cXX.txt files, and trait 2's assoc checksums are
identical whether K came from the constant pheno or the real one.

Consequence, measured on the BXD fixture: a second trait over the same
animals costs +1.0 MB of store and 7.5 s (20 GWA jobs) instead of
+13 MB and 14.6 s (a second kinship scatter), because the 20 K matrices
are identical inputs for both traits.  Three keys now: workflow (the
ccwl source), kinship (genotype + snps + lmm + the NA pattern) and
trait (kinship key + the values), so the compiled CWL and the kinship
pheno file are shared and only the trait's pheno file and inputs YAML
are per-trait.

This is also stricter than gemma-wrapper's own key, which ignores the
phenotype entirely and would reuse a K computed for a different set of
animals.
Diffstat (limited to 'src')
-rw-r--r--src/gemma.rs92
1 files changed, 83 insertions, 9 deletions
diff --git a/src/gemma.rs b/src/gemma.rs
index bfe0065..6eb7b61 100644
--- a/src/gemma.rs
+++ b/src/gemma.rs
@@ -130,6 +130,21 @@ pub fn write_pheno_file(path: &Path, values: &TraitValues) -> io::Result<()> {
     std::fs::write(path, contents)
 }
 
+/// The same file with the trait's *pattern* but constant values: what the
+/// kinship step is given, so that K is a function of the animal set alone.
+///
+/// GEMMA reads `-p` in a `-gk` run only to know which individuals are
+/// analysed; the matrix is computed from the genotypes.  Verified by running
+/// `-gk` on a real trait and on its constant-valued twin: the `.cXX.txt` files
+/// are byte-identical.
+pub fn write_constant_pheno_file(path: &Path, values: &TraitValues) -> io::Result<()> {
+    let constant: TraitValues = values
+        .iter()
+        .map(|value| value.map(|_| 1.0))
+        .collect();
+    write_pheno_file(path, &constant)
+}
+
 /// The one line worth quoting from a failed gemma-wrapper run.
 ///
 /// The wrapper prints GEMMA's own diagnostics in the middle of its
@@ -1040,7 +1055,32 @@ fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str>
 /// file, interned verbatim into ravanan's store).  The SNP file is hashed,
 /// because the chromosome list -- and therefore the whole scatter -- comes
 /// from it.
-fn ravanan_input_key(
+fn sha16(material: &[u8]) -> String {
+    crate::indexer::sha256::hex_digest(material)[..16].to_string()
+}
+
+/// A key for the compiled workflow: the ccwl source is the only input, so the
+/// same source compiles to the same file and ravanan's (script, inputs) pair
+/// stays stable across calls.
+fn workflow_key(workflow_scm_path: &str) -> String {
+    match std::fs::read(workflow_scm_path) {
+        Ok(contents) => sha16(&contents),
+        Err(_) => sha16(workflow_scm_path.as_bytes()),
+    }
+}
+
+/// A key for the *kinship* step's inputs: genotype, SNP annotation, `lmm`, and
+/// the pattern of which individuals have a value -- deliberately not the
+/// values themselves.
+///
+/// GEMMA's `-gk` reads the phenotype file only to learn which individuals are
+/// analysed; the matrix itself comes from the genotypes.  Keying K on the
+/// animal set is what lets two traits over the same animals share all twenty
+/// K matrices, instead of storing a fresh 11 MB scatter per trait.  It is also
+/// stricter than gemma-wrapper's own key, which ignores the phenotype
+/// altogether and would therefore reuse a K computed for a different set of
+/// animals.
+fn kinship_input_key(
     genotype_file: &str,
     snps_file: &str,
     trait_values: &TraitValues,
@@ -1060,13 +1100,30 @@ fn ravanan_input_key(
     material.extend_from_slice(lmm.to_string().as_bytes());
     material.push(0);
     for value in trait_values {
+        material.extend_from_slice(match value {
+            Some(_) => b"1",
+            None => b"NA",
+        });
+        material.push(b',');
+    }
+    sha16(&material)
+}
+
+/// A key for one trait's own inputs: the kinship key plus the values, so a
+/// different trait gets its own phenotype file while sharing the kinship
+/// step's.
+fn trait_input_key(kinship_key: &str, trait_values: &TraitValues) -> String {
+    let mut material = Vec::new();
+    material.extend_from_slice(kinship_key.as_bytes());
+    material.push(0);
+    for value in trait_values {
         match value {
             Some(value) => material.extend_from_slice(&value.to_bits().to_le_bytes()),
             None => material.extend_from_slice(b"NA"),
         }
         material.push(b',');
     }
-    crate::indexer::sha256::hex_digest(&material)[..16].to_string()
+    sha16(&material)
 }
 
 fn chromosomes_from(contents: &str) -> Vec<String> {
@@ -1115,11 +1172,26 @@ pub async fn compute_ravanan(
     // is unchanged, and that pair is built from these paths.  Unique names made
     // every request cold, which cost the whole computation again for a repeat
     // of the same request -- 14.7 s versus 1.6 s on the BXD fixture.
-    let input_key = ravanan_input_key(genotype_file, snps_file, trait_values, lmm);
-    let pheno_path = Path::new(cache_dir).join(format!("{input_key}-pheno.txt"));
+    //
+    // Two keys, not one: the kinship step is keyed on the *individual set*
+    // only (see `kinship_input_key`), so a second trait over the same animals
+    // reuses all twenty K matrices, and only the GWA step's inputs change.
+    let kinship_key = kinship_input_key(genotype_file, snps_file, trait_values, lmm);
+    let trait_key = trait_input_key(&kinship_key, trait_values);
+    let workflow = workflow_key(workflow_scm_path);
+
+    // A constant-valued phenotype file with the trait's missing-data pattern:
+    // what the kinship step gets, so its files are the same for every trait
+    // over the same animals.  K depends on the genotypes and the individual
+    // set, never on the values -- verified byte-for-byte against a real trait.
+    let kinship_pheno_path =
+        Path::new(cache_dir).join(format!("{kinship_key}-kin-pheno.txt"));
+    write_constant_pheno_file(&kinship_pheno_path, trait_values).map_err(GemmaError::Io)?;
+
+    let pheno_path = Path::new(cache_dir).join(format!("{trait_key}-pheno.txt"));
     // Written aside and renamed, so a concurrent request with the same key
     // never reads a half-written file.
-    let pheno_tmp = Path::new(cache_dir).join(format!("{input_key}-pheno.tmp-{}", std::process::id()));
+    let pheno_tmp = Path::new(cache_dir).join(format!("{trait_key}-pheno.tmp-{}", std::process::id()));
     write_pheno_file(&pheno_tmp, trait_values).map_err(GemmaError::Io)?;
     std::fs::rename(&pheno_tmp, &pheno_path).map_err(GemmaError::Io)?;
 
@@ -1133,12 +1205,12 @@ pub async fn compute_ravanan(
     // involved) and never checked into the repo -- see the doc comment
     // on Config::gemma_ravanan_workflow.
     let compiled_workflow_path =
-        Path::new(cache_dir).join(format!("{input_key}-gemma-loco.cwl"));
+        Path::new(cache_dir).join(format!("{workflow}-gemma-loco.cwl"));
     // Compile to a per-process path and rename: the compiled CWL is
     // deterministic, so two concurrent identical requests would otherwise
     // race on the same file.
     let compiled_tmp =
-        Path::new(cache_dir).join(format!("{input_key}-gemma-loco.tmp-{}.cwl", std::process::id()));
+        Path::new(cache_dir).join(format!("{workflow}-gemma-loco.tmp-{}.cwl", std::process::id()));
     let ccwl_output_arg = format!("--output={}", compiled_tmp.display());
     let ccwl_args = vec!["compile", ccwl_output_arg.as_str(), workflow_scm_path];
     let ccwl_output = streaming::run_streaming(ccwl_cmd, &ccwl_args, None)
@@ -1174,18 +1246,20 @@ pub async fn compute_ravanan(
         .collect::<Vec<_>>()
         .join(", ");
 
-    let inputs_yaml_path = Path::new(cache_dir).join(format!("{input_key}-ravanan-inputs.yaml"));
+    let inputs_yaml_path = Path::new(cache_dir).join(format!("{trait_key}-ravanan-inputs.yaml"));
     let inputs_tmp =
-        Path::new(cache_dir).join(format!("{input_key}-ravanan-inputs.tmp-{}.yaml", std::process::id()));
+        Path::new(cache_dir).join(format!("{trait_key}-ravanan-inputs.tmp-{}.yaml", std::process::id()));
     std::fs::write(
         &inputs_tmp,
         format!(
             "genotype_file: {{class: File, path: {genotype_file}}}\n\
              pheno_file: {{class: File, path: {}}}\n\
+             kinship_pheno_file: {{class: File, path: {}}}\n\
              snps_file: {{class: File, path: {snps_file}}}\n\
              lmm: \"{lmm}\"\n\
              chromosomes: [{chromosomes_yaml}]\n",
             pheno_path.display(),
+            kinship_pheno_path.display(),
         ),
     )
     .map_err(GemmaError::Io)?;