diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-14 13:57:41 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-14 13:57:41 +0000 |
| commit | dc40e89ce72b3213d227a1ee90df701aa17a1c7f (patch) | |
| tree | 0665da1af8fdd3823f33c739c9572e9c8837e3f5 /src | |
| parent | f64a7084694fc28fc2069e39778a83716c1b858e (diff) | |
| download | genenetwork4-dc40e89ce72b3213d227a1ee90df701aa17a1c7f.tar.gz | |
ravanan: key the kinship step on the animal set, so K is shared across traits
gemma -gk reads the phenotype file only to learn which individuals are analysed -- the matrix comes from the genotypes -- so the workflow now gives the kinship step a constant-valued pheno file carrying just the trait's missing-data pattern, and gives gwa the real values. Verified byte-for-byte: -gk on a real trait and on its constant-valued twin produce identical .cXX.txt files, and trait 2's assoc checksums are identical whether K came from the constant pheno or the real one. Consequence, measured on the BXD fixture: a second trait over the same animals costs +1.0 MB of store and 7.5 s (20 GWA jobs) instead of +13 MB and 14.6 s (a second kinship scatter), because the 20 K matrices are identical inputs for both traits. Three keys now: workflow (the ccwl source), kinship (genotype + snps + lmm + the NA pattern) and trait (kinship key + the values), so the compiled CWL and the kinship pheno file are shared and only the trait's pheno file and inputs YAML are per-trait. This is also stricter than gemma-wrapper's own key, which ignores the phenotype entirely and would reuse a K computed for a different set of animals.
Diffstat (limited to 'src')
| -rw-r--r-- | src/gemma.rs | 92 |
1 files changed, 83 insertions, 9 deletions
diff --git a/src/gemma.rs b/src/gemma.rs index bfe0065..6eb7b61 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -130,6 +130,21 @@ pub fn write_pheno_file(path: &Path, values: &TraitValues) -> io::Result<()> { std::fs::write(path, contents) } +/// The same file with the trait's *pattern* but constant values: what the +/// kinship step is given, so that K is a function of the animal set alone. +/// +/// GEMMA reads `-p` in a `-gk` run only to know which individuals are +/// analysed; the matrix is computed from the genotypes. Verified by running +/// `-gk` on a real trait and on its constant-valued twin: the `.cXX.txt` files +/// are byte-identical. +pub fn write_constant_pheno_file(path: &Path, values: &TraitValues) -> io::Result<()> { + let constant: TraitValues = values + .iter() + .map(|value| value.map(|_| 1.0)) + .collect(); + write_pheno_file(path, &constant) +} + /// The one line worth quoting from a failed gemma-wrapper run. /// /// The wrapper prints GEMMA's own diagnostics in the middle of its @@ -1040,7 +1055,32 @@ fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str> /// file, interned verbatim into ravanan's store). The SNP file is hashed, /// because the chromosome list -- and therefore the whole scatter -- comes /// from it. -fn ravanan_input_key( +fn sha16(material: &[u8]) -> String { + crate::indexer::sha256::hex_digest(material)[..16].to_string() +} + +/// A key for the compiled workflow: the ccwl source is the only input, so the +/// same source compiles to the same file and ravanan's (script, inputs) pair +/// stays stable across calls. +fn workflow_key(workflow_scm_path: &str) -> String { + match std::fs::read(workflow_scm_path) { + Ok(contents) => sha16(&contents), + Err(_) => sha16(workflow_scm_path.as_bytes()), + } +} + +/// A key for the *kinship* step's inputs: genotype, SNP annotation, `lmm`, and +/// the pattern of which individuals have a value -- deliberately not the +/// values themselves. +/// +/// GEMMA's `-gk` reads the phenotype file only to learn which individuals are +/// analysed; the matrix itself comes from the genotypes. Keying K on the +/// animal set is what lets two traits over the same animals share all twenty +/// K matrices, instead of storing a fresh 11 MB scatter per trait. It is also +/// stricter than gemma-wrapper's own key, which ignores the phenotype +/// altogether and would therefore reuse a K computed for a different set of +/// animals. +fn kinship_input_key( genotype_file: &str, snps_file: &str, trait_values: &TraitValues, @@ -1060,13 +1100,30 @@ fn ravanan_input_key( material.extend_from_slice(lmm.to_string().as_bytes()); material.push(0); for value in trait_values { + material.extend_from_slice(match value { + Some(_) => b"1", + None => b"NA", + }); + material.push(b','); + } + sha16(&material) +} + +/// A key for one trait's own inputs: the kinship key plus the values, so a +/// different trait gets its own phenotype file while sharing the kinship +/// step's. +fn trait_input_key(kinship_key: &str, trait_values: &TraitValues) -> String { + let mut material = Vec::new(); + material.extend_from_slice(kinship_key.as_bytes()); + material.push(0); + for value in trait_values { match value { Some(value) => material.extend_from_slice(&value.to_bits().to_le_bytes()), None => material.extend_from_slice(b"NA"), } material.push(b','); } - crate::indexer::sha256::hex_digest(&material)[..16].to_string() + sha16(&material) } fn chromosomes_from(contents: &str) -> Vec<String> { @@ -1115,11 +1172,26 @@ pub async fn compute_ravanan( // is unchanged, and that pair is built from these paths. Unique names made // every request cold, which cost the whole computation again for a repeat // of the same request -- 14.7 s versus 1.6 s on the BXD fixture. - let input_key = ravanan_input_key(genotype_file, snps_file, trait_values, lmm); - let pheno_path = Path::new(cache_dir).join(format!("{input_key}-pheno.txt")); + // + // Two keys, not one: the kinship step is keyed on the *individual set* + // only (see `kinship_input_key`), so a second trait over the same animals + // reuses all twenty K matrices, and only the GWA step's inputs change. + let kinship_key = kinship_input_key(genotype_file, snps_file, trait_values, lmm); + let trait_key = trait_input_key(&kinship_key, trait_values); + let workflow = workflow_key(workflow_scm_path); + + // A constant-valued phenotype file with the trait's missing-data pattern: + // what the kinship step gets, so its files are the same for every trait + // over the same animals. K depends on the genotypes and the individual + // set, never on the values -- verified byte-for-byte against a real trait. + let kinship_pheno_path = + Path::new(cache_dir).join(format!("{kinship_key}-kin-pheno.txt")); + write_constant_pheno_file(&kinship_pheno_path, trait_values).map_err(GemmaError::Io)?; + + let pheno_path = Path::new(cache_dir).join(format!("{trait_key}-pheno.txt")); // Written aside and renamed, so a concurrent request with the same key // never reads a half-written file. - let pheno_tmp = Path::new(cache_dir).join(format!("{input_key}-pheno.tmp-{}", std::process::id())); + let pheno_tmp = Path::new(cache_dir).join(format!("{trait_key}-pheno.tmp-{}", std::process::id())); write_pheno_file(&pheno_tmp, trait_values).map_err(GemmaError::Io)?; std::fs::rename(&pheno_tmp, &pheno_path).map_err(GemmaError::Io)?; @@ -1133,12 +1205,12 @@ pub async fn compute_ravanan( // involved) and never checked into the repo -- see the doc comment // on Config::gemma_ravanan_workflow. let compiled_workflow_path = - Path::new(cache_dir).join(format!("{input_key}-gemma-loco.cwl")); + Path::new(cache_dir).join(format!("{workflow}-gemma-loco.cwl")); // Compile to a per-process path and rename: the compiled CWL is // deterministic, so two concurrent identical requests would otherwise // race on the same file. let compiled_tmp = - Path::new(cache_dir).join(format!("{input_key}-gemma-loco.tmp-{}.cwl", std::process::id())); + Path::new(cache_dir).join(format!("{workflow}-gemma-loco.tmp-{}.cwl", std::process::id())); let ccwl_output_arg = format!("--output={}", compiled_tmp.display()); let ccwl_args = vec!["compile", ccwl_output_arg.as_str(), workflow_scm_path]; let ccwl_output = streaming::run_streaming(ccwl_cmd, &ccwl_args, None) @@ -1174,18 +1246,20 @@ pub async fn compute_ravanan( .collect::<Vec<_>>() .join(", "); - let inputs_yaml_path = Path::new(cache_dir).join(format!("{input_key}-ravanan-inputs.yaml")); + let inputs_yaml_path = Path::new(cache_dir).join(format!("{trait_key}-ravanan-inputs.yaml")); let inputs_tmp = - Path::new(cache_dir).join(format!("{input_key}-ravanan-inputs.tmp-{}.yaml", std::process::id())); + Path::new(cache_dir).join(format!("{trait_key}-ravanan-inputs.tmp-{}.yaml", std::process::id())); std::fs::write( &inputs_tmp, format!( "genotype_file: {{class: File, path: {genotype_file}}}\n\ pheno_file: {{class: File, path: {}}}\n\ + kinship_pheno_file: {{class: File, path: {}}}\n\ snps_file: {{class: File, path: {snps_file}}}\n\ lmm: \"{lmm}\"\n\ chromosomes: [{chromosomes_yaml}]\n", pheno_path.display(), + kinship_pheno_path.display(), ), ) .map_err(GemmaError::Io)?; |
