diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-14 15:23:42 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-14 15:23:42 +0000 |
| commit | 0d016ee2211fe08722377c4089ce24fa4c98ccdf (patch) | |
| tree | de670057ada6534074f843d560b63e273e8fcc9a /src | |
| parent | 6daec6b5d03a80af3e2496b6bf666340b3d718fd (diff) | |
| download | genenetwork4-0d016ee2211fe08722377c4089ce24fa4c98ccdf.tar.gz | |
gemma: refuse a workflow from another commit, by name
cwl/gemma/gemma-loco.scm is read from the deployment's checkout at request time (GEMMA_RAVANAN_WORKFLOW), not compiled into the binary, so a checkout and a binary from different commits can meet. When they do, cwltool answers "Type mismatch for input `kinship_pheno_file'; expected `File' but got `null'" -- which says nothing about the cause. Found by pointing the deployed f64a708 binary at the current workflow. The compiled CWL now has its declared inputs compared with what this build is about to send, before anything is staged for ravanan: a missing input or an undeclared one is refused with both lists named, plus the instruction to take the .scm and the binary from the same commit.
Diffstat (limited to 'src')
| -rw-r--r-- | src/gemma.rs | 166 |
1 files changed, 166 insertions, 0 deletions
diff --git a/src/gemma.rs b/src/gemma.rs index 6eb7b61..e221c3b 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -56,6 +56,14 @@ pub enum GemmaError { /// `ravanan`'s own stdout (the CWL outputs record) was not valid /// JSON, or didn't carry the output this code expects. RavananOutput(String), + /// The compiled workflow and this build disagree about what the + /// workflow takes: the `gemma-loco.scm` in the deployment's checkout + /// is not the one this binary was written against. Caught before + /// ravanan runs, because cwltool's own message for this ("Type + /// mismatch for input X; expected File but got null") says nothing + /// about why -- and the why is "upgrade the checkout and the binary + /// together". + WorkflowMismatch(String), } impl std::fmt::Display for GemmaError { @@ -85,6 +93,13 @@ impl std::fmt::Display for GemmaError { GemmaError::RavananOutput(detail) => { write!(f, "could not read ravanan's output: {detail}") } + GemmaError::WorkflowMismatch(detail) => write!( + f, + "the workflow source and this build disagree about the workflow's inputs: {detail}. \ + cwl/gemma/gemma-loco.scm comes from the deployment's checkout, not from the \ + binary, so put both from the same commit (or set GEMMA_RAVANAN_WORKFLOW to the \ + matching file)." + ), } } } @@ -1032,6 +1047,79 @@ fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str> .collect() } +/// The input names a compiled CWL workflow declares, and which of them have +/// no default (so they must be supplied). +/// +/// ccwl compiles to JSON, so this is a plain parse: `{"inputs": {"name": +/// {...}}}`. Used to catch a binary and a workflow source from different +/// commits before ravanan runs -- cwltool's own complaint about a missing +/// input ("Type mismatch for input `kinship_pheno_file'; expected `File' but +/// got `null'") does not say that the cause is a stale `.scm` in the +/// deployment's checkout. +fn workflow_inputs(compiled_path: &Path) -> Result<(Vec<String>, Vec<String>), GemmaError> { + let text = std::fs::read_to_string(compiled_path).map_err(GemmaError::Io)?; + let document: Value = serde_json::from_str(&text).map_err(|error| { + GemmaError::WorkflowMismatch(format!( + "the compiled workflow is not JSON ({error}); is GEMMA_RAVANAN_WORKFLOW a ccwl source?" + )) + })?; + let inputs = document + .get("inputs") + .and_then(Value::as_object) + .ok_or_else(|| { + GemmaError::WorkflowMismatch("the compiled workflow declares no inputs".to_string()) + })?; + let mut declared = Vec::new(); + let mut required = Vec::new(); + for (name, spec) in inputs { + declared.push(name.clone()); + let has_default = spec.get("default").is_some(); + if !has_default { + required.push(name.clone()); + } + } + declared.sort(); + required.sort(); + Ok((declared, required)) +} + +/// Refuse to invoke ravanan when the workflow and this build disagree, naming +/// the mismatch rather than letting cwltool report a `null` input. +fn check_workflow_inputs(compiled_path: &Path, supplied: &[&str]) -> Result<(), GemmaError> { + let (declared, required) = workflow_inputs(compiled_path)?; + let missing: Vec<&str> = required + .iter() + .filter(|name| !supplied.contains(&name.as_str())) + .map(String::as_str) + .collect(); + let unknown: Vec<&str> = supplied + .iter() + .copied() + .filter(|name| !declared.iter().any(|declared| declared == name)) + .collect(); + if missing.is_empty() && unknown.is_empty() { + return Ok(()); + } + let mut detail = String::new(); + if !missing.is_empty() { + detail.push_str(&format!( + "the workflow needs {} and this build does not supply it; ", + missing.iter().map(|n| format!("`{n}'")).collect::<Vec<_>>().join(", ") + )); + } + if !unknown.is_empty() { + detail.push_str(&format!( + "this build supplies {} which the workflow does not declare; ", + unknown.iter().map(|n| format!("`{n}'")).collect::<Vec<_>>().join(", ") + )); + } + detail.push_str(&format!( + "the workflow declares [{}]", + declared.join(", ") + )); + Err(GemmaError::WorkflowMismatch(detail)) +} + /// The distinct chromosome names in a SNP annotation file's contents /// (whitespace-separated: rs id, position, chromosome), in first-seen /// order, used here to build `compute_ravanan`'s `chromosomes` scatter @@ -1226,6 +1314,19 @@ pub async fn compute_ravanan( }); } std::fs::rename(&compiled_tmp, &compiled_workflow_path).map_err(GemmaError::Io)?; + // Before anything is staged for ravanan: does the workflow in this + // deployment's checkout take what this build is about to send it? + check_workflow_inputs( + &compiled_workflow_path, + &[ + "genotype_file", + "pheno_file", + "kinship_pheno_file", + "snps_file", + "lmm", + "chromosomes", + ], + )?; let compiled_workflow_arg = compiled_workflow_path.to_string_lossy().into_owned(); let store_dir = Path::new(cache_dir).join("ravanan-store"); @@ -1364,6 +1465,71 @@ pub async fn compute_ravanan( mod tests { use super::*; + /// A stale `gemma-loco.scm` in the deployment's checkout is a deployment + /// mistake with an inscrutable symptom (cwltool: "expected File but got + /// null"); the check names it instead. + #[test] + fn a_workflow_from_another_commit_is_refused_by_name() { + let path = std::env::temp_dir().join(format!("gn4-wf-{}.json", std::process::id())); + std::fs::write( + &path, + r#"{"class":"Workflow","inputs":{ + "genotype_file":{"type":"File"}, + "pheno_file":{"type":"File"}, + "snps_file":{"type":"File"}, + "lmm":{"type":"string"}, + "chromosomes":{"type":"string[]"}}}"#, + ) + .unwrap(); + // This build sends kinship_pheno_file, which the older workflow + // above does not declare. + let error = check_workflow_inputs( + &path, + &[ + "genotype_file", + "pheno_file", + "kinship_pheno_file", + "snps_file", + "lmm", + "chromosomes", + ], + ) + .expect_err("must refuse"); + let message = error.to_string(); + assert!( + message.contains("kinship_pheno_file") && message.contains("does not declare"), + "{message}" + ); + assert!(message.contains("same commit"), "says what to do: {message}"); + assert!(matches!(error, GemmaError::WorkflowMismatch(_))); + + // The matching workflow passes. + std::fs::write( + &path, + r#"{"class":"Workflow","inputs":{ + "genotype_file":{"type":"File"}, + "pheno_file":{"type":"File"}, + "kinship_pheno_file":{"type":"File"}, + "snps_file":{"type":"File"}, + "lmm":{"type":"string"}, + "chromosomes":{"type":"string[]"}}}"#, + ) + .unwrap(); + check_workflow_inputs( + &path, + &[ + "genotype_file", + "pheno_file", + "kinship_pheno_file", + "snps_file", + "lmm", + "chromosomes", + ], + ) + .expect("matching workflow"); + let _ = std::fs::remove_file(&path); + } + #[test] fn a_missing_binary_names_itself_and_a_file_error_does_not_blame_gemma() { let spawn = GemmaError::Spawn { |
