diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-19 07:13:15 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-19 07:13:15 +0000 |
| commit | d504b5da9f5deeece9ba2ff4dcd8baf3dfa4dc32 (patch) | |
| tree | eaed193876da61ad7d6ae55906dccf4c6198c583 /src | |
| parent | 477294a2aa9ae7348137271b66831ac93a6149d6 (diff) | |
| download | genenetwork4-d504b5da9f5deeece9ba2ff4dcd8baf3dfa4dc32.tar.gz | |
gemma: compile ravanan's ccwl workflow on demand, don't check in the CWL gemma-cwl-parallel
gemma-loco.cwl was a generated artifact (ccwl compile's output), but was being committed like hand-written source. Remove it, and have compute_ravanan compile cwl/gemma/gemma-loco.scm to a throwaway CWL file under its own cache dir on every call instead -- cheap (pure macro expansion, no Guix build involved), so there's no real cost to regenerating it fresh each time, and no risk of a stale compiled file drifting from its ccwl source. Config gains a ccwl_cmd knob (default "ccwl", resolved via PATH, alongside ravanan_cmd/gemma_wrapper_cmd); gemma_ravanan_workflow now names the .scm source, not a .cwl file. Verified end to end by running the exact compile-then-run sequence compute_ravanan now performs against the real ccwl/ravanan/gemma-wrapper binaries and the BXD fixtures: same correctness and timings as before.
Diffstat (limited to 'src')
| -rw-r--r-- | src/config.rs | 24 | ||||
| -rw-r--r-- | src/gemma.rs | 56 | ||||
| -rw-r--r-- | src/routes/gemma.rs | 5 |
3 files changed, 62 insertions, 23 deletions
diff --git a/src/config.rs b/src/config.rs index fa8b610..e64f2ad 100644 --- a/src/config.rs +++ b/src/config.rs @@ -72,10 +72,17 @@ pub struct Config { /// (see src/gemma.rs's compute_ravanan). Defaults to "ravanan", /// resolved via PATH. pub ravanan_cmd: String, - /// Path to the CWL workflow POST /compute/gemma/ravanan runs under - /// ravanan. Defaults to that path relative to the working - /// directory the server is started from, same convention as - /// `rqtl_wrapper_path`. + /// The ccwl (https://ccwl.systemreboot.net/) executable used to + /// compile `gemma_ravanan_workflow` to CWL before each ravanan run + /// (see src/gemma.rs's compute_ravanan). Defaults to "ccwl", + /// resolved via PATH. + pub ccwl_cmd: String, + /// Path to the ccwl source POST /compute/gemma/ravanan compiles to + /// CWL and runs under ravanan. Not a CWL file itself -- the + /// compiled .cwl is a generated, throwaway artifact (written under + /// `ravanan_cache_dir` on every call), never checked into the repo. + /// Defaults to that path relative to the working directory the + /// server is started from, same convention as `rqtl_wrapper_path`. pub gemma_ravanan_workflow: String, /// Path to the Guix manifest ravanan resolves its jobs' commands /// against (its own --guix-manifest switch, required -- ravanan @@ -149,9 +156,10 @@ impl Config { .into_owned(), ); values.insert("RAVANAN_CMD".to_string(), "ravanan".to_string()); + values.insert("CCWL_CMD".to_string(), "ccwl".to_string()); values.insert( "GEMMA_RAVANAN_WORKFLOW".to_string(), - "cwl/gemma/gemma-loco.cwl".to_string(), + "cwl/gemma/gemma-loco.scm".to_string(), ); values.insert( "GEMMA_RAVANAN_MANIFEST".to_string(), @@ -203,6 +211,7 @@ impl Config { "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", "RAVANAN_CMD", + "CCWL_CMD", "GEMMA_RAVANAN_WORKFLOW", "GEMMA_RAVANAN_MANIFEST", "RAVANAN_CACHE_DIR", @@ -237,6 +246,7 @@ impl Config { gemma_wrapper_cmd: values["GEMMA_WRAPPER_CMD"].clone(), gemma_cache_dir: values["GEMMA_CACHE_DIR"].clone(), ravanan_cmd: values["RAVANAN_CMD"].clone(), + ccwl_cmd: values["CCWL_CMD"].clone(), gemma_ravanan_workflow: values["GEMMA_RAVANAN_WORKFLOW"].clone(), gemma_ravanan_manifest: values["GEMMA_RAVANAN_MANIFEST"].clone(), ravanan_cache_dir: values["RAVANAN_CACHE_DIR"].clone(), @@ -315,6 +325,7 @@ mod tests { "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", "RAVANAN_CMD", + "CCWL_CMD", "GEMMA_RAVANAN_WORKFLOW", "GEMMA_RAVANAN_MANIFEST", "RAVANAN_CACHE_DIR", @@ -332,7 +343,8 @@ mod tests { assert_eq!(config.gemma_wrapper_cmd, "gemma-wrapper"); assert!(config.gemma_cache_dir.ends_with("gn4-gemma")); assert_eq!(config.ravanan_cmd, "ravanan"); - assert_eq!(config.gemma_ravanan_workflow, "cwl/gemma/gemma-loco.cwl"); + assert_eq!(config.ccwl_cmd, "ccwl"); + assert_eq!(config.gemma_ravanan_workflow, "cwl/gemma/gemma-loco.scm"); assert_eq!(config.gemma_ravanan_manifest, "cwl/gemma/manifest.scm"); assert!(config.ravanan_cache_dir.ends_with("gn4-gemma-ravanan")); assert_eq!(config.rqtl_wrapper_path, "scripts/rqtl_wrapper.R"); diff --git a/src/gemma.rs b/src/gemma.rs index 860ef9e..51e3fce 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -462,7 +462,7 @@ pub async fn compute( /// Read a scattered `File[]` output's list of `path`s from ravanan's /// stdout (the CWL outputs record for a scattered output: /// `{"<output id>": [{"class": "File", "path": ..., ...}, ...], ...}`, -/// one entry per `cwl/gemma/gemma-loco.cwl` chromosome). +/// one entry per `cwl/gemma/gemma-loco.scm` chromosome). fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str>, GemmaError> { let files = outputs .get(id) @@ -484,7 +484,7 @@ fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str> /// `compute_ravanan`'s `chromosomes` scatter input (CWL's `scatter` /// needs a concrete array at invocation time, which can't be /// discovered by a step inside the workflow itself without another -/// round of plumbing -- see `cwl/gemma/gemma-loco.cwl`'s own doc +/// round of plumbing -- see `cwl/gemma/gemma-loco.scm`'s own doc /// comment on this input). fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> { let contents = std::fs::read_to_string(snps_file).map_err(GemmaError::Io)?; @@ -502,7 +502,8 @@ fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> { /// The same GEMMA `--loco` kinship-then-GWA computation as `compute`, /// but run via `ravanan` (https://github.com/vepiteski/ravanan) driving -/// the CWL workflow at `workflow_path` (see `cwl/gemma/gemma-loco.cwl`) +/// a CWL workflow compiled from the ccwl (https://ccwl.systemreboot.net/) +/// source at `workflow_scm_path` (see `cwl/gemma/gemma-loco.scm`) /// instead of this module's own direct subprocess calls: one /// gemma-wrapper invocation per chromosome, scattered by ravanan /// itself (`--chromosomes`/`--no-parallel`, see @@ -512,7 +513,8 @@ fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> { /// Backs `POST /compute/gemma/ravanan`. pub async fn compute_ravanan( ravanan_cmd: &str, - workflow_path: &str, + ccwl_cmd: &str, + workflow_scm_path: &str, manifest_path: &str, cache_dir: &str, genotype_file: &str, @@ -528,6 +530,26 @@ pub async fn compute_ravanan( let log_path = streaming::create_run_log(cache_dir, run_id)?; + // gemma-loco.scm is the maintained source; the CWL ravanan actually + // runs is a generated, throwaway artifact compiled fresh on every + // call (cheap: ccwl compile is pure macro expansion, no Guix build + // involved) and never checked into the repo -- see the doc comment + // on Config::gemma_ravanan_workflow. + let compiled_workflow_path = + Path::new(cache_dir).join(format!("{unique}-gemma-loco.cwl")); + let ccwl_output_arg = format!("--output={}", compiled_workflow_path.display()); + let ccwl_args = vec!["compile", ccwl_output_arg.as_str(), workflow_scm_path]; + let ccwl_output = streaming::run_streaming(ccwl_cmd, &ccwl_args, None, &[]) + .await + .map_err(GemmaError::Io)?; + if !ccwl_output.status.success() { + return Err(GemmaError::CommandFailed { + step: "ccwl compile", + stderr: String::from_utf8_lossy(&ccwl_output.stderr).into_owned(), + }); + } + let compiled_workflow_arg = compiled_workflow_path.to_string_lossy().into_owned(); + let store_dir = Path::new(cache_dir).join("ravanan-store"); // ravanan's --store needs these three subdirectories to already // exist; it does not create them itself. @@ -584,19 +606,19 @@ pub async fn compute_ravanan( let store_arg = format!("--store={}", store_dir.display()); let manifest_arg = format!("--guix-manifest={manifest_path}"); let inputs_yaml_arg = inputs_yaml_path.to_string_lossy().into_owned(); - // No --output-directory: gwa.cwl's per-chromosome archive outputs - // all share gemma-wrapper's own content-hash basename (its hash - // doesn't vary with --chromosomes), and ravanan's --output-directory - // step symlinks every scattered output into one flat directory by - // basename, which collides across chromosomes -- a ravanan/ - // gemma-wrapper interaction bug, not something fixable on either - // side alone. Harmless to skip: it's optional, and this code reads - // paths straight out of ravanan's own --store via its stdout - // output record, never through --output-directory. + // No --output-directory: the gwa step's per-chromosome archive + // outputs all share gemma-wrapper's own content-hash basename (its + // hash doesn't vary with --chromosomes), and ravanan's + // --output-directory step symlinks every scattered output into one + // flat directory by basename, which collides across chromosomes -- + // a ravanan/gemma-wrapper interaction bug, not something fixable + // on either side alone. Harmless to skip: it's optional, and this + // code reads paths straight out of ravanan's own --store via its + // stdout output record, never through --output-directory. let args = vec![ store_arg.as_str(), manifest_arg.as_str(), - workflow_path, + compiled_workflow_arg.as_str(), inputs_yaml_arg.as_str(), ]; let ravanan_note = format!("ravanan: {ravanan_cmd} {}", args.join(" ")); @@ -677,7 +699,11 @@ pub async fn compute_ravanan( Ok(GemmaOutput { columns, rows: Value::Array(rows), - notes: vec![ravanan_note, chromosomes_note, format!("workflow: {workflow_path}")], + notes: vec![ + ravanan_note, + chromosomes_note, + format!("workflow: {workflow_scm_path} (compiled to {compiled_workflow_arg})"), + ], }) } diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs index 85672ab..32d1818 100644 --- a/src/routes/gemma.rs +++ b/src/routes/gemma.rs @@ -152,8 +152,8 @@ pub async fn compute_gemma( description = "Same computation and request/response shape as POST \ /compute/gemma (gemma-wrapper's K-matrix step, then its LMM association \ step, both --loco), but driven by ravanan \ -(https://github.com/vepiteski/ravanan) running the CWL workflow at \ -cwl/gemma/gemma-loco.cwl instead of this server's own direct subprocess \ +(https://github.com/vepiteski/ravanan) running a CWL workflow compiled \ +from the ccwl source at cwl/gemma/gemma-loco.scm instead of this server's own direct subprocess \ calls -- an alternative runner for the same two gemma-wrapper steps, \ for comparing the two (correctness, timing). Exploratory: no caching of \ ravanan's own store between requests beyond its --store directory, and \ @@ -204,6 +204,7 @@ pub async fn compute_gemma_ravanan( let config = Config::load(); gemma::compute_ravanan( &config.ravanan_cmd, + &config.ccwl_cmd, &config.gemma_ravanan_workflow, &config.gemma_ravanan_manifest, &config.ravanan_cache_dir, |
