diff options
| -rw-r--r-- | .gitignore | 4 | ||||
| -rw-r--r-- | README.md | 37 | ||||
| -rw-r--r-- | cwl/gemma/gemma-loco.cwl | 175 | ||||
| -rw-r--r-- | cwl/gemma/gemma-loco.scm | 9 | ||||
| -rw-r--r-- | src/config.rs | 24 | ||||
| -rw-r--r-- | src/gemma.rs | 56 | ||||
| -rw-r--r-- | src/routes/gemma.rs | 5 | ||||
| -rw-r--r-- | tests/gemma.rs | 15 |
8 files changed, 106 insertions, 219 deletions
diff --git a/.gitignore b/.gitignore index e653f36..f6c2a1b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ /target __pycache__ +# Generated by `ccwl compile` from cwl/gemma/gemma-loco.scm -- a +# throwaway artifact compute_ravanan regenerates fresh on every call; +# never commit it. +/cwl/gemma/gemma-loco.cwl diff --git a/README.md b/README.md index 5c5719f..421e56f 100644 --- a/README.md +++ b/README.md @@ -1002,32 +1002,37 @@ yet) is a 404, not an empty response -- genuinely different from Same request/response shape as `/compute/gemma`, same two gemma-wrapper steps (kinship, then GWA, both `--loco`), but driven by -[ravanan](https://github.com/vepiteski/ravanan) running the CWL -workflow at `cwl/gemma/gemma-loco.cwl` instead of this server's own +[ravanan](https://github.com/vepiteski/ravanan) running a CWL workflow +compiled on every call (via `ccwl compile`, see below) from the ccwl +source at `cwl/gemma/gemma-loco.scm`, instead of this server's own direct subprocess calls -- an alternative runner for the same computation, kept side by side with `/compute/gemma` so the two can be compared (correctness, timing). Unlike gemma-wrapper's own `--loco` mode, which fans its per-chromosome kinship/GWA jobs out internally via GNU `parallel`, this workflow has ravanan itself do that fan-out: it scatters one `gemma-wrapper --chromosomes <chr> --no-parallel ...` -invocation per chromosome (written in +invocation per chromosome, each restricted to exactly the chromosome +it was given (gemma-wrapper itself still does the correct "exclude +this chromosome from kinship, test only its markers" LOCO logic, +against the *full* genotype/SNP files -- `--chromosomes` only +overrides which chromosomes gemma-wrapper loops over, not the +per-chromosome math itself). This workflow is written in [ccwl](https://ccwl.systemreboot.net/) at `cwl/gemma/gemma-loco.scm`, -compiled to `gemma-loco.cwl` with `ccwl compile`), each restricted to -exactly the chromosome it was -given (gemma-wrapper itself still does the correct "exclude this -chromosome from kinship, test only its markers" LOCO logic, against -the *full* genotype/SNP files -- `--chromosomes` only overrides which -chromosomes gemma-wrapper loops over, not the per-chromosome math -itself). `compute_ravanan` computes the chromosome list from -`snps_file` itself (gemma-wrapper's own `get_chromosomes`, reproduced -as `chromosomes_in`) and passes it as the workflow's `chromosomes` -input, since CWL's `scatter` needs a concrete array at invocation -time. `ravanan_cmd`, `gemma_ravanan_workflow`, `gemma_ravanan_manifest` +not raw CWL; `compute_ravanan` compiles it to CWL with `ccwl compile` +fresh on every call (cheap -- pure macro expansion, no Guix build +involved) into a generated, throwaway file under `ravanan_cache_dir`, +never checked into the repo. `compute_ravanan` computes the chromosome +list from `snps_file` itself (gemma-wrapper's own `get_chromosomes`, +reproduced as `chromosomes_in`) and passes it as the workflow's +`chromosomes` input, since CWL's `scatter` needs a concrete array at +invocation time. `ravanan_cmd`, `ccwl_cmd`, `gemma_ravanan_workflow` +(the ccwl source, not a CWL file), `gemma_ravanan_manifest` (ravanan's own required `--guix-manifest`, a Guix manifest naming `gemma-wrapper`, at `cwl/gemma/manifest.scm`) and `ravanan_cache_dir` (its own cache directory, separate from `gemma_cache_dir`, for pheno -files, run logs, and ravanan's own `--store`) are configurable the -same way as the rest of `src/config.rs`'s layered config. +files, run logs, the compiled workflow, and ravanan's own `--store`) +are configurable the same way as the rest of `src/config.rs`'s layered +config. `GET /compute/gemma/status/{run_id}` is shared between both endpoints and checks whichever cache directory the `run_id` actually logged to. diff --git a/cwl/gemma/gemma-loco.cwl b/cwl/gemma/gemma-loco.cwl deleted file mode 100644 index 5a27e7d..0000000 --- a/cwl/gemma/gemma-loco.cwl +++ /dev/null @@ -1,175 +0,0 @@ -cwlVersion: v1.2 -class: Workflow -requirements: - SubworkflowFeatureRequirement: {} - ScatterFeatureRequirement: {} -inputs: - genotype_file: - type: File - pheno_file: - type: File - snps_file: - type: File - lmm: - type: string - chromosomes: - type: - type: array - items: string - cache_dirs: - type: - type: array - items: string -outputs: - gwa_json: - type: - type: array - items: File - outputSource: gwa/gwa_json - archive: - type: - type: array - items: File - outputSource: gwa/archive -steps: - kinship: - in: - genotype_file: genotype_file - pheno_file: pheno_file - snps_file: snps_file - chr: chromosomes - cache_dir: cache_dirs - out: - - kinship_json - - archive - run: - cwlVersion: v1.2 - class: CommandLineTool - requirements: {} - arguments: - - position: 0 - valueFrom: "gemma-wrapper" - - position: 1 - valueFrom: "--json" - - position: 2 - valueFrom: "--loco" - - position: 3 - valueFrom: "--no-parallel" - - position: 6 - valueFrom: "--" - - position: 10 - valueFrom: "-gk" - inputs: - cache_dir: - type: string - inputBinding: - position: 4 - prefix: "--cache-dir" - chr: - type: string - inputBinding: - position: 5 - prefix: "--chromosomes" - genotype_file: - type: File - inputBinding: - position: 7 - prefix: "-g" - pheno_file: - type: File - inputBinding: - position: 8 - prefix: "-p" - snps_file: - type: File - inputBinding: - position: 9 - prefix: "-a" - outputs: - kinship_json: - type: stdout - archive: - type: File - outputBinding: - glob: "$(inputs.cache_dir)/*-gemma-cXX.tar.xz" - stdout: kinship.json - scatter: - - chr - - cache_dir - scatterMethod: dotproduct - gwa: - in: - genotype_file: genotype_file - pheno_file: pheno_file - snps_file: snps_file - lmm: lmm - chr: chromosomes - cache_dir: cache_dirs - kinship_json: kinship/kinship_json - out: - - gwa_json - - archive - run: - cwlVersion: v1.2 - class: CommandLineTool - requirements: {} - arguments: - - position: 0 - valueFrom: "gemma-wrapper" - - position: 1 - valueFrom: "--json" - - position: 2 - valueFrom: "--loco" - - position: 3 - valueFrom: "--no-parallel" - - position: 7 - valueFrom: "--" - inputs: - cache_dir: - type: string - inputBinding: - position: 4 - prefix: "--cache-dir" - kinship_json: - type: File - inputBinding: - position: 5 - prefix: "--input" - chr: - type: string - inputBinding: - position: 6 - prefix: "--chromosomes" - genotype_file: - type: File - inputBinding: - position: 8 - prefix: "-g" - pheno_file: - type: File - inputBinding: - position: 9 - prefix: "-p" - snps_file: - type: File - inputBinding: - position: 10 - prefix: "-a" - lmm: - type: string - inputBinding: - position: 11 - prefix: "-lmm" - outputs: - gwa_json: - type: stdout - archive: - type: File - outputBinding: - glob: "$(inputs.cache_dir)/*-gemma-GWA.tar.xz" - stdout: "$(inputs.chr)-gwa.json" - scatter: - - chr - - cache_dir - - kinship_json - scatterMethod: dotproduct diff --git a/cwl/gemma/gemma-loco.scm b/cwl/gemma/gemma-loco.scm index 31c12e2..8c3ecda 100644 --- a/cwl/gemma/gemma-loco.scm +++ b/cwl/gemma/gemma-loco.scm @@ -7,10 +7,13 @@ ;; do, so the two can be compared (correctness, timing). ;; ;; Written in ccwl (https://ccwl.systemreboot.net/), per Arun's -;; request, and compiled to cwl/gemma/gemma-loco.cwl: +;; request, not raw CWL. ravanan itself only runs compiled CWL, so +;; src/gemma.rs's compute_ravanan compiles this file fresh on every +;; call (`ccwl compile --output=<generated path> gemma-loco.scm`, +;; cheap: pure macro expansion, no Guix build involved) into a +;; throwaway file under its cache dir -- the compiled .cwl is never +;; checked into the repo. To compile it by hand for inspection: ;; ccwl compile --output=gemma-loco.cwl gemma-loco.scm -;; ravanan itself still only runs the compiled .cwl; nothing here -;; changes how src/gemma.rs's compute_ravanan invokes ravanan. (define kinship ;; gemma-wrapper's --loco kinship (-gk) step, restricted to one diff --git a/src/config.rs b/src/config.rs index fa8b610..e64f2ad 100644 --- a/src/config.rs +++ b/src/config.rs @@ -72,10 +72,17 @@ pub struct Config { /// (see src/gemma.rs's compute_ravanan). Defaults to "ravanan", /// resolved via PATH. pub ravanan_cmd: String, - /// Path to the CWL workflow POST /compute/gemma/ravanan runs under - /// ravanan. Defaults to that path relative to the working - /// directory the server is started from, same convention as - /// `rqtl_wrapper_path`. + /// The ccwl (https://ccwl.systemreboot.net/) executable used to + /// compile `gemma_ravanan_workflow` to CWL before each ravanan run + /// (see src/gemma.rs's compute_ravanan). Defaults to "ccwl", + /// resolved via PATH. + pub ccwl_cmd: String, + /// Path to the ccwl source POST /compute/gemma/ravanan compiles to + /// CWL and runs under ravanan. Not a CWL file itself -- the + /// compiled .cwl is a generated, throwaway artifact (written under + /// `ravanan_cache_dir` on every call), never checked into the repo. + /// Defaults to that path relative to the working directory the + /// server is started from, same convention as `rqtl_wrapper_path`. pub gemma_ravanan_workflow: String, /// Path to the Guix manifest ravanan resolves its jobs' commands /// against (its own --guix-manifest switch, required -- ravanan @@ -149,9 +156,10 @@ impl Config { .into_owned(), ); values.insert("RAVANAN_CMD".to_string(), "ravanan".to_string()); + values.insert("CCWL_CMD".to_string(), "ccwl".to_string()); values.insert( "GEMMA_RAVANAN_WORKFLOW".to_string(), - "cwl/gemma/gemma-loco.cwl".to_string(), + "cwl/gemma/gemma-loco.scm".to_string(), ); values.insert( "GEMMA_RAVANAN_MANIFEST".to_string(), @@ -203,6 +211,7 @@ impl Config { "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", "RAVANAN_CMD", + "CCWL_CMD", "GEMMA_RAVANAN_WORKFLOW", "GEMMA_RAVANAN_MANIFEST", "RAVANAN_CACHE_DIR", @@ -237,6 +246,7 @@ impl Config { gemma_wrapper_cmd: values["GEMMA_WRAPPER_CMD"].clone(), gemma_cache_dir: values["GEMMA_CACHE_DIR"].clone(), ravanan_cmd: values["RAVANAN_CMD"].clone(), + ccwl_cmd: values["CCWL_CMD"].clone(), gemma_ravanan_workflow: values["GEMMA_RAVANAN_WORKFLOW"].clone(), gemma_ravanan_manifest: values["GEMMA_RAVANAN_MANIFEST"].clone(), ravanan_cache_dir: values["RAVANAN_CACHE_DIR"].clone(), @@ -315,6 +325,7 @@ mod tests { "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", "RAVANAN_CMD", + "CCWL_CMD", "GEMMA_RAVANAN_WORKFLOW", "GEMMA_RAVANAN_MANIFEST", "RAVANAN_CACHE_DIR", @@ -332,7 +343,8 @@ mod tests { assert_eq!(config.gemma_wrapper_cmd, "gemma-wrapper"); assert!(config.gemma_cache_dir.ends_with("gn4-gemma")); assert_eq!(config.ravanan_cmd, "ravanan"); - assert_eq!(config.gemma_ravanan_workflow, "cwl/gemma/gemma-loco.cwl"); + assert_eq!(config.ccwl_cmd, "ccwl"); + assert_eq!(config.gemma_ravanan_workflow, "cwl/gemma/gemma-loco.scm"); assert_eq!(config.gemma_ravanan_manifest, "cwl/gemma/manifest.scm"); assert!(config.ravanan_cache_dir.ends_with("gn4-gemma-ravanan")); assert_eq!(config.rqtl_wrapper_path, "scripts/rqtl_wrapper.R"); diff --git a/src/gemma.rs b/src/gemma.rs index 860ef9e..51e3fce 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -462,7 +462,7 @@ pub async fn compute( /// Read a scattered `File[]` output's list of `path`s from ravanan's /// stdout (the CWL outputs record for a scattered output: /// `{"<output id>": [{"class": "File", "path": ..., ...}, ...], ...}`, -/// one entry per `cwl/gemma/gemma-loco.cwl` chromosome). +/// one entry per `cwl/gemma/gemma-loco.scm` chromosome). fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str>, GemmaError> { let files = outputs .get(id) @@ -484,7 +484,7 @@ fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str> /// `compute_ravanan`'s `chromosomes` scatter input (CWL's `scatter` /// needs a concrete array at invocation time, which can't be /// discovered by a step inside the workflow itself without another -/// round of plumbing -- see `cwl/gemma/gemma-loco.cwl`'s own doc +/// round of plumbing -- see `cwl/gemma/gemma-loco.scm`'s own doc /// comment on this input). fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> { let contents = std::fs::read_to_string(snps_file).map_err(GemmaError::Io)?; @@ -502,7 +502,8 @@ fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> { /// The same GEMMA `--loco` kinship-then-GWA computation as `compute`, /// but run via `ravanan` (https://github.com/vepiteski/ravanan) driving -/// the CWL workflow at `workflow_path` (see `cwl/gemma/gemma-loco.cwl`) +/// a CWL workflow compiled from the ccwl (https://ccwl.systemreboot.net/) +/// source at `workflow_scm_path` (see `cwl/gemma/gemma-loco.scm`) /// instead of this module's own direct subprocess calls: one /// gemma-wrapper invocation per chromosome, scattered by ravanan /// itself (`--chromosomes`/`--no-parallel`, see @@ -512,7 +513,8 @@ fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> { /// Backs `POST /compute/gemma/ravanan`. pub async fn compute_ravanan( ravanan_cmd: &str, - workflow_path: &str, + ccwl_cmd: &str, + workflow_scm_path: &str, manifest_path: &str, cache_dir: &str, genotype_file: &str, @@ -528,6 +530,26 @@ pub async fn compute_ravanan( let log_path = streaming::create_run_log(cache_dir, run_id)?; + // gemma-loco.scm is the maintained source; the CWL ravanan actually + // runs is a generated, throwaway artifact compiled fresh on every + // call (cheap: ccwl compile is pure macro expansion, no Guix build + // involved) and never checked into the repo -- see the doc comment + // on Config::gemma_ravanan_workflow. + let compiled_workflow_path = + Path::new(cache_dir).join(format!("{unique}-gemma-loco.cwl")); + let ccwl_output_arg = format!("--output={}", compiled_workflow_path.display()); + let ccwl_args = vec!["compile", ccwl_output_arg.as_str(), workflow_scm_path]; + let ccwl_output = streaming::run_streaming(ccwl_cmd, &ccwl_args, None, &[]) + .await + .map_err(GemmaError::Io)?; + if !ccwl_output.status.success() { + return Err(GemmaError::CommandFailed { + step: "ccwl compile", + stderr: String::from_utf8_lossy(&ccwl_output.stderr).into_owned(), + }); + } + let compiled_workflow_arg = compiled_workflow_path.to_string_lossy().into_owned(); + let store_dir = Path::new(cache_dir).join("ravanan-store"); // ravanan's --store needs these three subdirectories to already // exist; it does not create them itself. @@ -584,19 +606,19 @@ pub async fn compute_ravanan( let store_arg = format!("--store={}", store_dir.display()); let manifest_arg = format!("--guix-manifest={manifest_path}"); let inputs_yaml_arg = inputs_yaml_path.to_string_lossy().into_owned(); - // No --output-directory: gwa.cwl's per-chromosome archive outputs - // all share gemma-wrapper's own content-hash basename (its hash - // doesn't vary with --chromosomes), and ravanan's --output-directory - // step symlinks every scattered output into one flat directory by - // basename, which collides across chromosomes -- a ravanan/ - // gemma-wrapper interaction bug, not something fixable on either - // side alone. Harmless to skip: it's optional, and this code reads - // paths straight out of ravanan's own --store via its stdout - // output record, never through --output-directory. + // No --output-directory: the gwa step's per-chromosome archive + // outputs all share gemma-wrapper's own content-hash basename (its + // hash doesn't vary with --chromosomes), and ravanan's + // --output-directory step symlinks every scattered output into one + // flat directory by basename, which collides across chromosomes -- + // a ravanan/gemma-wrapper interaction bug, not something fixable + // on either side alone. Harmless to skip: it's optional, and this + // code reads paths straight out of ravanan's own --store via its + // stdout output record, never through --output-directory. let args = vec![ store_arg.as_str(), manifest_arg.as_str(), - workflow_path, + compiled_workflow_arg.as_str(), inputs_yaml_arg.as_str(), ]; let ravanan_note = format!("ravanan: {ravanan_cmd} {}", args.join(" ")); @@ -677,7 +699,11 @@ pub async fn compute_ravanan( Ok(GemmaOutput { columns, rows: Value::Array(rows), - notes: vec![ravanan_note, chromosomes_note, format!("workflow: {workflow_path}")], + notes: vec![ + ravanan_note, + chromosomes_note, + format!("workflow: {workflow_scm_path} (compiled to {compiled_workflow_arg})"), + ], }) } diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs index 85672ab..32d1818 100644 --- a/src/routes/gemma.rs +++ b/src/routes/gemma.rs @@ -152,8 +152,8 @@ pub async fn compute_gemma( description = "Same computation and request/response shape as POST \ /compute/gemma (gemma-wrapper's K-matrix step, then its LMM association \ step, both --loco), but driven by ravanan \ -(https://github.com/vepiteski/ravanan) running the CWL workflow at \ -cwl/gemma/gemma-loco.cwl instead of this server's own direct subprocess \ +(https://github.com/vepiteski/ravanan) running a CWL workflow compiled \ +from the ccwl source at cwl/gemma/gemma-loco.scm instead of this server's own direct subprocess \ calls -- an alternative runner for the same two gemma-wrapper steps, \ for comparing the two (correctness, timing). Exploratory: no caching of \ ravanan's own store between requests beyond its --store directory, and \ @@ -204,6 +204,7 @@ pub async fn compute_gemma_ravanan( let config = Config::load(); gemma::compute_ravanan( &config.ravanan_cmd, + &config.ccwl_cmd, &config.gemma_ravanan_workflow, &config.gemma_ravanan_manifest, &config.ravanan_cache_dir, diff --git a/tests/gemma.rs b/tests/gemma.rs index ae1a3c1..7ce47e3 100644 --- a/tests/gemma.rs +++ b/tests/gemma.rs @@ -27,6 +27,17 @@ fn ravanan_available() -> bool { .is_ok() } +/// True only when ccwl is actually runnable. compute_ravanan compiles +/// cwl/gemma/gemma-loco.scm to CWL with it before every ravanan run. +/// ccwl has no top-level --version; "compile --help" is the cheapest +/// subcommand invocation that doesn't need a real source file. +fn ccwl_available() -> bool { + std::process::Command::new("ccwl") + .args(["compile", "--help"]) + .output() + .is_ok() +} + fn test_data_path(name: &str) -> String { format!("{}/tests/test_data/{name}", env!("CARGO_MANIFEST_DIR")) } @@ -180,9 +191,9 @@ async fn compute_gemma_maps_a_qtl_from_the_bxd_fixtures() { #[tokio::test] async fn compute_gemma_ravanan_maps_a_qtl_from_the_bxd_fixtures() { - if !ravanan_available() || !gemma_wrapper_available() { + if !ravanan_available() || !gemma_wrapper_available() || !ccwl_available() { eprintln!( - "skipping compute_gemma_ravanan_maps_a_qtl_from_the_bxd_fixtures: ravanan or gemma-wrapper not on PATH" + "skipping compute_gemma_ravanan_maps_a_qtl_from_the_bxd_fixtures: ravanan, gemma-wrapper or ccwl not on PATH" ); return; } |
