about summary refs log tree commit diff
diff options
context:
space:
mode:
-rw-r--r--.gitignore4
-rw-r--r--README.md37
-rw-r--r--cwl/gemma/gemma-loco.cwl175
-rw-r--r--cwl/gemma/gemma-loco.scm9
-rw-r--r--src/config.rs24
-rw-r--r--src/gemma.rs56
-rw-r--r--src/routes/gemma.rs5
-rw-r--r--tests/gemma.rs15
8 files changed, 106 insertions, 219 deletions
diff --git a/.gitignore b/.gitignore
index e653f36..f6c2a1b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,6 @@
 /target
 __pycache__
+# Generated by `ccwl compile` from cwl/gemma/gemma-loco.scm -- a
+# throwaway artifact compute_ravanan regenerates fresh on every call;
+# never commit it.
+/cwl/gemma/gemma-loco.cwl
diff --git a/README.md b/README.md
index 5c5719f..421e56f 100644
--- a/README.md
+++ b/README.md
@@ -1002,32 +1002,37 @@ yet) is a 404, not an empty response -- genuinely different from
 
 Same request/response shape as `/compute/gemma`, same two gemma-wrapper
 steps (kinship, then GWA, both `--loco`), but driven by
-[ravanan](https://github.com/vepiteski/ravanan) running the CWL
-workflow at `cwl/gemma/gemma-loco.cwl` instead of this server's own
+[ravanan](https://github.com/vepiteski/ravanan) running a CWL workflow
+compiled on every call (via `ccwl compile`, see below) from the ccwl
+source at `cwl/gemma/gemma-loco.scm`, instead of this server's own
 direct subprocess calls -- an alternative runner for the same
 computation, kept side by side with `/compute/gemma` so the two can be
 compared (correctness, timing). Unlike gemma-wrapper's own `--loco`
 mode, which fans its per-chromosome kinship/GWA jobs out internally via
 GNU `parallel`, this workflow has ravanan itself do that fan-out: it
 scatters one `gemma-wrapper --chromosomes <chr> --no-parallel ...`
-invocation per chromosome (written in
+invocation per chromosome, each restricted to exactly the chromosome
+it was given (gemma-wrapper itself still does the correct "exclude
+this chromosome from kinship, test only its markers" LOCO logic,
+against the *full* genotype/SNP files -- `--chromosomes` only
+overrides which chromosomes gemma-wrapper loops over, not the
+per-chromosome math itself). This workflow is written in
 [ccwl](https://ccwl.systemreboot.net/) at `cwl/gemma/gemma-loco.scm`,
-compiled to `gemma-loco.cwl` with `ccwl compile`), each restricted to
-exactly the chromosome it was
-given (gemma-wrapper itself still does the correct "exclude this
-chromosome from kinship, test only its markers" LOCO logic, against
-the *full* genotype/SNP files -- `--chromosomes` only overrides which
-chromosomes gemma-wrapper loops over, not the per-chromosome math
-itself). `compute_ravanan` computes the chromosome list from
-`snps_file` itself (gemma-wrapper's own `get_chromosomes`, reproduced
-as `chromosomes_in`) and passes it as the workflow's `chromosomes`
-input, since CWL's `scatter` needs a concrete array at invocation
-time. `ravanan_cmd`, `gemma_ravanan_workflow`, `gemma_ravanan_manifest`
+not raw CWL; `compute_ravanan` compiles it to CWL with `ccwl compile`
+fresh on every call (cheap -- pure macro expansion, no Guix build
+involved) into a generated, throwaway file under `ravanan_cache_dir`,
+never checked into the repo. `compute_ravanan` computes the chromosome
+list from `snps_file` itself (gemma-wrapper's own `get_chromosomes`,
+reproduced as `chromosomes_in`) and passes it as the workflow's
+`chromosomes` input, since CWL's `scatter` needs a concrete array at
+invocation time. `ravanan_cmd`, `ccwl_cmd`, `gemma_ravanan_workflow`
+(the ccwl source, not a CWL file), `gemma_ravanan_manifest`
 (ravanan's own required `--guix-manifest`, a Guix manifest naming
 `gemma-wrapper`, at `cwl/gemma/manifest.scm`) and `ravanan_cache_dir`
 (its own cache directory, separate from `gemma_cache_dir`, for pheno
-files, run logs, and ravanan's own `--store`) are configurable the
-same way as the rest of `src/config.rs`'s layered config.
+files, run logs, the compiled workflow, and ravanan's own `--store`)
+are configurable the same way as the rest of `src/config.rs`'s layered
+config.
 `GET /compute/gemma/status/{run_id}` is shared between both endpoints
 and checks whichever cache directory the `run_id` actually logged to.
 
diff --git a/cwl/gemma/gemma-loco.cwl b/cwl/gemma/gemma-loco.cwl
deleted file mode 100644
index 5a27e7d..0000000
--- a/cwl/gemma/gemma-loco.cwl
+++ /dev/null
@@ -1,175 +0,0 @@
-cwlVersion: v1.2
-class: Workflow
-requirements:
-  SubworkflowFeatureRequirement: {}
-  ScatterFeatureRequirement: {}
-inputs:
-  genotype_file:
-    type: File
-  pheno_file:
-    type: File
-  snps_file:
-    type: File
-  lmm:
-    type: string
-  chromosomes:
-    type:
-      type: array
-      items: string
-  cache_dirs:
-    type:
-      type: array
-      items: string
-outputs:
-  gwa_json:
-    type:
-      type: array
-      items: File
-    outputSource: gwa/gwa_json
-  archive:
-    type:
-      type: array
-      items: File
-    outputSource: gwa/archive
-steps:
-  kinship:
-    in:
-      genotype_file: genotype_file
-      pheno_file: pheno_file
-      snps_file: snps_file
-      chr: chromosomes
-      cache_dir: cache_dirs
-    out:
-      - kinship_json
-      - archive
-    run:
-      cwlVersion: v1.2
-      class: CommandLineTool
-      requirements: {}
-      arguments:
-        - position: 0
-          valueFrom: "gemma-wrapper"
-        - position: 1
-          valueFrom: "--json"
-        - position: 2
-          valueFrom: "--loco"
-        - position: 3
-          valueFrom: "--no-parallel"
-        - position: 6
-          valueFrom: "--"
-        - position: 10
-          valueFrom: "-gk"
-      inputs:
-        cache_dir:
-          type: string
-          inputBinding:
-            position: 4
-            prefix: "--cache-dir"
-        chr:
-          type: string
-          inputBinding:
-            position: 5
-            prefix: "--chromosomes"
-        genotype_file:
-          type: File
-          inputBinding:
-            position: 7
-            prefix: "-g"
-        pheno_file:
-          type: File
-          inputBinding:
-            position: 8
-            prefix: "-p"
-        snps_file:
-          type: File
-          inputBinding:
-            position: 9
-            prefix: "-a"
-      outputs:
-        kinship_json:
-          type: stdout
-        archive:
-          type: File
-          outputBinding:
-            glob: "$(inputs.cache_dir)/*-gemma-cXX.tar.xz"
-      stdout: kinship.json
-    scatter:
-      - chr
-      - cache_dir
-    scatterMethod: dotproduct
-  gwa:
-    in:
-      genotype_file: genotype_file
-      pheno_file: pheno_file
-      snps_file: snps_file
-      lmm: lmm
-      chr: chromosomes
-      cache_dir: cache_dirs
-      kinship_json: kinship/kinship_json
-    out:
-      - gwa_json
-      - archive
-    run:
-      cwlVersion: v1.2
-      class: CommandLineTool
-      requirements: {}
-      arguments:
-        - position: 0
-          valueFrom: "gemma-wrapper"
-        - position: 1
-          valueFrom: "--json"
-        - position: 2
-          valueFrom: "--loco"
-        - position: 3
-          valueFrom: "--no-parallel"
-        - position: 7
-          valueFrom: "--"
-      inputs:
-        cache_dir:
-          type: string
-          inputBinding:
-            position: 4
-            prefix: "--cache-dir"
-        kinship_json:
-          type: File
-          inputBinding:
-            position: 5
-            prefix: "--input"
-        chr:
-          type: string
-          inputBinding:
-            position: 6
-            prefix: "--chromosomes"
-        genotype_file:
-          type: File
-          inputBinding:
-            position: 8
-            prefix: "-g"
-        pheno_file:
-          type: File
-          inputBinding:
-            position: 9
-            prefix: "-p"
-        snps_file:
-          type: File
-          inputBinding:
-            position: 10
-            prefix: "-a"
-        lmm:
-          type: string
-          inputBinding:
-            position: 11
-            prefix: "-lmm"
-      outputs:
-        gwa_json:
-          type: stdout
-        archive:
-          type: File
-          outputBinding:
-            glob: "$(inputs.cache_dir)/*-gemma-GWA.tar.xz"
-      stdout: "$(inputs.chr)-gwa.json"
-    scatter:
-      - chr
-      - cache_dir
-      - kinship_json
-    scatterMethod: dotproduct
diff --git a/cwl/gemma/gemma-loco.scm b/cwl/gemma/gemma-loco.scm
index 31c12e2..8c3ecda 100644
--- a/cwl/gemma/gemma-loco.scm
+++ b/cwl/gemma/gemma-loco.scm
@@ -7,10 +7,13 @@
 ;; do, so the two can be compared (correctness, timing).
 ;;
 ;; Written in ccwl (https://ccwl.systemreboot.net/), per Arun's
-;; request, and compiled to cwl/gemma/gemma-loco.cwl:
+;; request, not raw CWL. ravanan itself only runs compiled CWL, so
+;; src/gemma.rs's compute_ravanan compiles this file fresh on every
+;; call (`ccwl compile --output=<generated path> gemma-loco.scm`,
+;; cheap: pure macro expansion, no Guix build involved) into a
+;; throwaway file under its cache dir -- the compiled .cwl is never
+;; checked into the repo. To compile it by hand for inspection:
 ;;   ccwl compile --output=gemma-loco.cwl gemma-loco.scm
-;; ravanan itself still only runs the compiled .cwl; nothing here
-;; changes how src/gemma.rs's compute_ravanan invokes ravanan.
 
 (define kinship
   ;; gemma-wrapper's --loco kinship (-gk) step, restricted to one
diff --git a/src/config.rs b/src/config.rs
index fa8b610..e64f2ad 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -72,10 +72,17 @@ pub struct Config {
     /// (see src/gemma.rs's compute_ravanan). Defaults to "ravanan",
     /// resolved via PATH.
     pub ravanan_cmd: String,
-    /// Path to the CWL workflow POST /compute/gemma/ravanan runs under
-    /// ravanan. Defaults to that path relative to the working
-    /// directory the server is started from, same convention as
-    /// `rqtl_wrapper_path`.
+    /// The ccwl (https://ccwl.systemreboot.net/) executable used to
+    /// compile `gemma_ravanan_workflow` to CWL before each ravanan run
+    /// (see src/gemma.rs's compute_ravanan). Defaults to "ccwl",
+    /// resolved via PATH.
+    pub ccwl_cmd: String,
+    /// Path to the ccwl source POST /compute/gemma/ravanan compiles to
+    /// CWL and runs under ravanan. Not a CWL file itself -- the
+    /// compiled .cwl is a generated, throwaway artifact (written under
+    /// `ravanan_cache_dir` on every call), never checked into the repo.
+    /// Defaults to that path relative to the working directory the
+    /// server is started from, same convention as `rqtl_wrapper_path`.
     pub gemma_ravanan_workflow: String,
     /// Path to the Guix manifest ravanan resolves its jobs' commands
     /// against (its own --guix-manifest switch, required -- ravanan
@@ -149,9 +156,10 @@ impl Config {
                 .into_owned(),
         );
         values.insert("RAVANAN_CMD".to_string(), "ravanan".to_string());
+        values.insert("CCWL_CMD".to_string(), "ccwl".to_string());
         values.insert(
             "GEMMA_RAVANAN_WORKFLOW".to_string(),
-            "cwl/gemma/gemma-loco.cwl".to_string(),
+            "cwl/gemma/gemma-loco.scm".to_string(),
         );
         values.insert(
             "GEMMA_RAVANAN_MANIFEST".to_string(),
@@ -203,6 +211,7 @@ impl Config {
             "GEMMA_WRAPPER_CMD",
             "GEMMA_CACHE_DIR",
             "RAVANAN_CMD",
+            "CCWL_CMD",
             "GEMMA_RAVANAN_WORKFLOW",
             "GEMMA_RAVANAN_MANIFEST",
             "RAVANAN_CACHE_DIR",
@@ -237,6 +246,7 @@ impl Config {
             gemma_wrapper_cmd: values["GEMMA_WRAPPER_CMD"].clone(),
             gemma_cache_dir: values["GEMMA_CACHE_DIR"].clone(),
             ravanan_cmd: values["RAVANAN_CMD"].clone(),
+            ccwl_cmd: values["CCWL_CMD"].clone(),
             gemma_ravanan_workflow: values["GEMMA_RAVANAN_WORKFLOW"].clone(),
             gemma_ravanan_manifest: values["GEMMA_RAVANAN_MANIFEST"].clone(),
             ravanan_cache_dir: values["RAVANAN_CACHE_DIR"].clone(),
@@ -315,6 +325,7 @@ mod tests {
             "GEMMA_WRAPPER_CMD",
             "GEMMA_CACHE_DIR",
             "RAVANAN_CMD",
+            "CCWL_CMD",
             "GEMMA_RAVANAN_WORKFLOW",
             "GEMMA_RAVANAN_MANIFEST",
             "RAVANAN_CACHE_DIR",
@@ -332,7 +343,8 @@ mod tests {
         assert_eq!(config.gemma_wrapper_cmd, "gemma-wrapper");
         assert!(config.gemma_cache_dir.ends_with("gn4-gemma"));
         assert_eq!(config.ravanan_cmd, "ravanan");
-        assert_eq!(config.gemma_ravanan_workflow, "cwl/gemma/gemma-loco.cwl");
+        assert_eq!(config.ccwl_cmd, "ccwl");
+        assert_eq!(config.gemma_ravanan_workflow, "cwl/gemma/gemma-loco.scm");
         assert_eq!(config.gemma_ravanan_manifest, "cwl/gemma/manifest.scm");
         assert!(config.ravanan_cache_dir.ends_with("gn4-gemma-ravanan"));
         assert_eq!(config.rqtl_wrapper_path, "scripts/rqtl_wrapper.R");
diff --git a/src/gemma.rs b/src/gemma.rs
index 860ef9e..51e3fce 100644
--- a/src/gemma.rs
+++ b/src/gemma.rs
@@ -462,7 +462,7 @@ pub async fn compute(
 /// Read a scattered `File[]` output's list of `path`s from ravanan's
 /// stdout (the CWL outputs record for a scattered output:
 /// `{"<output id>": [{"class": "File", "path": ..., ...}, ...], ...}`,
-/// one entry per `cwl/gemma/gemma-loco.cwl` chromosome).
+/// one entry per `cwl/gemma/gemma-loco.scm` chromosome).
 fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str>, GemmaError> {
     let files = outputs
         .get(id)
@@ -484,7 +484,7 @@ fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str>
 /// `compute_ravanan`'s `chromosomes` scatter input (CWL's `scatter`
 /// needs a concrete array at invocation time, which can't be
 /// discovered by a step inside the workflow itself without another
-/// round of plumbing -- see `cwl/gemma/gemma-loco.cwl`'s own doc
+/// round of plumbing -- see `cwl/gemma/gemma-loco.scm`'s own doc
 /// comment on this input).
 fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> {
     let contents = std::fs::read_to_string(snps_file).map_err(GemmaError::Io)?;
@@ -502,7 +502,8 @@ fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> {
 
 /// The same GEMMA `--loco` kinship-then-GWA computation as `compute`,
 /// but run via `ravanan` (https://github.com/vepiteski/ravanan) driving
-/// the CWL workflow at `workflow_path` (see `cwl/gemma/gemma-loco.cwl`)
+/// a CWL workflow compiled from the ccwl (https://ccwl.systemreboot.net/)
+/// source at `workflow_scm_path` (see `cwl/gemma/gemma-loco.scm`)
 /// instead of this module's own direct subprocess calls: one
 /// gemma-wrapper invocation per chromosome, scattered by ravanan
 /// itself (`--chromosomes`/`--no-parallel`, see
@@ -512,7 +513,8 @@ fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> {
 /// Backs `POST /compute/gemma/ravanan`.
 pub async fn compute_ravanan(
     ravanan_cmd: &str,
-    workflow_path: &str,
+    ccwl_cmd: &str,
+    workflow_scm_path: &str,
     manifest_path: &str,
     cache_dir: &str,
     genotype_file: &str,
@@ -528,6 +530,26 @@ pub async fn compute_ravanan(
 
     let log_path = streaming::create_run_log(cache_dir, run_id)?;
 
+    // gemma-loco.scm is the maintained source; the CWL ravanan actually
+    // runs is a generated, throwaway artifact compiled fresh on every
+    // call (cheap: ccwl compile is pure macro expansion, no Guix build
+    // involved) and never checked into the repo -- see the doc comment
+    // on Config::gemma_ravanan_workflow.
+    let compiled_workflow_path =
+        Path::new(cache_dir).join(format!("{unique}-gemma-loco.cwl"));
+    let ccwl_output_arg = format!("--output={}", compiled_workflow_path.display());
+    let ccwl_args = vec!["compile", ccwl_output_arg.as_str(), workflow_scm_path];
+    let ccwl_output = streaming::run_streaming(ccwl_cmd, &ccwl_args, None, &[])
+        .await
+        .map_err(GemmaError::Io)?;
+    if !ccwl_output.status.success() {
+        return Err(GemmaError::CommandFailed {
+            step: "ccwl compile",
+            stderr: String::from_utf8_lossy(&ccwl_output.stderr).into_owned(),
+        });
+    }
+    let compiled_workflow_arg = compiled_workflow_path.to_string_lossy().into_owned();
+
     let store_dir = Path::new(cache_dir).join("ravanan-store");
     // ravanan's --store needs these three subdirectories to already
     // exist; it does not create them itself.
@@ -584,19 +606,19 @@ pub async fn compute_ravanan(
     let store_arg = format!("--store={}", store_dir.display());
     let manifest_arg = format!("--guix-manifest={manifest_path}");
     let inputs_yaml_arg = inputs_yaml_path.to_string_lossy().into_owned();
-    // No --output-directory: gwa.cwl's per-chromosome archive outputs
-    // all share gemma-wrapper's own content-hash basename (its hash
-    // doesn't vary with --chromosomes), and ravanan's --output-directory
-    // step symlinks every scattered output into one flat directory by
-    // basename, which collides across chromosomes -- a ravanan/
-    // gemma-wrapper interaction bug, not something fixable on either
-    // side alone. Harmless to skip: it's optional, and this code reads
-    // paths straight out of ravanan's own --store via its stdout
-    // output record, never through --output-directory.
+    // No --output-directory: the gwa step's per-chromosome archive
+    // outputs all share gemma-wrapper's own content-hash basename (its
+    // hash doesn't vary with --chromosomes), and ravanan's
+    // --output-directory step symlinks every scattered output into one
+    // flat directory by basename, which collides across chromosomes --
+    // a ravanan/gemma-wrapper interaction bug, not something fixable
+    // on either side alone. Harmless to skip: it's optional, and this
+    // code reads paths straight out of ravanan's own --store via its
+    // stdout output record, never through --output-directory.
     let args = vec![
         store_arg.as_str(),
         manifest_arg.as_str(),
-        workflow_path,
+        compiled_workflow_arg.as_str(),
         inputs_yaml_arg.as_str(),
     ];
     let ravanan_note = format!("ravanan: {ravanan_cmd} {}", args.join(" "));
@@ -677,7 +699,11 @@ pub async fn compute_ravanan(
     Ok(GemmaOutput {
         columns,
         rows: Value::Array(rows),
-        notes: vec![ravanan_note, chromosomes_note, format!("workflow: {workflow_path}")],
+        notes: vec![
+            ravanan_note,
+            chromosomes_note,
+            format!("workflow: {workflow_scm_path} (compiled to {compiled_workflow_arg})"),
+        ],
     })
 }
 
diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs
index 85672ab..32d1818 100644
--- a/src/routes/gemma.rs
+++ b/src/routes/gemma.rs
@@ -152,8 +152,8 @@ pub async fn compute_gemma(
     description = "Same computation and request/response shape as POST \
 /compute/gemma (gemma-wrapper's K-matrix step, then its LMM association \
 step, both --loco), but driven by ravanan \
-(https://github.com/vepiteski/ravanan) running the CWL workflow at \
-cwl/gemma/gemma-loco.cwl instead of this server's own direct subprocess \
+(https://github.com/vepiteski/ravanan) running a CWL workflow compiled \
+from the ccwl source at cwl/gemma/gemma-loco.scm instead of this server's own direct subprocess \
 calls -- an alternative runner for the same two gemma-wrapper steps, \
 for comparing the two (correctness, timing). Exploratory: no caching of \
 ravanan's own store between requests beyond its --store directory, and \
@@ -204,6 +204,7 @@ pub async fn compute_gemma_ravanan(
     let config = Config::load();
     gemma::compute_ravanan(
         &config.ravanan_cmd,
+        &config.ccwl_cmd,
         &config.gemma_ravanan_workflow,
         &config.gemma_ravanan_manifest,
         &config.ravanan_cache_dir,
diff --git a/tests/gemma.rs b/tests/gemma.rs
index ae1a3c1..7ce47e3 100644
--- a/tests/gemma.rs
+++ b/tests/gemma.rs
@@ -27,6 +27,17 @@ fn ravanan_available() -> bool {
         .is_ok()
 }
 
+/// True only when ccwl is actually runnable. compute_ravanan compiles
+/// cwl/gemma/gemma-loco.scm to CWL with it before every ravanan run.
+/// ccwl has no top-level --version; "compile --help" is the cheapest
+/// subcommand invocation that doesn't need a real source file.
+fn ccwl_available() -> bool {
+    std::process::Command::new("ccwl")
+        .args(["compile", "--help"])
+        .output()
+        .is_ok()
+}
+
 fn test_data_path(name: &str) -> String {
     format!("{}/tests/test_data/{name}", env!("CARGO_MANIFEST_DIR"))
 }
@@ -180,9 +191,9 @@ async fn compute_gemma_maps_a_qtl_from_the_bxd_fixtures() {
 
 #[tokio::test]
 async fn compute_gemma_ravanan_maps_a_qtl_from_the_bxd_fixtures() {
-    if !ravanan_available() || !gemma_wrapper_available() {
+    if !ravanan_available() || !gemma_wrapper_available() || !ccwl_available() {
         eprintln!(
-            "skipping compute_gemma_ravanan_maps_a_qtl_from_the_bxd_fixtures: ravanan or gemma-wrapper not on PATH"
+            "skipping compute_gemma_ravanan_maps_a_qtl_from_the_bxd_fixtures: ravanan, gemma-wrapper or ccwl not on PATH"
         );
         return;
     }