about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-10-15 13:40:55 +0200
committerPjotr Prins <pjotr.public01@thebird.nl>2026-10-15 13:40:55 +0200
commitba9016c729c020a62cb2b4481214a19f5e027ba6 (patch)
tree3f8631fba703dfc72d03ae1df49fda9bf353e2fe /src
parentb3c43a2bb0cb7f1e5d0c0a3d08f76f82d9ef881f (diff)
downloadgenenetwork4-ba9016c729c020a62cb2b4481214a19f5e027ba6.tar.gz
CWL used to run gemma
Diffstat (limited to 'src')
-rw-r--r--src/config.rs39
-rw-r--r--src/gemma.rs149
-rw-r--r--src/lib.rs1
-rw-r--r--src/routes/gemma.rs99
-rw-r--r--src/rqtl.rs2
-rw-r--r--src/streaming.rs8
6 files changed, 293 insertions, 5 deletions
diff --git a/src/config.rs b/src/config.rs
index 5030576..f0fe946 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -68,6 +68,21 @@ pub struct Config {
     /// results between calls (its own --cache-dir switch). Defaults to
     /// a "gn4-gemma" directory under the system temp dir.
     pub gemma_cache_dir: String,
+    /// The ravanan executable to invoke for POST /compute/gemma/ravanan
+    /// (see src/gemma.rs's compute_ravanan). Defaults to "ravanan",
+    /// resolved via PATH.
+    pub ravanan_cmd: String,
+    /// Path to the CWL workflow POST /compute/gemma/ravanan runs under
+    /// ravanan. Defaults to that path relative to the working
+    /// directory the server is started from, same convention as
+    /// `rqtl_wrapper_path`.
+    pub gemma_ravanan_workflow: String,
+    /// Directory POST /compute/gemma/ravanan uses for phenotype files,
+    /// run logs, and ravanan's own --store and --output-directory.
+    /// Separate from `gemma_cache_dir` so the two runners don't share
+    /// state. Defaults to a "gn4-gemma-ravanan" directory under the
+    /// system temp dir.
+    pub ravanan_cache_dir: String,
     /// Path to scripts/rqtl_wrapper.R (copied here from GeneNetwork3)
     /// for POST /compute/rqtl. Defaults to that path relative to the
     /// working directory the server is started from.
@@ -127,6 +142,18 @@ impl Config {
                 .to_string_lossy()
                 .into_owned(),
         );
+        values.insert("RAVANAN_CMD".to_string(), "ravanan".to_string());
+        values.insert(
+            "GEMMA_RAVANAN_WORKFLOW".to_string(),
+            "cwl/gemma/gemma-loco.cwl".to_string(),
+        );
+        values.insert(
+            "RAVANAN_CACHE_DIR".to_string(),
+            std::env::temp_dir()
+                .join("gn4-gemma-ravanan")
+                .to_string_lossy()
+                .into_owned(),
+        );
         values.insert(
             "RQTL_WRAPPER_PATH".to_string(),
             "scripts/rqtl_wrapper.R".to_string(),
@@ -165,6 +192,9 @@ impl Config {
             "SEARCH_ENGINE",
             "GEMMA_WRAPPER_CMD",
             "GEMMA_CACHE_DIR",
+            "RAVANAN_CMD",
+            "GEMMA_RAVANAN_WORKFLOW",
+            "RAVANAN_CACHE_DIR",
             "RQTL_WRAPPER_PATH",
             "RQTL_CACHE_DIR",
         ] {
@@ -195,6 +225,9 @@ impl Config {
             dev_token_password: values["GN_AUTH_DEV_PASSWORD"].clone(),
             gemma_wrapper_cmd: values["GEMMA_WRAPPER_CMD"].clone(),
             gemma_cache_dir: values["GEMMA_CACHE_DIR"].clone(),
+            ravanan_cmd: values["RAVANAN_CMD"].clone(),
+            gemma_ravanan_workflow: values["GEMMA_RAVANAN_WORKFLOW"].clone(),
+            ravanan_cache_dir: values["RAVANAN_CACHE_DIR"].clone(),
             rqtl_wrapper_path: values["RQTL_WRAPPER_PATH"].clone(),
             rqtl_cache_dir: values["RQTL_CACHE_DIR"].clone(),
         }
@@ -269,6 +302,9 @@ mod tests {
             "SEARCH_ENGINE",
             "GEMMA_WRAPPER_CMD",
             "GEMMA_CACHE_DIR",
+            "RAVANAN_CMD",
+            "GEMMA_RAVANAN_WORKFLOW",
+            "RAVANAN_CACHE_DIR",
             "RQTL_WRAPPER_PATH",
             "RQTL_CACHE_DIR",
         ] {
@@ -282,6 +318,9 @@ mod tests {
         assert_eq!(config.port, 3000);
         assert_eq!(config.gemma_wrapper_cmd, "gemma-wrapper");
         assert!(config.gemma_cache_dir.ends_with("gn4-gemma"));
+        assert_eq!(config.ravanan_cmd, "ravanan");
+        assert_eq!(config.gemma_ravanan_workflow, "cwl/gemma/gemma-loco.cwl");
+        assert!(config.ravanan_cache_dir.ends_with("gn4-gemma-ravanan"));
         assert_eq!(config.rqtl_wrapper_path, "scripts/rqtl_wrapper.R");
         assert!(config.rqtl_cache_dir.ends_with("gn4-rqtl"));
         assert_eq!(config.auth_server_url, "http://127.0.0.1:8081/");
diff --git a/src/gemma.rs b/src/gemma.rs
index 9563131..b0b27a2 100644
--- a/src/gemma.rs
+++ b/src/gemma.rs
@@ -43,6 +43,12 @@ pub enum GemmaError {
     /// before any subprocess runs, so this never silently truncates
     /// another run's in-progress log.
     RunIdInUse(String),
+    /// `ravanan` (see `compute_ravanan`) exited non-zero; carries its
+    /// stderr.
+    RavananFailed(String),
+    /// `ravanan`'s own stdout (the CWL outputs record) was not valid
+    /// JSON, or didn't carry the output this code expects.
+    RavananOutput(String),
 }
 
 impl std::fmt::Display for GemmaError {
@@ -62,6 +68,10 @@ impl std::fmt::Display for GemmaError {
             GemmaError::RunIdInUse(id) => {
                 write!(f, "run id '{id}' is already in use; pick a different id")
             }
+            GemmaError::RavananFailed(stderr) => write!(f, "ravanan failed: {stderr}"),
+            GemmaError::RavananOutput(detail) => {
+                write!(f, "could not read ravanan's output: {detail}")
+            }
         }
     }
 }
@@ -139,7 +149,7 @@ async fn compute_kinship(
     log_path: Option<&Path>,
 ) -> Result<Value, GemmaError> {
     let args = kinship_args(cache_dir, genotype_file, pheno_file, snps_file);
-    let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path)
+    let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path, &[])
         .await
         .map_err(GemmaError::Io)?;
     parse_wrapper_output("kinship", output)
@@ -182,7 +192,7 @@ async fn compute_gwa(
         &lmm_str,
         k_json_path,
     );
-    let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path)
+    let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path, &[])
         .await
         .map_err(GemmaError::Io)?;
     parse_wrapper_output("gwa", output)
@@ -250,6 +260,7 @@ async fn extract_archive(cache_dir: &str, archive: &str, dest_dir: &str) -> Resu
         "tar",
         &["xJf", &archive_path.to_string_lossy(), "-C", dest_dir],
         None,
+        &[],
     )
     .await
     .map_err(GemmaError::Io)?;
@@ -432,6 +443,140 @@ pub async fn compute(
     })
 }
 
+/// Read a `File` output's `path` from ravanan's stdout (the CWL
+/// outputs record: `{"<output id>": {"class": "File", "path": ..., ...}, ...}`).
+fn ravanan_output_path<'a>(outputs: &'a Value, id: &str) -> Result<&'a str, GemmaError> {
+    outputs
+        .get(id)
+        .and_then(|v| v.get("path"))
+        .and_then(Value::as_str)
+        .ok_or_else(|| {
+            GemmaError::RavananOutput(format!("no \"{id}\".path in ravanan's output"))
+        })
+}
+
+/// The same GEMMA `--loco` kinship-then-GWA computation as `compute`,
+/// but run via `ravanan` (https://github.com/vepiteski/ravanan) driving
+/// the CWL workflow at `workflow_path` (see `cwl/gemma/gemma-loco.cwl`)
+/// instead of this module's own direct subprocess calls -- an
+/// alternative runner for the same two gemma-wrapper steps, so the two
+/// can be compared. Backs `POST /compute/gemma/ravanan`.
+pub async fn compute_ravanan(
+    ravanan_cmd: &str,
+    workflow_path: &str,
+    cache_dir: &str,
+    genotype_file: &str,
+    snps_file: &str,
+    trait_values: &TraitValues,
+    lmm: u8,
+    run_id: Option<&str>,
+) -> Result<GemmaOutput, GemmaError> {
+    std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?;
+    let unique = format!("{}-{}", std::process::id(), nanos_since_epoch());
+    let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt"));
+    write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?;
+
+    let log_path = streaming::create_run_log(cache_dir, run_id)?;
+
+    let store_dir = Path::new(cache_dir).join("ravanan-store");
+    std::fs::create_dir_all(&store_dir).map_err(GemmaError::Io)?;
+    // ravanan does not create --output-directory itself; it must
+    // pre-exist.
+    let output_dir = Path::new(cache_dir).join(format!("{unique}-ravanan-out"));
+    std::fs::create_dir_all(&output_dir).map_err(GemmaError::Io)?;
+
+    let inputs_yaml_path = Path::new(cache_dir).join(format!("{unique}-ravanan-inputs.yaml"));
+    std::fs::write(
+        &inputs_yaml_path,
+        format!(
+            "genotype_file: {{class: File, path: {genotype_file}}}\n\
+             pheno_file: {{class: File, path: {}}}\n\
+             snps_file: {{class: File, path: {snps_file}}}\n\
+             lmm: \"{lmm}\"\n",
+            pheno_path.display(),
+        ),
+    )
+    .map_err(GemmaError::Io)?;
+
+    let store_arg = format!("--store={}", store_dir.display());
+    let outdir_arg = format!("--output-directory={}", output_dir.display());
+    let inputs_yaml_arg = inputs_yaml_path.to_string_lossy().into_owned();
+    let args = vec![
+        store_arg.as_str(),
+        outdir_arg.as_str(),
+        workflow_path,
+        inputs_yaml_arg.as_str(),
+    ];
+    let ravanan_note = format!("ravanan: {ravanan_cmd} {}", args.join(" "));
+
+    // gemma-wrapper's --loco step shells out via GNU parallel, which
+    // dispatches each per-chromosome job through $SHELL; bash (unlike
+    // sh) treats "time" as a reserved word that doesn't understand
+    // GNU time's "-v" flag, so every job fails instantly if $SHELL is
+    // (as is typical) a real bash. Force parallel to use sh instead.
+    let output = streaming::run_streaming(
+        ravanan_cmd,
+        &args,
+        log_path.as_deref(),
+        &[("PARALLEL_SHELL", "/bin/sh")],
+    )
+    .await
+    .map_err(GemmaError::Io)?;
+    if !output.status.success() {
+        return Err(GemmaError::RavananFailed(
+            String::from_utf8_lossy(&output.stderr).into_owned(),
+        ));
+    }
+    let outputs: Value = serde_json::from_str(&String::from_utf8_lossy(&output.stdout))
+        .map_err(|e| GemmaError::RavananOutput(format!("could not parse ravanan's stdout: {e}")))?;
+    let gwa_json_path = ravanan_output_path(&outputs, "gwa_json")?.to_string();
+    let archive_path = ravanan_output_path(&outputs, "archive")?.to_string();
+
+    let gwa_record: Value = serde_json::from_str(
+        &std::fs::read_to_string(&gwa_json_path).map_err(GemmaError::AssocFile)?,
+    )
+    .map_err(|e| GemmaError::WrapperError {
+        step: "gwa",
+        detail: format!("could not parse gemma-wrapper's --json output: {e}"),
+    })?;
+    let basenames = assoc_file_basenames(&gwa_record);
+    if basenames.is_empty() {
+        return Err(GemmaError::NoAssocFiles);
+    }
+
+    let extract_dir = Path::new(cache_dir).join(format!("{unique}-ravanan-extract"));
+    std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?;
+    let tar_output = streaming::run_streaming(
+        "tar",
+        &["xJf", &archive_path, "-C", &extract_dir.to_string_lossy()],
+        None,
+        &[],
+    )
+    .await
+    .map_err(GemmaError::Io)?;
+    if !tar_output.status.success() {
+        return Err(GemmaError::CommandFailed {
+            step: "extract-archive",
+            stderr: String::from_utf8_lossy(&tar_output.stderr).into_owned(),
+        });
+    }
+
+    let mut rows = Vec::new();
+    for basename in basenames {
+        let contents = std::fs::read_to_string(extract_dir.join(basename))
+            .map_err(GemmaError::AssocFile)?;
+        rows.extend(parse_assoc_file(&contents).into_iter().map(Value::Object));
+    }
+    let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless
+
+    streaming::mark_complete(&log_path);
+
+    Ok(GemmaOutput {
+        rows: Value::Array(rows),
+        notes: vec![ravanan_note, format!("workflow: {workflow_path}")],
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/lib.rs b/src/lib.rs
index f011219..a5a8729 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -83,6 +83,7 @@ pub fn app() -> Router {
         .routes(routes!(routes::search::search))
         .routes(routes!(routes::correlation::correlation))
         .routes(routes!(routes::gemma::compute_gemma))
+        .routes(routes!(routes::gemma::compute_gemma_ravanan))
         .routes(routes!(routes::gemma::gemma_status))
         .routes(routes!(routes::rqtl::compute_rqtl))
         .routes(routes!(routes::rqtl::rqtl_status));
diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs
index 0b6b748..ffa3cad 100644
--- a/src/routes/gemma.rs
+++ b/src/routes/gemma.rs
@@ -143,6 +143,94 @@ pub async fn compute_gemma(
     })
 }
 
+#[utoipa::path(post, path = "/compute/gemma/ravanan",
+    summary = "Map a QTL with GEMMA, run via ravanan instead of gemma-wrapper directly",
+    description = "Same computation and request/response shape as POST \
+/compute/gemma (gemma-wrapper's K-matrix step, then its LMM association \
+step, both --loco), but driven by ravanan \
+(https://github.com/vepiteski/ravanan) running the CWL workflow at \
+cwl/gemma/gemma-loco.cwl instead of this server's own direct subprocess \
+calls -- an alternative runner for the same two gemma-wrapper steps, \
+for comparing the two (correctness, timing). Exploratory: no caching of \
+ravanan's own store between requests beyond its --store directory, and \
+?id=/status polling tails ravanan's own stderr rather than gemma-wrapper's.",
+    params(ComputeGemmaParams),
+    request_body = GemmaRequest,
+    responses(
+        (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as a JSON array of row objects", body = Envelope<serde_json::Value>),
+        Unauthorized,
+        (status = 409, description = "id is already in use by another run", body = ApiError),
+        (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9), or id not alnum/-/_ (max 64 chars)", body = ApiError),
+        (status = 502, description = "ravanan or gemma-wrapper failed", body = ApiError)
+    ))]
+#[gnauth(token_only)]
+pub async fn compute_gemma_ravanan(
+    ctx: Ctx,
+    _token: AuthToken,
+    Query(params): Query<ComputeGemmaParams>,
+    Json(request): Json<GemmaRequest>,
+) -> Result<Json<Envelope<serde_json::Value>>, ApiError> {
+    let rid = ctx.request_info();
+    let lmm = request.lmm.unwrap_or(9);
+    if !matches!(lmm, 1 | 2 | 3 | 4 | 9) {
+        return Err(ApiError::new(
+            StatusCode::UNPROCESSABLE_ENTITY,
+            "invalid_lmm",
+            "Invalid lmm",
+            format!(
+                "lmm must be one of 1, 2, 3, 4, 9 (got {lmm}); 1=Wald, 2=LRT, 3=score, 4=all three, 9=GeneNetwork's convention (observed to behave like LRT)."
+            ),
+            "compute_gemma_ravanan",
+        )
+        .with_request_id(rid));
+    }
+    if let Some(id) = &params.id {
+        if !streaming::is_valid_run_id(id) {
+            return Err(ApiError::new(
+                StatusCode::UNPROCESSABLE_ENTITY,
+                "invalid_run_id",
+                "Invalid id",
+                "id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(),
+                "compute_gemma_ravanan",
+            )
+            .with_request_id(rid));
+        }
+    }
+
+    let config = Config::load();
+    gemma::compute_ravanan(
+        &config.ravanan_cmd,
+        &config.gemma_ravanan_workflow,
+        &config.ravanan_cache_dir,
+        &request.genotype_file,
+        &request.snps_file,
+        &request.trait_values,
+        lmm,
+        params.id.as_deref(),
+    )
+    .await
+    .map(|output| {
+        let mut envelope = ctx.envelope(output.rows).with_note(
+            "Rows come from GEMMA's own .assoc.txt output; column names depend on the lmm value used.",
+        );
+        for note in &output.notes {
+            envelope = envelope.with_note(note);
+        }
+        Json(envelope)
+    })
+    .map_err(|e| match e {
+        GemmaError::RunIdInUse(id) => ApiError::new(
+            StatusCode::CONFLICT,
+            "run_id_in_use",
+            "Run id already in use",
+            format!("id '{id}' is already in use; pick a different id."),
+            "compute_gemma_ravanan",
+        )
+        .with_request_id(rid.clone()),
+        other => gemma_error_to_api_error(other, rid.clone()),
+    })
+}
+
 #[utoipa::path(get, path = "/compute/gemma/status/{run_id}",
     summary = "Poll a running (or finished) /compute/gemma call's progress",
     description = "Returns the lines gemma-wrapper has written to stderr \
@@ -177,7 +265,16 @@ pub async fn gemma_status(
     }
     let config = Config::load();
     let peek = params.peek.unwrap_or(0);
-    streaming::status(&config.gemma_cache_dir, &run_id, peek)
+    // A run_id may belong to either runner (POST /compute/gemma or
+    // POST /compute/gemma/ravanan), which log to different cache
+    // dirs; try gemma's first, then fall back to ravanan's.
+    let result = match streaming::status(&config.gemma_cache_dir, &run_id, peek) {
+        Err(StatusError::NotFound) => {
+            streaming::status(&config.ravanan_cache_dir, &run_id, peek)
+        }
+        other => other,
+    };
+    result
         .map(|(lines, pointer)| {
             Json(ctx.envelope(RunStatus {
                 run_id: run_id.clone(),
diff --git a/src/rqtl.rs b/src/rqtl.rs
index d68f7e8..bac386d 100644
--- a/src/rqtl.rs
+++ b/src/rqtl.rs
@@ -318,7 +318,7 @@ pub async fn compute(
         &filename,
     );
     let note = format!("Rscript {}", args.join(" "));
-    let output = streaming::run_streaming("Rscript", &args, log_path.as_deref())
+    let output = streaming::run_streaming("Rscript", &args, log_path.as_deref(), &[])
         .await
         .map_err(RqtlError::Io)?;
     if !output.status.success() {
diff --git a/src/streaming.rs b/src/streaming.rs
index 90ce0f4..b8e6c03 100644
--- a/src/streaming.rs
+++ b/src/streaming.rs
@@ -111,9 +111,15 @@ pub fn mark_complete(log_path: &Option<PathBuf>) {
 /// tailing stderr here is a deliberate, uniform choice across both
 /// tools' call sites, not a guarantee every line of every tool's
 /// progress ends up in the log.
-pub async fn run_streaming(cmd: &str, args: &[&str], log_path: Option<&Path>) -> io::Result<Output> {
+pub async fn run_streaming(
+    cmd: &str,
+    args: &[&str],
+    log_path: Option<&Path>,
+    extra_env: &[(&str, &str)],
+) -> io::Result<Output> {
     let mut child = Command::new(cmd)
         .args(args)
+        .envs(extra_env.iter().copied())
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
         .spawn()?;