diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-15 13:40:55 +0200 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-15 13:40:55 +0200 |
| commit | ba9016c729c020a62cb2b4481214a19f5e027ba6 (patch) | |
| tree | 3f8631fba703dfc72d03ae1df49fda9bf353e2fe /src | |
| parent | b3c43a2bb0cb7f1e5d0c0a3d08f76f82d9ef881f (diff) | |
| download | genenetwork4-ba9016c729c020a62cb2b4481214a19f5e027ba6.tar.gz | |
CWL used to run gemma
Diffstat (limited to 'src')
| -rw-r--r-- | src/config.rs | 39 | ||||
| -rw-r--r-- | src/gemma.rs | 149 | ||||
| -rw-r--r-- | src/lib.rs | 1 | ||||
| -rw-r--r-- | src/routes/gemma.rs | 99 | ||||
| -rw-r--r-- | src/rqtl.rs | 2 | ||||
| -rw-r--r-- | src/streaming.rs | 8 |
6 files changed, 293 insertions, 5 deletions
diff --git a/src/config.rs b/src/config.rs index 5030576..f0fe946 100644 --- a/src/config.rs +++ b/src/config.rs @@ -68,6 +68,21 @@ pub struct Config { /// results between calls (its own --cache-dir switch). Defaults to /// a "gn4-gemma" directory under the system temp dir. pub gemma_cache_dir: String, + /// The ravanan executable to invoke for POST /compute/gemma/ravanan + /// (see src/gemma.rs's compute_ravanan). Defaults to "ravanan", + /// resolved via PATH. + pub ravanan_cmd: String, + /// Path to the CWL workflow POST /compute/gemma/ravanan runs under + /// ravanan. Defaults to that path relative to the working + /// directory the server is started from, same convention as + /// `rqtl_wrapper_path`. + pub gemma_ravanan_workflow: String, + /// Directory POST /compute/gemma/ravanan uses for phenotype files, + /// run logs, and ravanan's own --store and --output-directory. + /// Separate from `gemma_cache_dir` so the two runners don't share + /// state. Defaults to a "gn4-gemma-ravanan" directory under the + /// system temp dir. + pub ravanan_cache_dir: String, /// Path to scripts/rqtl_wrapper.R (copied here from GeneNetwork3) /// for POST /compute/rqtl. Defaults to that path relative to the /// working directory the server is started from. @@ -127,6 +142,18 @@ impl Config { .to_string_lossy() .into_owned(), ); + values.insert("RAVANAN_CMD".to_string(), "ravanan".to_string()); + values.insert( + "GEMMA_RAVANAN_WORKFLOW".to_string(), + "cwl/gemma/gemma-loco.cwl".to_string(), + ); + values.insert( + "RAVANAN_CACHE_DIR".to_string(), + std::env::temp_dir() + .join("gn4-gemma-ravanan") + .to_string_lossy() + .into_owned(), + ); values.insert( "RQTL_WRAPPER_PATH".to_string(), "scripts/rqtl_wrapper.R".to_string(), @@ -165,6 +192,9 @@ impl Config { "SEARCH_ENGINE", "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", + "RAVANAN_CMD", + "GEMMA_RAVANAN_WORKFLOW", + "RAVANAN_CACHE_DIR", "RQTL_WRAPPER_PATH", "RQTL_CACHE_DIR", ] { @@ -195,6 +225,9 @@ impl Config { dev_token_password: values["GN_AUTH_DEV_PASSWORD"].clone(), gemma_wrapper_cmd: values["GEMMA_WRAPPER_CMD"].clone(), gemma_cache_dir: values["GEMMA_CACHE_DIR"].clone(), + ravanan_cmd: values["RAVANAN_CMD"].clone(), + gemma_ravanan_workflow: values["GEMMA_RAVANAN_WORKFLOW"].clone(), + ravanan_cache_dir: values["RAVANAN_CACHE_DIR"].clone(), rqtl_wrapper_path: values["RQTL_WRAPPER_PATH"].clone(), rqtl_cache_dir: values["RQTL_CACHE_DIR"].clone(), } @@ -269,6 +302,9 @@ mod tests { "SEARCH_ENGINE", "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", + "RAVANAN_CMD", + "GEMMA_RAVANAN_WORKFLOW", + "RAVANAN_CACHE_DIR", "RQTL_WRAPPER_PATH", "RQTL_CACHE_DIR", ] { @@ -282,6 +318,9 @@ mod tests { assert_eq!(config.port, 3000); assert_eq!(config.gemma_wrapper_cmd, "gemma-wrapper"); assert!(config.gemma_cache_dir.ends_with("gn4-gemma")); + assert_eq!(config.ravanan_cmd, "ravanan"); + assert_eq!(config.gemma_ravanan_workflow, "cwl/gemma/gemma-loco.cwl"); + assert!(config.ravanan_cache_dir.ends_with("gn4-gemma-ravanan")); assert_eq!(config.rqtl_wrapper_path, "scripts/rqtl_wrapper.R"); assert!(config.rqtl_cache_dir.ends_with("gn4-rqtl")); assert_eq!(config.auth_server_url, "http://127.0.0.1:8081/"); diff --git a/src/gemma.rs b/src/gemma.rs index 9563131..b0b27a2 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -43,6 +43,12 @@ pub enum GemmaError { /// before any subprocess runs, so this never silently truncates /// another run's in-progress log. RunIdInUse(String), + /// `ravanan` (see `compute_ravanan`) exited non-zero; carries its + /// stderr. + RavananFailed(String), + /// `ravanan`'s own stdout (the CWL outputs record) was not valid + /// JSON, or didn't carry the output this code expects. + RavananOutput(String), } impl std::fmt::Display for GemmaError { @@ -62,6 +68,10 @@ impl std::fmt::Display for GemmaError { GemmaError::RunIdInUse(id) => { write!(f, "run id '{id}' is already in use; pick a different id") } + GemmaError::RavananFailed(stderr) => write!(f, "ravanan failed: {stderr}"), + GemmaError::RavananOutput(detail) => { + write!(f, "could not read ravanan's output: {detail}") + } } } } @@ -139,7 +149,7 @@ async fn compute_kinship( log_path: Option<&Path>, ) -> Result<Value, GemmaError> { let args = kinship_args(cache_dir, genotype_file, pheno_file, snps_file); - let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path) + let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path, &[]) .await .map_err(GemmaError::Io)?; parse_wrapper_output("kinship", output) @@ -182,7 +192,7 @@ async fn compute_gwa( &lmm_str, k_json_path, ); - let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path) + let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path, &[]) .await .map_err(GemmaError::Io)?; parse_wrapper_output("gwa", output) @@ -250,6 +260,7 @@ async fn extract_archive(cache_dir: &str, archive: &str, dest_dir: &str) -> Resu "tar", &["xJf", &archive_path.to_string_lossy(), "-C", dest_dir], None, + &[], ) .await .map_err(GemmaError::Io)?; @@ -432,6 +443,140 @@ pub async fn compute( }) } +/// Read a `File` output's `path` from ravanan's stdout (the CWL +/// outputs record: `{"<output id>": {"class": "File", "path": ..., ...}, ...}`). +fn ravanan_output_path<'a>(outputs: &'a Value, id: &str) -> Result<&'a str, GemmaError> { + outputs + .get(id) + .and_then(|v| v.get("path")) + .and_then(Value::as_str) + .ok_or_else(|| { + GemmaError::RavananOutput(format!("no \"{id}\".path in ravanan's output")) + }) +} + +/// The same GEMMA `--loco` kinship-then-GWA computation as `compute`, +/// but run via `ravanan` (https://github.com/vepiteski/ravanan) driving +/// the CWL workflow at `workflow_path` (see `cwl/gemma/gemma-loco.cwl`) +/// instead of this module's own direct subprocess calls -- an +/// alternative runner for the same two gemma-wrapper steps, so the two +/// can be compared. Backs `POST /compute/gemma/ravanan`. +pub async fn compute_ravanan( + ravanan_cmd: &str, + workflow_path: &str, + cache_dir: &str, + genotype_file: &str, + snps_file: &str, + trait_values: &TraitValues, + lmm: u8, + run_id: Option<&str>, +) -> Result<GemmaOutput, GemmaError> { + std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?; + let unique = format!("{}-{}", std::process::id(), nanos_since_epoch()); + let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt")); + write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?; + + let log_path = streaming::create_run_log(cache_dir, run_id)?; + + let store_dir = Path::new(cache_dir).join("ravanan-store"); + std::fs::create_dir_all(&store_dir).map_err(GemmaError::Io)?; + // ravanan does not create --output-directory itself; it must + // pre-exist. + let output_dir = Path::new(cache_dir).join(format!("{unique}-ravanan-out")); + std::fs::create_dir_all(&output_dir).map_err(GemmaError::Io)?; + + let inputs_yaml_path = Path::new(cache_dir).join(format!("{unique}-ravanan-inputs.yaml")); + std::fs::write( + &inputs_yaml_path, + format!( + "genotype_file: {{class: File, path: {genotype_file}}}\n\ + pheno_file: {{class: File, path: {}}}\n\ + snps_file: {{class: File, path: {snps_file}}}\n\ + lmm: \"{lmm}\"\n", + pheno_path.display(), + ), + ) + .map_err(GemmaError::Io)?; + + let store_arg = format!("--store={}", store_dir.display()); + let outdir_arg = format!("--output-directory={}", output_dir.display()); + let inputs_yaml_arg = inputs_yaml_path.to_string_lossy().into_owned(); + let args = vec![ + store_arg.as_str(), + outdir_arg.as_str(), + workflow_path, + inputs_yaml_arg.as_str(), + ]; + let ravanan_note = format!("ravanan: {ravanan_cmd} {}", args.join(" ")); + + // gemma-wrapper's --loco step shells out via GNU parallel, which + // dispatches each per-chromosome job through $SHELL; bash (unlike + // sh) treats "time" as a reserved word that doesn't understand + // GNU time's "-v" flag, so every job fails instantly if $SHELL is + // (as is typical) a real bash. Force parallel to use sh instead. + let output = streaming::run_streaming( + ravanan_cmd, + &args, + log_path.as_deref(), + &[("PARALLEL_SHELL", "/bin/sh")], + ) + .await + .map_err(GemmaError::Io)?; + if !output.status.success() { + return Err(GemmaError::RavananFailed( + String::from_utf8_lossy(&output.stderr).into_owned(), + )); + } + let outputs: Value = serde_json::from_str(&String::from_utf8_lossy(&output.stdout)) + .map_err(|e| GemmaError::RavananOutput(format!("could not parse ravanan's stdout: {e}")))?; + let gwa_json_path = ravanan_output_path(&outputs, "gwa_json")?.to_string(); + let archive_path = ravanan_output_path(&outputs, "archive")?.to_string(); + + let gwa_record: Value = serde_json::from_str( + &std::fs::read_to_string(&gwa_json_path).map_err(GemmaError::AssocFile)?, + ) + .map_err(|e| GemmaError::WrapperError { + step: "gwa", + detail: format!("could not parse gemma-wrapper's --json output: {e}"), + })?; + let basenames = assoc_file_basenames(&gwa_record); + if basenames.is_empty() { + return Err(GemmaError::NoAssocFiles); + } + + let extract_dir = Path::new(cache_dir).join(format!("{unique}-ravanan-extract")); + std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?; + let tar_output = streaming::run_streaming( + "tar", + &["xJf", &archive_path, "-C", &extract_dir.to_string_lossy()], + None, + &[], + ) + .await + .map_err(GemmaError::Io)?; + if !tar_output.status.success() { + return Err(GemmaError::CommandFailed { + step: "extract-archive", + stderr: String::from_utf8_lossy(&tar_output.stderr).into_owned(), + }); + } + + let mut rows = Vec::new(); + for basename in basenames { + let contents = std::fs::read_to_string(extract_dir.join(basename)) + .map_err(GemmaError::AssocFile)?; + rows.extend(parse_assoc_file(&contents).into_iter().map(Value::Object)); + } + let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless + + streaming::mark_complete(&log_path); + + Ok(GemmaOutput { + rows: Value::Array(rows), + notes: vec![ravanan_note, format!("workflow: {workflow_path}")], + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/lib.rs b/src/lib.rs index f011219..a5a8729 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -83,6 +83,7 @@ pub fn app() -> Router { .routes(routes!(routes::search::search)) .routes(routes!(routes::correlation::correlation)) .routes(routes!(routes::gemma::compute_gemma)) + .routes(routes!(routes::gemma::compute_gemma_ravanan)) .routes(routes!(routes::gemma::gemma_status)) .routes(routes!(routes::rqtl::compute_rqtl)) .routes(routes!(routes::rqtl::rqtl_status)); diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs index 0b6b748..ffa3cad 100644 --- a/src/routes/gemma.rs +++ b/src/routes/gemma.rs @@ -143,6 +143,94 @@ pub async fn compute_gemma( }) } +#[utoipa::path(post, path = "/compute/gemma/ravanan", + summary = "Map a QTL with GEMMA, run via ravanan instead of gemma-wrapper directly", + description = "Same computation and request/response shape as POST \ +/compute/gemma (gemma-wrapper's K-matrix step, then its LMM association \ +step, both --loco), but driven by ravanan \ +(https://github.com/vepiteski/ravanan) running the CWL workflow at \ +cwl/gemma/gemma-loco.cwl instead of this server's own direct subprocess \ +calls -- an alternative runner for the same two gemma-wrapper steps, \ +for comparing the two (correctness, timing). Exploratory: no caching of \ +ravanan's own store between requests beyond its --store directory, and \ +?id=/status polling tails ravanan's own stderr rather than gemma-wrapper's.", + params(ComputeGemmaParams), + request_body = GemmaRequest, + responses( + (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as a JSON array of row objects", body = Envelope<serde_json::Value>), + Unauthorized, + (status = 409, description = "id is already in use by another run", body = ApiError), + (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9), or id not alnum/-/_ (max 64 chars)", body = ApiError), + (status = 502, description = "ravanan or gemma-wrapper failed", body = ApiError) + ))] +#[gnauth(token_only)] +pub async fn compute_gemma_ravanan( + ctx: Ctx, + _token: AuthToken, + Query(params): Query<ComputeGemmaParams>, + Json(request): Json<GemmaRequest>, +) -> Result<Json<Envelope<serde_json::Value>>, ApiError> { + let rid = ctx.request_info(); + let lmm = request.lmm.unwrap_or(9); + if !matches!(lmm, 1 | 2 | 3 | 4 | 9) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_lmm", + "Invalid lmm", + format!( + "lmm must be one of 1, 2, 3, 4, 9 (got {lmm}); 1=Wald, 2=LRT, 3=score, 4=all three, 9=GeneNetwork's convention (observed to behave like LRT)." + ), + "compute_gemma_ravanan", + ) + .with_request_id(rid)); + } + if let Some(id) = ¶ms.id { + if !streaming::is_valid_run_id(id) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_run_id", + "Invalid id", + "id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), + "compute_gemma_ravanan", + ) + .with_request_id(rid)); + } + } + + let config = Config::load(); + gemma::compute_ravanan( + &config.ravanan_cmd, + &config.gemma_ravanan_workflow, + &config.ravanan_cache_dir, + &request.genotype_file, + &request.snps_file, + &request.trait_values, + lmm, + params.id.as_deref(), + ) + .await + .map(|output| { + let mut envelope = ctx.envelope(output.rows).with_note( + "Rows come from GEMMA's own .assoc.txt output; column names depend on the lmm value used.", + ); + for note in &output.notes { + envelope = envelope.with_note(note); + } + Json(envelope) + }) + .map_err(|e| match e { + GemmaError::RunIdInUse(id) => ApiError::new( + StatusCode::CONFLICT, + "run_id_in_use", + "Run id already in use", + format!("id '{id}' is already in use; pick a different id."), + "compute_gemma_ravanan", + ) + .with_request_id(rid.clone()), + other => gemma_error_to_api_error(other, rid.clone()), + }) +} + #[utoipa::path(get, path = "/compute/gemma/status/{run_id}", summary = "Poll a running (or finished) /compute/gemma call's progress", description = "Returns the lines gemma-wrapper has written to stderr \ @@ -177,7 +265,16 @@ pub async fn gemma_status( } let config = Config::load(); let peek = params.peek.unwrap_or(0); - streaming::status(&config.gemma_cache_dir, &run_id, peek) + // A run_id may belong to either runner (POST /compute/gemma or + // POST /compute/gemma/ravanan), which log to different cache + // dirs; try gemma's first, then fall back to ravanan's. + let result = match streaming::status(&config.gemma_cache_dir, &run_id, peek) { + Err(StatusError::NotFound) => { + streaming::status(&config.ravanan_cache_dir, &run_id, peek) + } + other => other, + }; + result .map(|(lines, pointer)| { Json(ctx.envelope(RunStatus { run_id: run_id.clone(), diff --git a/src/rqtl.rs b/src/rqtl.rs index d68f7e8..bac386d 100644 --- a/src/rqtl.rs +++ b/src/rqtl.rs @@ -318,7 +318,7 @@ pub async fn compute( &filename, ); let note = format!("Rscript {}", args.join(" ")); - let output = streaming::run_streaming("Rscript", &args, log_path.as_deref()) + let output = streaming::run_streaming("Rscript", &args, log_path.as_deref(), &[]) .await .map_err(RqtlError::Io)?; if !output.status.success() { diff --git a/src/streaming.rs b/src/streaming.rs index 90ce0f4..b8e6c03 100644 --- a/src/streaming.rs +++ b/src/streaming.rs @@ -111,9 +111,15 @@ pub fn mark_complete(log_path: &Option<PathBuf>) { /// tailing stderr here is a deliberate, uniform choice across both /// tools' call sites, not a guarantee every line of every tool's /// progress ends up in the log. -pub async fn run_streaming(cmd: &str, args: &[&str], log_path: Option<&Path>) -> io::Result<Output> { +pub async fn run_streaming( + cmd: &str, + args: &[&str], + log_path: Option<&Path>, + extra_env: &[(&str, &str)], +) -> io::Result<Output> { let mut child = Command::new(cmd) .args(args) + .envs(extra_env.iter().copied()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn()?; |
