diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-19 10:50:05 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-19 10:50:05 +0000 |
| commit | cda325b6334ef36e3f49501cfd77d68a7bd6a295 (patch) | |
| tree | 56003d3b3cccb1cf4f12e36bb6065844ba5f2219 /src | |
| parent | 15c735742f35e4688b6448d3c7e6922c78ec0746 (diff) | |
| parent | d504b5da9f5deeece9ba2ff4dcd8baf3dfa4dc32 (diff) | |
| download | genenetwork4-cda325b6334ef36e3f49501cfd77d68a7bd6a295.tar.gz | |
Merge branch 'gemma-cwl-parallel'
# Conflicts: # src/gemma.rs # src/rqtl.rs # src/streaming.rs
Diffstat (limited to 'src')
| -rw-r--r-- | src/config.rs | 65 | ||||
| -rw-r--r-- | src/gemma.rs | 355 | ||||
| -rw-r--r-- | src/lib.rs | 1 | ||||
| -rw-r--r-- | src/response.rs | 28 | ||||
| -rw-r--r-- | src/routes/gemma.rs | 117 | ||||
| -rw-r--r-- | src/routes/rqtl.rs | 12 | ||||
| -rw-r--r-- | src/rqtl.rs | 27 |
7 files changed, 556 insertions, 49 deletions
diff --git a/src/config.rs b/src/config.rs index 7d8893c..47cb4bf 100644 --- a/src/config.rs +++ b/src/config.rs @@ -74,6 +74,34 @@ pub struct Config { /// results between calls (its own --cache-dir switch). Defaults to /// a "gn4-gemma" directory under the system temp dir. pub gemma_cache_dir: String, + /// The ravanan executable to invoke for POST /compute/gemma/ravanan + /// (see src/gemma.rs's compute_ravanan). Defaults to "ravanan", + /// resolved via PATH. + pub ravanan_cmd: String, + /// The ccwl (https://ccwl.systemreboot.net/) executable used to + /// compile `gemma_ravanan_workflow` to CWL before each ravanan run + /// (see src/gemma.rs's compute_ravanan). Defaults to "ccwl", + /// resolved via PATH. + pub ccwl_cmd: String, + /// Path to the ccwl source POST /compute/gemma/ravanan compiles to + /// CWL and runs under ravanan. Not a CWL file itself -- the + /// compiled .cwl is a generated, throwaway artifact (written under + /// `ravanan_cache_dir` on every call), never checked into the repo. + /// Defaults to that path relative to the working directory the + /// server is started from, same convention as `rqtl_wrapper_path`. + pub gemma_ravanan_workflow: String, + /// Path to the Guix manifest ravanan resolves its jobs' commands + /// against (its own --guix-manifest switch, required -- ravanan + /// refuses to run without one). Defaults to that path relative to + /// the working directory the server is started from, same + /// convention as `gemma_ravanan_workflow`. + pub gemma_ravanan_manifest: String, + /// Directory POST /compute/gemma/ravanan uses for phenotype files, + /// run logs, and ravanan's own --store and --output-directory. + /// Separate from `gemma_cache_dir` so the two runners don't share + /// state. Defaults to a "gn4-gemma-ravanan" directory under the + /// system temp dir. + pub ravanan_cache_dir: String, /// Path to scripts/rqtl_wrapper.R (copied here from GeneNetwork3) /// for POST /compute/rqtl. Defaults to that path relative to the /// working directory the server is started from. @@ -136,6 +164,23 @@ impl Config { .to_string_lossy() .into_owned(), ); + values.insert("RAVANAN_CMD".to_string(), "ravanan".to_string()); + values.insert("CCWL_CMD".to_string(), "ccwl".to_string()); + values.insert( + "GEMMA_RAVANAN_WORKFLOW".to_string(), + "cwl/gemma/gemma-loco.scm".to_string(), + ); + values.insert( + "GEMMA_RAVANAN_MANIFEST".to_string(), + "cwl/gemma/manifest.scm".to_string(), + ); + values.insert( + "RAVANAN_CACHE_DIR".to_string(), + std::env::temp_dir() + .join("gn4-gemma-ravanan") + .to_string_lossy() + .into_owned(), + ); values.insert( "RQTL_WRAPPER_PATH".to_string(), "scripts/rqtl_wrapper.R".to_string(), @@ -175,6 +220,11 @@ impl Config { "SEARCH_ENGINE", "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", + "RAVANAN_CMD", + "CCWL_CMD", + "GEMMA_RAVANAN_WORKFLOW", + "GEMMA_RAVANAN_MANIFEST", + "RAVANAN_CACHE_DIR", "RQTL_WRAPPER_PATH", "RQTL_CACHE_DIR", ] { @@ -206,6 +256,11 @@ impl Config { dev_token_password: values["GN_AUTH_DEV_PASSWORD"].clone(), gemma_wrapper_cmd: values["GEMMA_WRAPPER_CMD"].clone(), gemma_cache_dir: values["GEMMA_CACHE_DIR"].clone(), + ravanan_cmd: values["RAVANAN_CMD"].clone(), + ccwl_cmd: values["CCWL_CMD"].clone(), + gemma_ravanan_workflow: values["GEMMA_RAVANAN_WORKFLOW"].clone(), + gemma_ravanan_manifest: values["GEMMA_RAVANAN_MANIFEST"].clone(), + ravanan_cache_dir: values["RAVANAN_CACHE_DIR"].clone(), rqtl_wrapper_path: values["RQTL_WRAPPER_PATH"].clone(), rqtl_cache_dir: values["RQTL_CACHE_DIR"].clone(), } @@ -306,6 +361,11 @@ mod tests { "SEARCH_ENGINE", "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", + "RAVANAN_CMD", + "CCWL_CMD", + "GEMMA_RAVANAN_WORKFLOW", + "GEMMA_RAVANAN_MANIFEST", + "RAVANAN_CACHE_DIR", "RQTL_WRAPPER_PATH", "RQTL_CACHE_DIR", ] { @@ -324,6 +384,11 @@ mod tests { assert_eq!(config.port, 3000); assert_eq!(config.gemma_wrapper_cmd, "gemma-wrapper"); assert!(config.gemma_cache_dir.ends_with("gn4-gemma")); + assert_eq!(config.ravanan_cmd, "ravanan"); + assert_eq!(config.ccwl_cmd, "ccwl"); + assert_eq!(config.gemma_ravanan_workflow, "cwl/gemma/gemma-loco.scm"); + assert_eq!(config.gemma_ravanan_manifest, "cwl/gemma/manifest.scm"); + assert!(config.ravanan_cache_dir.ends_with("gn4-gemma-ravanan")); assert_eq!(config.rqtl_wrapper_path, "scripts/rqtl_wrapper.R"); assert!(config.rqtl_cache_dir.ends_with("gn4-rqtl")); assert_eq!(config.auth_server_url, "http://127.0.0.1:8081/"); diff --git a/src/gemma.rs b/src/gemma.rs index 2c7f08e..af5113a 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -43,6 +43,12 @@ pub enum GemmaError { /// before any subprocess runs, so this never silently truncates /// another run's in-progress log. RunIdInUse(String), + /// `ravanan` (see `compute_ravanan`) exited non-zero; carries its + /// stderr. + RavananFailed(String), + /// `ravanan`'s own stdout (the CWL outputs record) was not valid + /// JSON, or didn't carry the output this code expects. + RavananOutput(String), } impl std::fmt::Display for GemmaError { @@ -65,6 +71,10 @@ impl std::fmt::Display for GemmaError { GemmaError::RunIdInUse(id) => { write!(f, "run id '{id}' is already in use; pick a different id") } + GemmaError::RavananFailed(stderr) => write!(f, "ravanan failed: {stderr}"), + GemmaError::RavananOutput(detail) => { + write!(f, "could not read ravanan's output: {detail}") + } } } } @@ -208,36 +218,34 @@ fn assoc_file_basenames(record: &Value) -> Vec<&str> { .unwrap_or_default() } -/// One row of a GEMMA `.assoc.txt` file, keyed by its header column -/// names. Columns vary with `-lmm` (stock 1/2/3 run one test, 4 runs -/// all three; this build's `9`, GeneNetwork's convention, was observed -/// to carry `p_lrt`), so this stays a generic map rather than a fixed -/// struct. -pub type AssocRow = serde_json::Map<String, Value>; - /// Parse a GEMMA `.assoc.txt` file (tab-separated, header row first) -/// into one row per marker. A value that parses as a number is kept -/// numeric; "NA" and anything else stays a string. -fn parse_assoc_file(contents: &str) -> Vec<AssocRow> { +/// into its column names and one row per marker, each row a +/// `Value::Array` in that same column order (not keyed by column name: +/// columns vary with `-lmm` -- stock 1/2/3 run one test, 4 runs all +/// three; this build's `9`, GeneNetwork's convention, was observed to +/// carry `p_lrt` -- so callers needing a name match it up against the +/// returned column list themselves). A value that parses as a number is +/// kept numeric; "NA" and anything else stays a string. +fn parse_assoc_file(contents: &str) -> (Vec<String>, Vec<Vec<Value>>) { let mut lines = contents.lines(); let Some(header) = lines.next() else { - return Vec::new(); + return (Vec::new(), Vec::new()); }; - let columns: Vec<&str> = header.split('\t').collect(); - lines + let columns: Vec<String> = header.split('\t').map(str::to_string).collect(); + let rows = lines .filter(|line| !line.is_empty()) .map(|line| { - let mut row = AssocRow::new(); - for (column, value) in columns.iter().zip(line.split('\t')) { - let json_value = value - .parse::<f64>() - .map(|n| serde_json::json!(n)) - .unwrap_or_else(|_| Value::String(value.to_string())); - row.insert((*column).to_string(), json_value); - } - row + line.split('\t') + .map(|value| { + value + .parse::<f64>() + .map(|n| serde_json::json!(n)) + .unwrap_or_else(|_| Value::String(value.to_string())) + }) + .collect() }) - .collect() + .collect(); + (columns, rows) } /// Extract `archive` (a `.tar.xz`, relative to `cache_dir`) into @@ -265,6 +273,25 @@ async fn extract_archive(cache_dir: &str, archive: &str, dest_dir: &str) -> Resu Ok(()) } +/// Read and parse every `basenames` file under `extract_dir` (one per +/// `--loco` chromosome), returning the column names (from the first +/// file; every chromosome's `.assoc.txt` shares the same header) and +/// every file's rows concatenated, each row a `Value::Array`. +fn parse_assoc_files(extract_dir: &Path, basenames: Vec<&str>) -> Result<(Vec<String>, Vec<Value>), GemmaError> { + let mut columns = Vec::new(); + let mut rows = Vec::new(); + for basename in basenames { + let contents = std::fs::read_to_string(extract_dir.join(basename)) + .map_err(GemmaError::AssocFile)?; + let (file_columns, file_rows) = parse_assoc_file(&contents); + if columns.is_empty() { + columns = file_columns; + } + rows.extend(file_rows.into_iter().map(Value::Array)); + } + Ok((columns, rows)) +} + fn nanos_since_epoch() -> u128 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -272,10 +299,13 @@ fn nanos_since_epoch() -> u128 { .unwrap_or(0) } -/// The mapped QTL rows, plus the exact gemma-wrapper invocations used -/// to produce them (see `compute`'s `notes` below), so a caller can see -/// precisely what was run without reading server logs. +/// The mapped QTL rows (each a `Value::Array` in `columns`' order, not +/// a keyed object -- see `parse_assoc_file`), plus the exact +/// gemma-wrapper invocations used to produce them (see `compute`'s +/// `notes` below), so a caller can see precisely what was run without +/// reading server logs. pub struct GemmaOutput { + pub columns: Vec<String>, pub rows: Value, pub notes: Vec<String>, } @@ -425,15 +455,11 @@ pub async fn compute( std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?; extract_archive(cache_dir, archive, &extract_dir.to_string_lossy()).await?; - let mut rows = Vec::new(); - for basename in basenames { - let contents = std::fs::read_to_string(extract_dir.join(basename)) - .map_err(GemmaError::AssocFile)?; - rows.extend(parse_assoc_file(&contents).into_iter().map(Value::Object)); - } + let (columns, rows) = parse_assoc_files(&extract_dir, basenames)?; let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless Ok(GemmaOutput { + columns, rows: Value::Array(rows), notes: vec![kinship_note, gwa_note], }) @@ -452,6 +478,253 @@ pub async fn compute( } } +/// Read a scattered `File[]` output's list of `path`s from ravanan's +/// stdout (the CWL outputs record for a scattered output: +/// `{"<output id>": [{"class": "File", "path": ..., ...}, ...], ...}`, +/// one entry per `cwl/gemma/gemma-loco.scm` chromosome). +fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str>, GemmaError> { + let files = outputs + .get(id) + .and_then(Value::as_array) + .ok_or_else(|| GemmaError::RavananOutput(format!("no \"{id}\" array in ravanan's output")))?; + files + .iter() + .map(|file| { + file.get("path").and_then(Value::as_str).ok_or_else(|| { + GemmaError::RavananOutput(format!("no .path on one of \"{id}\"'s entries")) + }) + }) + .collect() +} + +/// The distinct chromosome names in `snps_file` (whitespace-separated: +/// rs id, position, chromosome), in first-seen order -- reproduces +/// gemma-wrapper's own `get_chromosomes`, used here to build +/// `compute_ravanan`'s `chromosomes` scatter input (CWL's `scatter` +/// needs a concrete array at invocation time, which can't be +/// discovered by a step inside the workflow itself without another +/// round of plumbing -- see `cwl/gemma/gemma-loco.scm`'s own doc +/// comment on this input). +fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> { + let contents = std::fs::read_to_string(snps_file).map_err(GemmaError::Io)?; + let mut seen = std::collections::HashSet::new(); + let mut chromosomes = Vec::new(); + for line in contents.lines() { + if let Some(chr) = line.split_whitespace().nth(2) { + if seen.insert(chr.to_string()) { + chromosomes.push(chr.to_string()); + } + } + } + Ok(chromosomes) +} + +/// The same GEMMA `--loco` kinship-then-GWA computation as `compute`, +/// but run via `ravanan` (https://github.com/vepiteski/ravanan) driving +/// a CWL workflow compiled from the ccwl (https://ccwl.systemreboot.net/) +/// source at `workflow_scm_path` (see `cwl/gemma/gemma-loco.scm`) +/// instead of this module's own direct subprocess calls: one +/// gemma-wrapper invocation per chromosome, scattered by ravanan +/// itself (`--chromosomes`/`--no-parallel`, see +/// `cwl/gemma/gemma-loco.scm`'s `kinship` and `gwa` commands) rather +/// than gemma-wrapper's own internal GNU parallel fan-out -- an alternative +/// runner for the same computation, so the two can be compared. +/// Backs `POST /compute/gemma/ravanan`. +pub async fn compute_ravanan( + ravanan_cmd: &str, + ccwl_cmd: &str, + workflow_scm_path: &str, + manifest_path: &str, + cache_dir: &str, + genotype_file: &str, + snps_file: &str, + trait_values: &TraitValues, + lmm: u8, + run_id: Option<&str>, +) -> Result<GemmaOutput, GemmaError> { + std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?; + let unique = format!("{}-{}", std::process::id(), nanos_since_epoch()); + let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt")); + write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?; + + let log_path = streaming::create_run_log(cache_dir, run_id)?; + + // gemma-loco.scm is the maintained source; the CWL ravanan actually + // runs is a generated, throwaway artifact compiled fresh on every + // call (cheap: ccwl compile is pure macro expansion, no Guix build + // involved) and never checked into the repo -- see the doc comment + // on Config::gemma_ravanan_workflow. + let compiled_workflow_path = + Path::new(cache_dir).join(format!("{unique}-gemma-loco.cwl")); + let ccwl_output_arg = format!("--output={}", compiled_workflow_path.display()); + let ccwl_args = vec!["compile", ccwl_output_arg.as_str(), workflow_scm_path]; + let ccwl_output = streaming::run_streaming(ccwl_cmd, &ccwl_args, None) + .await + .map_err(GemmaError::Io)?; + if !ccwl_output.status.success() { + return Err(GemmaError::CommandFailed { + step: "ccwl compile", + stderr: String::from_utf8_lossy(&ccwl_output.stderr).into_owned(), + }); + } + let compiled_workflow_arg = compiled_workflow_path.to_string_lossy().into_owned(); + + let store_dir = Path::new(cache_dir).join("ravanan-store"); + // ravanan's --store needs these three subdirectories to already + // exist; it does not create them itself. + for subdir in ["files", "logs", "data"] { + std::fs::create_dir_all(store_dir.join(subdir)).map_err(GemmaError::Io)?; + } + + let chromosomes = chromosomes_in(snps_file)?; + if chromosomes.is_empty() { + return Err(GemmaError::NoAssocFiles); + } + let chromosomes_yaml = chromosomes + .iter() + .map(|c| format!("\"{c}\"")) + .collect::<Vec<_>>() + .join(", "); + + // One persistent cache directory per chromosome, reused across + // requests, so gemma-wrapper's content-hash cache actually gets hit + // on repeated identical requests -- unlike "." (each job's own + // hermetic CWL working directory under ravanan's --store, fresh + // every run), which never accumulates a cache at all. Keyed by + // chromosome, not shared, because gemma-wrapper's cache-hit hash + // does not vary with --chromosomes: a shared directory would let + // one chromosome's job silently reuse another's cached result. + let wrapper_cache_dir = Path::new(cache_dir).join("gemma-wrapper-cache"); + let mut chr_cache_dirs = Vec::new(); + for chr in &chromosomes { + let chr_cache_dir = wrapper_cache_dir.join(format!("chr-{chr}")); + std::fs::create_dir_all(&chr_cache_dir).map_err(GemmaError::Io)?; + chr_cache_dirs.push(chr_cache_dir); + } + let cache_dirs_yaml = chr_cache_dirs + .iter() + .map(|d| format!("\"{}\"", d.display())) + .collect::<Vec<_>>() + .join(", "); + + let inputs_yaml_path = Path::new(cache_dir).join(format!("{unique}-ravanan-inputs.yaml")); + std::fs::write( + &inputs_yaml_path, + format!( + "genotype_file: {{class: File, path: {genotype_file}}}\n\ + pheno_file: {{class: File, path: {}}}\n\ + snps_file: {{class: File, path: {snps_file}}}\n\ + lmm: \"{lmm}\"\n\ + chromosomes: [{chromosomes_yaml}]\n\ + cache_dirs: [{cache_dirs_yaml}]\n", + pheno_path.display(), + ), + ) + .map_err(GemmaError::Io)?; + + let store_arg = format!("--store={}", store_dir.display()); + let manifest_arg = format!("--guix-manifest={manifest_path}"); + let inputs_yaml_arg = inputs_yaml_path.to_string_lossy().into_owned(); + // No --output-directory: the gwa step's per-chromosome archive + // outputs all share gemma-wrapper's own content-hash basename (its + // hash doesn't vary with --chromosomes), and ravanan's + // --output-directory step symlinks every scattered output into one + // flat directory by basename, which collides across chromosomes -- + // a ravanan/gemma-wrapper interaction bug, not something fixable + // on either side alone. Harmless to skip: it's optional, and this + // code reads paths straight out of ravanan's own --store via its + // stdout output record, never through --output-directory. + let args = vec![ + store_arg.as_str(), + manifest_arg.as_str(), + compiled_workflow_arg.as_str(), + inputs_yaml_arg.as_str(), + ]; + let ravanan_note = format!("ravanan: {ravanan_cmd} {}", args.join(" ")); + let chromosomes_note = format!( + "Scattered gemma-wrapper over {} chromosome(s) via CWL scatter \ + (--chromosomes/--no-parallel per job, so GNU parallel is never \ + invoked), each against its own persistent, chromosome-keyed \ + --cache-dir under {}: {}", + chromosomes.len(), + wrapper_cache_dir.display(), + chromosomes.join(", "), + ); + + let output = streaming::run_streaming(ravanan_cmd, &args, log_path.as_deref()) + .await + .map_err(GemmaError::Io)?; + if !output.status.success() { + return Err(GemmaError::RavananFailed( + String::from_utf8_lossy(&output.stderr).into_owned(), + )); + } + let outputs: Value = serde_json::from_str(&String::from_utf8_lossy(&output.stdout)) + .map_err(|e| GemmaError::RavananOutput(format!("could not parse ravanan's stdout: {e}")))?; + let gwa_json_paths = ravanan_output_paths(&outputs, "gwa_json")?; + let archive_paths = ravanan_output_paths(&outputs, "archive")?; + if gwa_json_paths.len() != archive_paths.len() { + return Err(GemmaError::RavananOutput( + "\"gwa_json\" and \"archive\" arrays have different lengths".to_string(), + )); + } + + // One archive per chromosome now (gwa.cwl's scatter), not one + // shared archive as compute() and this function's own kinship step + // get -- each gets its own extract dir and its basenames' rows are + // concatenated across all of them. + let mut columns = Vec::new(); + let mut rows = Vec::new(); + for (i, (gwa_json_path, archive_path)) in gwa_json_paths.iter().zip(archive_paths.iter()).enumerate() { + let gwa_record: Value = serde_json::from_str( + &std::fs::read_to_string(gwa_json_path).map_err(GemmaError::AssocFile)?, + ) + .map_err(|e| GemmaError::WrapperError { + step: "gwa", + detail: format!("could not parse gemma-wrapper's --json output: {e}"), + })?; + let basenames = assoc_file_basenames(&gwa_record); + if basenames.is_empty() { + return Err(GemmaError::NoAssocFiles); + } + + let extract_dir = Path::new(cache_dir).join(format!("{unique}-ravanan-extract-{i}")); + std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?; + let tar_output = streaming::run_streaming( + "tar", + &["xJf", archive_path, "-C", &extract_dir.to_string_lossy()], + None, + ) + .await + .map_err(GemmaError::Io)?; + if !tar_output.status.success() { + return Err(GemmaError::CommandFailed { + step: "extract-archive", + stderr: String::from_utf8_lossy(&tar_output.stderr).into_owned(), + }); + } + + let (file_columns, file_rows) = parse_assoc_files(&extract_dir, basenames)?; + if columns.is_empty() { + columns = file_columns; + } + rows.extend(file_rows); + let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless + } + + streaming::mark_complete(&log_path); + + Ok(GemmaOutput { + columns, + rows: Value::Array(rows), + notes: vec![ + ravanan_note, + chromosomes_note, + format!("workflow: {workflow_scm_path} (compiled to {compiled_workflow_arg})"), + ], + }) +} + #[cfg(test)] mod tests { use super::*; @@ -468,17 +741,23 @@ mod tests { #[test] fn parse_assoc_file_parses_header_and_numeric_rows() { let contents = "chr\trs\tps\tp_wald\n1\trs001\t3010274\t0.05\n1\trs002\t3492195\tNA\n"; - let rows = parse_assoc_file(contents); + let (columns, rows) = parse_assoc_file(contents); + assert_eq!(columns, vec!["chr", "rs", "ps", "p_wald"]); assert_eq!(rows.len(), 2); - assert_eq!(rows[0]["chr"], serde_json::json!(1.0)); - assert_eq!(rows[0]["rs"], serde_json::json!("rs001")); - assert_eq!(rows[0]["p_wald"], serde_json::json!(0.05)); - assert_eq!(rows[1]["p_wald"], serde_json::json!("NA")); + assert_eq!(rows[0], vec![ + serde_json::json!(1.0), + serde_json::json!("rs001"), + serde_json::json!(3010274.0), + serde_json::json!(0.05), + ]); + assert_eq!(rows[1][3], serde_json::json!("NA")); } #[test] fn parse_assoc_file_empty_contents_yields_no_rows() { - assert_eq!(parse_assoc_file("").len(), 0); + let (columns, rows) = parse_assoc_file(""); + assert_eq!(columns.len(), 0); + assert_eq!(rows.len(), 0); } #[test] diff --git a/src/lib.rs b/src/lib.rs index 5731147..b7407ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -83,6 +83,7 @@ pub fn app() -> Router { .routes(routes!(routes::search::search)) .routes(routes!(routes::correlation::correlation)) .routes(routes!(routes::gemma::compute_gemma)) + .routes(routes!(routes::gemma::compute_gemma_ravanan)) .routes(routes!(routes::gemma::gemma_status)) .routes(routes!(routes::rqtl::compute_rqtl)) .routes(routes!(routes::rqtl::rqtl_status)); diff --git a/src/response.rs b/src/response.rs index cb2adce..e85362a 100644 --- a/src/response.rs +++ b/src/response.rs @@ -38,6 +38,21 @@ pub struct Meta { pub took_ms: u64, /// The request's id, also in the `x-request-id` response header. pub request_id: Option<String>, + /// Column names for a `data` that is a list of rows, each row a + /// positional array rather than a keyed object (e.g. + /// `/compute/gemma`'s GEMMA `.assoc.txt` rows) -- cuts repeated key + /// names out of a large response. Absent for endpoints whose `data` + /// is already keyed or isn't tabular. + #[serde(skip_serializing_if = "Option::is_none")] + pub columns: Option<Vec<String>>, + /// How many rows `data` carries, alongside `columns` (same + /// condition for being present). + #[serde(skip_serializing_if = "Option::is_none")] + pub num_rows: Option<usize>, + /// `columns.len()`, alongside `columns`, so a caller doesn't need + /// to count it itself. + #[serde(skip_serializing_if = "Option::is_none")] + pub num_cols: Option<usize>, } /// A success response: the payload, facts about the response, notes on @@ -75,6 +90,16 @@ impl<T: Serialize + utoipa::ToSchema> Envelope<T> { self } + /// Sets `meta.columns`/`meta.num_cols`/`meta.num_rows` for a `data` + /// of `num_rows` positional rows, each `columns.len()` long (see + /// `Meta::columns`). + pub fn with_columns(mut self, columns: Vec<String>, num_rows: usize) -> Self { + self.meta.num_cols = Some(columns.len()); + self.meta.columns = Some(columns); + self.meta.num_rows = Some(num_rows); + self + } + /// Marks the response `partial` and records why in `notes`. pub fn partial(mut self, note: &str) -> Self { self.meta.state = State::Partial; @@ -148,6 +173,9 @@ impl Ctx { state: State::Complete, took_ms: self.info.took_ms(), request_id: self.info.id(), + columns: None, + num_rows: None, + num_cols: None, }, notes: Vec::new(), next: Vec::new(), diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs index 0b6b748..32d1818 100644 --- a/src/routes/gemma.rs +++ b/src/routes/gemma.rs @@ -67,7 +67,7 @@ trait values, and returns the resulting rows of GEMMA's own \ params(ComputeGemmaParams), request_body = GemmaRequest, responses( - (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as a JSON array of row objects", body = Envelope<serde_json::Value>), + (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as positional arrays (column names in meta.columns)", body = Envelope<serde_json::Value>), Unauthorized, (status = 409, description = "id is already in use by another run", body = ApiError), (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9), or id not alnum/-/_ (max 64 chars)", body = ApiError), @@ -122,9 +122,13 @@ pub async fn compute_gemma( ) .await .map(|output| { - let mut envelope = ctx.envelope(output.rows).with_note( - "Rows come from GEMMA's own .assoc.txt output; column names depend on the lmm value used.", - ); + let num_rows = output.rows.as_array().map(Vec::len).unwrap_or(0); + let mut envelope = ctx + .envelope(output.rows) + .with_columns(output.columns, num_rows) + .with_note( + "Rows come from GEMMA's own .assoc.txt output, positional (see meta.columns for names); column names depend on the lmm value used.", + ); for note in &output.notes { envelope = envelope.with_note(note); } @@ -143,6 +147,100 @@ pub async fn compute_gemma( }) } +#[utoipa::path(post, path = "/compute/gemma/ravanan", + summary = "Map a QTL with GEMMA, run via ravanan instead of gemma-wrapper directly", + description = "Same computation and request/response shape as POST \ +/compute/gemma (gemma-wrapper's K-matrix step, then its LMM association \ +step, both --loco), but driven by ravanan \ +(https://github.com/vepiteski/ravanan) running a CWL workflow compiled \ +from the ccwl source at cwl/gemma/gemma-loco.scm instead of this server's own direct subprocess \ +calls -- an alternative runner for the same two gemma-wrapper steps, \ +for comparing the two (correctness, timing). Exploratory: no caching of \ +ravanan's own store between requests beyond its --store directory, and \ +?id=/status polling tails ravanan's own stderr rather than gemma-wrapper's.", + params(ComputeGemmaParams), + request_body = GemmaRequest, + responses( + (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as positional arrays (column names in meta.columns)", body = Envelope<serde_json::Value>), + Unauthorized, + (status = 409, description = "id is already in use by another run", body = ApiError), + (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9), or id not alnum/-/_ (max 64 chars)", body = ApiError), + (status = 502, description = "ravanan or gemma-wrapper failed", body = ApiError) + ))] +#[gnauth(token_only)] +pub async fn compute_gemma_ravanan( + ctx: Ctx, + _token: AuthToken, + Query(params): Query<ComputeGemmaParams>, + Json(request): Json<GemmaRequest>, +) -> Result<Json<Envelope<serde_json::Value>>, ApiError> { + let rid = ctx.request_info(); + let lmm = request.lmm.unwrap_or(9); + if !matches!(lmm, 1 | 2 | 3 | 4 | 9) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_lmm", + "Invalid lmm", + format!( + "lmm must be one of 1, 2, 3, 4, 9 (got {lmm}); 1=Wald, 2=LRT, 3=score, 4=all three, 9=GeneNetwork's convention (observed to behave like LRT)." + ), + "compute_gemma_ravanan", + ) + .with_request_id(rid)); + } + if let Some(id) = ¶ms.id { + if !streaming::is_valid_run_id(id) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_run_id", + "Invalid id", + "id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), + "compute_gemma_ravanan", + ) + .with_request_id(rid)); + } + } + + let config = Config::load(); + gemma::compute_ravanan( + &config.ravanan_cmd, + &config.ccwl_cmd, + &config.gemma_ravanan_workflow, + &config.gemma_ravanan_manifest, + &config.ravanan_cache_dir, + &request.genotype_file, + &request.snps_file, + &request.trait_values, + lmm, + params.id.as_deref(), + ) + .await + .map(|output| { + let num_rows = output.rows.as_array().map(Vec::len).unwrap_or(0); + let mut envelope = ctx + .envelope(output.rows) + .with_columns(output.columns, num_rows) + .with_note( + "Rows come from GEMMA's own .assoc.txt output, positional (see meta.columns for names); column names depend on the lmm value used.", + ); + for note in &output.notes { + envelope = envelope.with_note(note); + } + Json(envelope) + }) + .map_err(|e| match e { + GemmaError::RunIdInUse(id) => ApiError::new( + StatusCode::CONFLICT, + "run_id_in_use", + "Run id already in use", + format!("id '{id}' is already in use; pick a different id."), + "compute_gemma_ravanan", + ) + .with_request_id(rid.clone()), + other => gemma_error_to_api_error(other, rid.clone()), + }) +} + #[utoipa::path(get, path = "/compute/gemma/status/{run_id}", summary = "Poll a running (or finished) /compute/gemma call's progress", description = "Returns the lines gemma-wrapper has written to stderr \ @@ -177,7 +275,16 @@ pub async fn gemma_status( } let config = Config::load(); let peek = params.peek.unwrap_or(0); - streaming::status(&config.gemma_cache_dir, &run_id, peek) + // A run_id may belong to either runner (POST /compute/gemma or + // POST /compute/gemma/ravanan), which log to different cache + // dirs; try gemma's first, then fall back to ravanan's. + let result = match streaming::status(&config.gemma_cache_dir, &run_id, peek) { + Err(StatusError::NotFound) => { + streaming::status(&config.ravanan_cache_dir, &run_id, peek) + } + other => other, + }; + result .map(|(lines, pointer)| { Json(ctx.envelope(RunStatus { run_id: run_id.clone(), diff --git a/src/routes/rqtl.rs b/src/routes/rqtl.rs index 92ee21c..2961124 100644 --- a/src/routes/rqtl.rs +++ b/src/routes/rqtl.rs @@ -73,7 +73,7 @@ concurrently -- same shape as /compute/gemma.", params(ComputeRqtlParams), request_body = RqtlRequest, responses( - (status = 200, description = "Mapped QTL markers (and, if requested, permutation thresholds)", body = Envelope<serde_json::Value>), + (status = 200, description = "Mapped QTL markers, as positional arrays (column names in meta.columns), and, if requested, permutation thresholds", body = Envelope<serde_json::Value>), Unauthorized, (status = 409, description = "id is already in use by another run", body = ApiError), (status = 422, description = "id not alnum/-/_ (max 64 chars)", body = ApiError), @@ -147,9 +147,13 @@ pub async fn compute_rqtl( .await .map(|output| { let notes = output.notes.clone(); - let mut envelope = ctx.envelope(rqtl::to_json(output)).with_note( - "markers come from R/qtl's scanone output; permutation is null unless nperm > 0 was requested.", - ); + let num_rows = output.markers.len(); + let mut envelope = ctx + .envelope(rqtl::to_json(output)) + .with_columns(rqtl::MARKER_COLUMNS.iter().map(|s| s.to_string()).collect(), num_rows) + .with_note( + "markers come from R/qtl's scanone output, positional (see meta.columns for names); permutation is null unless nperm > 0 was requested.", + ); for note in ¬es { envelope = envelope.with_note(note); } diff --git a/src/rqtl.rs b/src/rqtl.rs index 691cdbe..53d6d03 100644 --- a/src/rqtl.rs +++ b/src/rqtl.rs @@ -391,12 +391,35 @@ pub async fn compute( } } +/// `MarkerResult`'s fixed fields, in the order `to_json` lays out each +/// marker row -- the names a caller matches `markers`' positional rows +/// up against (a 4-way cross's `extra` effect columns, if present, +/// trail these and aren't named here; see `MarkerResult`'s own doc +/// comment). +pub const MARKER_COLUMNS: [&str; 4] = ["name", "chr", "pos", "lod"]; + /// `RqtlOutput` as the single JSON value the route returns -- see /// `crate::gemma`'s own note on why `serde_json::Value`, not a `Vec`, -/// is what utoipa needs here. +/// is what utoipa needs here. `markers` rows are positional (see +/// `MARKER_COLUMNS`), the same space-saving shape `crate::gemma::compute` +/// uses for its own (much larger) marker rows. pub fn to_json(output: RqtlOutput) -> Value { + let markers: Vec<Value> = output + .markers + .into_iter() + .map(|m| { + let mut row = vec![ + Value::String(m.name), + Value::String(m.chr), + serde_json::json!(m.pos), + serde_json::json!(m.lod), + ]; + row.extend(m.extra.into_iter().map(Value::String)); + Value::Array(row) + }) + .collect(); serde_json::json!({ - "markers": output.markers, + "markers": markers, "permutation": output.permutation, }) } |
