diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-12 18:00:53 +0200 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-12 18:00:53 +0200 |
| commit | ad3bb1e37e5feb22d3a8eb9e99e072f613fa186c (patch) | |
| tree | a587d6ceb1d72485096b175555572822aadddb85 /src | |
| parent | fb62e3e75b115206aeda0178aba4686234f8592e (diff) | |
| parent | 9cc1631324216d564aca4448cbe71f49dbf2a4af (diff) | |
| download | genenetwork4-ad3bb1e37e5feb22d3a8eb9e99e072f613fa186c.tar.gz | |
merge
Diffstat (limited to 'src')
| -rw-r--r-- | src/config.rs | 24 | ||||
| -rw-r--r-- | src/gemma.rs | 664 | ||||
| -rw-r--r-- | src/lib.rs | 5 | ||||
| -rw-r--r-- | src/routes/gemma.rs | 230 | ||||
| -rw-r--r-- | src/routes/mod.rs | 1 |
5 files changed, 923 insertions, 1 deletions
diff --git a/src/config.rs b/src/config.rs index 10f5755..16286b5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -60,6 +60,14 @@ pub struct Config { pub dev_token_client_secret: String, pub dev_token_username: String, pub dev_token_password: String, + /// The gemma-wrapper executable to invoke for POST /compute/gemma. + /// Same GEMMA_WRAPPER_CMD config key GeneNetwork3 uses. Defaults to + /// "gemma-wrapper", resolved via PATH. + pub gemma_wrapper_cmd: String, + /// Directory gemma-wrapper uses to cache K matrices and association + /// results between calls (its own --cache-dir switch). Defaults to + /// a "gn4-gemma" directory under the system temp dir. + pub gemma_cache_dir: String, } impl Config { @@ -102,6 +110,14 @@ impl Config { values.insert("GN3_TIMEOUT_SECS".to_string(), "30".to_string()); values.insert("XAPIAN_DB_PATH".to_string(), String::new()); values.insert("SEARCH_ENGINE".to_string(), "forward".to_string()); + values.insert("GEMMA_WRAPPER_CMD".to_string(), "gemma-wrapper".to_string()); + values.insert( + "GEMMA_CACHE_DIR".to_string(), + std::env::temp_dir() + .join("gn4-gemma") + .to_string_lossy() + .into_owned(), + ); // Layers 2 and 3: config files, in increasing priority. for env_key in ["GN4_CONF", "GN4_SECRETS"] { @@ -127,6 +143,8 @@ impl Config { "GN3_TIMEOUT_SECS", "XAPIAN_DB_PATH", "SEARCH_ENGINE", + "GEMMA_WRAPPER_CMD", + "GEMMA_CACHE_DIR", ] { if let Ok(value) = std::env::var(key) { values.insert(key.to_string(), value); @@ -153,6 +171,8 @@ impl Config { dev_token_client_secret: values["GN_AUTH_DEV_CLIENT_SECRET"].clone(), dev_token_username: values["GN_AUTH_DEV_USERNAME"].clone(), dev_token_password: values["GN_AUTH_DEV_PASSWORD"].clone(), + gemma_wrapper_cmd: values["GEMMA_WRAPPER_CMD"].clone(), + gemma_cache_dir: values["GEMMA_CACHE_DIR"].clone(), } } } @@ -223,6 +243,8 @@ mod tests { "GN3_TIMEOUT_SECS", "XAPIAN_DB_PATH", "SEARCH_ENGINE", + "GEMMA_WRAPPER_CMD", + "GEMMA_CACHE_DIR", ] { if std::env::var(key).is_ok() { eprintln!("skipping: {key} is set in the environment"); @@ -232,6 +254,8 @@ mod tests { let config = Config::load(); assert!(config.sql_uri.contains("db_webqtl")); assert_eq!(config.port, 3000); + assert_eq!(config.gemma_wrapper_cmd, "gemma-wrapper"); + assert!(config.gemma_cache_dir.ends_with("gn4-gemma")); assert_eq!(config.auth_server_url, "http://127.0.0.1:8081/"); assert_eq!(config.bind, "0.0.0.0"); assert!(config.test_token_enabled, "on by default"); diff --git a/src/gemma.rs b/src/gemma.rs new file mode 100644 index 0000000..c855673 --- /dev/null +++ b/src/gemma.rs @@ -0,0 +1,664 @@ +//! Core logic for POST /compute/gemma: compose and run gemma-wrapper +//! against a genotype file, a SNP annotation file and trait values, and +//! parse the resulting GEMMA association output into QTL mapping rows. +//! +//! Mirrors gn3/computations/gemma.py and gn3/api/gemma.py +//! (GeneNetwork3), but synchronously: axum/tokio can await an external +//! process directly, so there is no need for GeneNetwork3's Redis job +//! queue. The actual invocation also differs from GeneNetwork3's in two +//! respects, both found by running the real gemma-wrapper/GEMMA binaries +//! against the BXD fixtures rather than trusting GeneNetwork3's own +//! (entirely @unittest.skip'd) integration tests -- see `compute_gwa` +//! and `compute` below. + +use std::io; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::Path; +use std::process::{Output, Stdio}; + +use serde_json::Value; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; +use tokio::process::Command; + +/// One phenotype value per strain (genotype-file order), or `None` for +/// a missing ("NA") sample. +pub type TraitValues = Vec<Option<f64>>; + +#[derive(Debug)] +pub enum GemmaError { + /// Failed to spawn or run a child process. + Io(io::Error), + /// gemma-wrapper exited non-zero; carries its stderr. + CommandFailed { step: &'static str, stderr: String }, + /// gemma-wrapper's own --json stdout record was not valid JSON, or + /// reported an error (its own `errno` field was non-zero). + WrapperError { step: &'static str, detail: String }, + /// The real GEMMA `.assoc.txt` file gemma-wrapper pointed to could + /// not be found or read. + AssocFile(io::Error), + /// gemma-wrapper's GWA output archive did not list any files. + NoAssocFiles, + /// `run_id` already has a log file (a run with that id is in + /// progress, or finished and was never polled/cleaned up). Caught + /// before any subprocess runs, so this never silently truncates + /// another run's in-progress log. + RunIdInUse(String), +} + +impl std::fmt::Display for GemmaError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + GemmaError::Io(e) => write!(f, "failed to run gemma-wrapper: {e}"), + GemmaError::CommandFailed { step, stderr } => { + write!(f, "gemma-wrapper ({step}) failed: {stderr}") + } + GemmaError::WrapperError { step, detail } => { + write!(f, "gemma-wrapper ({step}) reported an error: {detail}") + } + GemmaError::AssocFile(e) => write!(f, "could not read GEMMA association output: {e}"), + GemmaError::NoAssocFiles => { + write!(f, "gemma-wrapper's response listed no association output files") + } + GemmaError::RunIdInUse(id) => { + write!(f, "run id '{id}' is already in use; pick a different id") + } + } + } +} + +/// Write `values` as a GEMMA/BIMBAM phenotype file: one value per line, +/// "NA" for a missing sample. Mirrors +/// gn3.computations.gemma.generate_pheno_txt_file, minus its hash-based +/// cache filename (the caller picks the path; see `compute`). +pub fn write_pheno_file(path: &Path, values: &TraitValues) -> io::Result<()> { + let mut contents = String::new(); + for value in values { + match value { + Some(v) => contents.push_str(&v.to_string()), + None => contents.push_str("NA"), + } + contents.push('\n'); + } + std::fs::write(path, contents) +} + +/// Run `cmd`, optionally tailing its stderr into `log_path` one line at +/// a time as it runs, and return the same `Output` shape +/// `Command::output()` would have (full stdout, full stderr, exit +/// status) -- callers that don't pass `log_path` see identical +/// behaviour to the plain, non-streaming call it replaces. +/// +/// stderr, not stdout, is what's tailed: gemma-wrapper's own `--json` +/// mode sends all of its human-readable progress (banner, chromosome +/// list, "Executing: ...", per-step info/debug lines) to stderr, and +/// reserves stdout for the one final JSON record, printed once at +/// the very end (`OUTPUT = (options[:json] ? $stderr : $stdout)` in +/// bin/gemma-wrapper) -- so stdout has nothing to stream. +async fn run_streaming(cmd: &str, args: &[&str], log_path: Option<&Path>) -> io::Result<Output> { + let mut child = Command::new(cmd) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let stdout_handle = child.stdout.take().expect("stdout was piped"); + let stderr_handle = child.stderr.take().expect("stderr was piped"); + + let stdout_task = tokio::spawn(async move { + let mut buf = Vec::new(); + let mut reader = stdout_handle; + let _ = reader.read_to_end(&mut buf).await; + buf + }); + + let log_path = log_path.map(Path::to_path_buf); + let stderr_task = tokio::spawn(async move { + let mut buf = Vec::new(); + let mut lines = BufReader::new(stderr_handle).lines(); + while let Ok(Some(line)) = lines.next_line().await { + if let Some(path) = &log_path { + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(path) { + let _ = writeln!(f, "{line}"); + } + } + buf.extend_from_slice(line.as_bytes()); + buf.push(b'\n'); + } + buf + }); + + let status = child.wait().await?; + let stdout = stdout_task.await.unwrap_or_default(); + let stderr = stderr_task.await.unwrap_or_default(); + Ok(Output { status, stdout, stderr }) +} + +fn parse_wrapper_output(step: &'static str, output: Output) -> Result<Value, GemmaError> { + if !output.status.success() { + return Err(GemmaError::CommandFailed { + step, + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + }); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let record: Value = serde_json::from_str(&stdout).map_err(|e| GemmaError::WrapperError { + step, + detail: format!("could not parse gemma-wrapper's --json output: {e}"), + })?; + if record.get("errno").and_then(Value::as_i64).unwrap_or(0) != 0 { + return Err(GemmaError::WrapperError { + step, + detail: record + .get("error") + .and_then(Value::as_str) + .unwrap_or("gemma-wrapper reported a non-zero errno") + .to_string(), + }); + } + Ok(record) +} + +/// Run gemma-wrapper's `--json` K-matrix step and return its stdout +/// record (parsed JSON). +/// +/// `--loco` (bare, no chromosome list) is required here: confirmed by +/// actually running gemma-wrapper against the BXD fixtures, a plain +/// (non-LOCO) `-gk` only ever leaves its kinship matrix inside a +/// `.tar.xz` archive in `--cache-dir`, with no loose file at the path +/// the non-LOCO branch of gemma-wrapper's own GWA step later assumes -- +/// that combination errors out (`gemma: ... open file ... CheckParam`). +/// `--loco` is the combination gemma-wrapper's own USAGE banner +/// documents and the one whose GWA-side code path actually unpacks the +/// archive it needs (see `compute_gwa` and `compute`). Passing it with +/// no chromosome list makes gemma-wrapper derive the full chromosome +/// list from the SNP annotation file itself. +async fn compute_kinship( + gemma_wrapper_cmd: &str, + cache_dir: &str, + genotype_file: &str, + pheno_file: &str, + snps_file: &str, + log_path: Option<&Path>, +) -> Result<Value, GemmaError> { + let args = kinship_args(cache_dir, genotype_file, pheno_file, snps_file); + let output = run_streaming(gemma_wrapper_cmd, &args, log_path) + .await + .map_err(GemmaError::Io)?; + parse_wrapper_output("kinship", output) +} + +/// Run gemma-wrapper's `--json` GWA (association) step, given the +/// K-step's JSON record written to `k_json_path` (gemma-wrapper's +/// `--input` takes a file path, not stdin), and return its stdout +/// record. `--loco` is required here for the same reason as in +/// `compute_kinship`. +/// +/// Note: unlike GeneNetwork3's `generate_gemma_cmd` +/// (gn3/computations/gemma.py), which always appends a literal "-gk" to +/// the forwarded GEMMA args regardless of whether an LMM run was +/// requested, this passes `-lmm` without `-gk`, matching gemma-wrapper's +/// own documented usage (its USAGE banner in bin/gemma-wrapper). +/// gemma-wrapper decides K-only vs. GWA purely by whether "-gk" is +/// present in those args (`DO_COMPUTE_KINSHIP = +/// gemma_args.include?("-gk")`); passing both flags together, as +/// GeneNetwork3 does, makes every one of its calls a K-only run, and the +/// `-lmm` value is silently ignored. That GeneNetwork3 code path has no +/// non-skipped test covering it (tests/integration/test_gemma.py is +/// `@unittest.skip`), so it is not knowingly replicated here. +async fn compute_gwa( + gemma_wrapper_cmd: &str, + cache_dir: &str, + genotype_file: &str, + pheno_file: &str, + snps_file: &str, + lmm: u8, + k_json_path: &str, + log_path: Option<&Path>, +) -> Result<Value, GemmaError> { + let lmm_str = lmm.to_string(); + let args = gwa_args( + cache_dir, + genotype_file, + pheno_file, + snps_file, + &lmm_str, + k_json_path, + ); + let output = run_streaming(gemma_wrapper_cmd, &args, log_path) + .await + .map_err(GemmaError::Io)?; + parse_wrapper_output("gwa", output) +} + +/// The association output basenames gemma-wrapper's GWA-step JSON +/// record points at: `record["files"]` is a list of +/// `[chr, log_basename, data_basename]`, one entry per chromosome in a +/// `--loco` run (see `compute`). +fn assoc_file_basenames(record: &Value) -> Vec<&str> { + record + .get("files") + .and_then(Value::as_array) + .map(|files| { + files + .iter() + .filter_map(|entry| entry.as_array()?.get(2)?.as_str()) + .collect() + }) + .unwrap_or_default() +} + +/// One row of a GEMMA `.assoc.txt` file, keyed by its header column +/// names. Columns vary with `-lmm` (stock 1/2/3 run one test, 4 runs +/// all three; this build's `9`, GeneNetwork's convention, was observed +/// to carry `p_lrt`), so this stays a generic map rather than a fixed +/// struct. +pub type AssocRow = serde_json::Map<String, Value>; + +/// Parse a GEMMA `.assoc.txt` file (tab-separated, header row first) +/// into one row per marker. A value that parses as a number is kept +/// numeric; "NA" and anything else stays a string. +fn parse_assoc_file(contents: &str) -> Vec<AssocRow> { + let mut lines = contents.lines(); + let Some(header) = lines.next() else { + return Vec::new(); + }; + let columns: Vec<&str> = header.split('\t').collect(); + lines + .filter(|line| !line.is_empty()) + .map(|line| { + let mut row = AssocRow::new(); + for (column, value) in columns.iter().zip(line.split('\t')) { + let json_value = value + .parse::<f64>() + .map(|n| serde_json::json!(n)) + .unwrap_or_else(|_| Value::String(value.to_string())); + row.insert((*column).to_string(), json_value); + } + row + }) + .collect() +} + +/// Extract `archive` (a `.tar.xz`, relative to `cache_dir`) into +/// `dest_dir`. Confirmed necessary by actually running the real +/// binaries: a `--loco` GWA run's `.assoc.txt` files (one per +/// chromosome) only ever end up inside this archive, never as loose +/// files in `cache_dir` -- unlike the kinship step's `.cXX.txt` files, +/// which do also appear loose (gemma-wrapper itself is inconsistent +/// about this between the two steps). +async fn extract_archive(cache_dir: &str, archive: &str, dest_dir: &str) -> Result<(), GemmaError> { + let archive_path = Path::new(cache_dir).join(archive); + let output = run_streaming( + "tar", + &["xJf", &archive_path.to_string_lossy(), "-C", dest_dir], + None, + ) + .await + .map_err(GemmaError::Io)?; + if !output.status.success() { + return Err(GemmaError::CommandFailed { + step: "extract-archive", + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + }); + } + Ok(()) +} + +fn nanos_since_epoch() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) +} + +/// The mapped QTL rows, plus the exact gemma-wrapper invocations used +/// to produce them (see `compute`'s `notes` below), so a caller can see +/// precisely what was run without reading server logs. +pub struct GemmaOutput { + pub rows: Value, + pub notes: Vec<String>, +} + +/// The gemma-wrapper arguments for the kinship step, shared between the +/// actual invocation (`compute_kinship`) and the note describing it +/// (`compute`), so the two can never drift apart. +fn kinship_args<'a>( + cache_dir: &'a str, + genotype_file: &'a str, + pheno_file: &'a str, + snps_file: &'a str, +) -> Vec<&'a str> { + vec![ + "--json", + "--cache-dir", + cache_dir, + "--loco", + "--", + "-g", + genotype_file, + "-p", + pheno_file, + "-a", + snps_file, + "-gk", + ] +} + +/// The gemma-wrapper arguments for the GWA step, same pairing as +/// `kinship_args`. +fn gwa_args<'a>( + cache_dir: &'a str, + genotype_file: &'a str, + pheno_file: &'a str, + snps_file: &'a str, + lmm_str: &'a str, + k_json_path: &'a str, +) -> Vec<&'a str> { + vec![ + "--json", + "--cache-dir", + cache_dir, + "--loco", + "--input", + k_json_path, + "--", + "-g", + genotype_file, + "-p", + pheno_file, + "-a", + snps_file, + "-lmm", + lmm_str, + ] +} + +/// Run the full gemma-wrapper flow -- compute kinship, then run the LMM +/// association test, both `--loco` -- and return the mapped QTL as a +/// JSON array of rows (each a JSON object), concatenated across every +/// chromosome's `.assoc.txt` from the real GEMMA output, plus the two +/// gemma-wrapper invocations used (`GemmaOutput::notes`). The rows are +/// one `Value::Array`, not a `Vec<Value>`: utoipa (for the OpenAPI +/// schema) implements `ToSchema` for `serde_json::Value` itself, but +/// `Vec<serde_json::Value>` needs `Value: ComposeSchema`, which it does +/// not implement in this utoipa version. +/// `run_id` must be a short token of ASCII letters, digits, `-` or `_` +/// (no path separators or `.`): it is joined straight onto `cache_dir` +/// to name the log file `status` reads back, so an unchecked value +/// would let a caller write/read outside `cache_dir` (the same +/// unchecked-`run_id`-as-path-component GeneNetwork3 does for its own +/// `/api/stream/{run_id}` -- not knowingly replicated here). +pub fn is_valid_run_id(run_id: &str) -> bool { + !run_id.is_empty() + && run_id.len() <= 64 + && run_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') +} + +/// The line `compute` appends to the log file once it finishes +/// successfully, so a `status` poller can tell "still running" apart +/// from "done" without needing a separate field -- same convention +/// GeneNetwork3 uses ("Done with GN3 Computation" in gn3/api/rqtl2.py), +/// under this project's own name. +pub const COMPLETION_MARKER: &str = "GN4: computation complete"; + +pub async fn compute( + gemma_wrapper_cmd: &str, + cache_dir: &str, + genotype_file: &str, + snps_file: &str, + trait_values: &TraitValues, + lmm: u8, + run_id: Option<&str>, +) -> Result<GemmaOutput, GemmaError> { + std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?; + let unique = format!("{}-{}", std::process::id(), nanos_since_epoch()); + let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt")); + write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?; + let pheno_file = pheno_path.to_string_lossy().into_owned(); + + // One log file for the whole request (both steps). create_new + // fails if it already exists, atomically: run_id must be unique, + // so a collision errors out here rather than silently truncating + // another run's (possibly still in-progress) log. + let log_path = run_id.map(|id| Path::new(cache_dir).join(format!("{id}.log"))); + if let Some(path) = &log_path { + match std::fs::OpenOptions::new().write(true).create_new(true).open(path) { + Ok(_) => {} + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + return Err(GemmaError::RunIdInUse(run_id.unwrap().to_string())); + } + Err(e) => return Err(GemmaError::Io(e)), + } + } + + let kinship_note = format!( + "Kinship: {gemma_wrapper_cmd} {}", + kinship_args(cache_dir, genotype_file, &pheno_file, snps_file).join(" ") + ); + let k_record = compute_kinship( + gemma_wrapper_cmd, + cache_dir, + genotype_file, + &pheno_file, + snps_file, + log_path.as_deref(), + ) + .await?; + let k_json_path = Path::new(cache_dir).join(format!("{unique}-k.json")); + std::fs::write(&k_json_path, k_record.to_string()).map_err(GemmaError::Io)?; + + let lmm_str = lmm.to_string(); + let gwa_note = format!( + "GWA: {gemma_wrapper_cmd} {}", + gwa_args( + cache_dir, + genotype_file, + &pheno_file, + snps_file, + &lmm_str, + &k_json_path.to_string_lossy(), + ) + .join(" ") + ); + let gwa_record = compute_gwa( + gemma_wrapper_cmd, + cache_dir, + genotype_file, + &pheno_file, + snps_file, + lmm, + &k_json_path.to_string_lossy(), + log_path.as_deref(), + ) + .await?; + + let basenames = assoc_file_basenames(&gwa_record); + if basenames.is_empty() { + return Err(GemmaError::NoAssocFiles); + } + let archive = gwa_record + .get("archive") + .and_then(Value::as_str) + .ok_or_else(|| GemmaError::WrapperError { + step: "gwa", + detail: "gemma-wrapper's response did not list an output archive".to_string(), + })?; + let extract_dir = Path::new(cache_dir).join(format!("{unique}-gwa-extract")); + std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?; + extract_archive(cache_dir, archive, &extract_dir.to_string_lossy()).await?; + + let mut rows = Vec::new(); + for basename in basenames { + let contents = std::fs::read_to_string(extract_dir.join(basename)) + .map_err(GemmaError::AssocFile)?; + rows.extend(parse_assoc_file(&contents).into_iter().map(Value::Object)); + } + let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless + + if let Some(path) = &log_path { + if let Ok(mut f) = std::fs::OpenOptions::new().append(true).open(path) { + let _ = writeln!(f, "{COMPLETION_MARKER}"); + } + } + + Ok(GemmaOutput { + rows: Value::Array(rows), + notes: vec![kinship_note, gwa_note], + }) +} + +/// Why `status` (below) couldn't read a run's log. +#[derive(Debug)] +pub enum StatusError { + /// No log file for this `run_id` (unknown id, or the run hasn't + /// written anything yet -- `compute` creates the file before its + /// first subprocess call, but there is a brief window before that). + NotFound, + Io(io::Error), +} + +/// Lines appended to `run_id`'s log since byte offset `peek`, and the +/// new offset to pass as `peek` next time. Mirrors GeneNetwork3's +/// `GET /api/stream/{run_id}?peak=` (gn3/api/streaming.py, spelled +/// "peak" there), stateless in the same way: the caller tracks `peek`, +/// not the server. +pub fn status(cache_dir: &str, run_id: &str, peek: u64) -> Result<(Vec<String>, u64), StatusError> { + let path = Path::new(cache_dir).join(format!("{run_id}.log")); + let mut file = std::fs::File::open(&path).map_err(|e| { + if e.kind() == io::ErrorKind::NotFound { + StatusError::NotFound + } else { + StatusError::Io(e) + } + })?; + file.seek(SeekFrom::Start(peek)).map_err(StatusError::Io)?; + let mut buf = String::new(); + file.read_to_string(&mut buf).map_err(StatusError::Io)?; + let pointer = peek + buf.len() as u64; + let lines = buf.lines().map(str::to_string).collect(); + Ok((lines, pointer)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn write_pheno_file_writes_one_value_per_line_and_na_for_missing() { + let path = std::env::temp_dir().join(format!("gn4-gemma-test-{}", std::process::id())); + write_pheno_file(&path, &vec![Some(9.944), None, Some(8.572)]).unwrap(); + let contents = std::fs::read_to_string(&path).unwrap(); + std::fs::remove_file(&path).unwrap(); + assert_eq!(contents, "9.944\nNA\n8.572\n"); + } + + #[test] + fn parse_assoc_file_parses_header_and_numeric_rows() { + let contents = "chr\trs\tps\tp_wald\n1\trs001\t3010274\t0.05\n1\trs002\t3492195\tNA\n"; + let rows = parse_assoc_file(contents); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0]["chr"], serde_json::json!(1.0)); + assert_eq!(rows[0]["rs"], serde_json::json!("rs001")); + assert_eq!(rows[0]["p_wald"], serde_json::json!(0.05)); + assert_eq!(rows[1]["p_wald"], serde_json::json!("NA")); + } + + #[test] + fn parse_assoc_file_empty_contents_yields_no_rows() { + assert_eq!(parse_assoc_file("").len(), 0); + } + + #[test] + fn assoc_file_basenames_reads_the_third_element_of_every_file_entry() { + let record = serde_json::json!({ + "files": [["1", "a.log.txt", "a.assoc.txt"], ["2", "b.log.txt", "b.assoc.txt"]] + }); + assert_eq!(assoc_file_basenames(&record), vec!["a.assoc.txt", "b.assoc.txt"]); + } + + #[test] + fn assoc_file_basenames_missing_files_yields_empty() { + assert_eq!(assoc_file_basenames(&serde_json::json!({})), Vec::<&str>::new()); + } + + #[test] + fn parse_wrapper_output_rejects_non_zero_errno() { + use std::os::unix::process::ExitStatusExt; + let output = Output { + status: std::process::ExitStatus::from_raw(0), + stdout: br#"{"errno": 1, "error": "boom"}"#.to_vec(), + stderr: Vec::new(), + }; + let err = parse_wrapper_output("kinship", output).unwrap_err(); + assert!(matches!(err, GemmaError::WrapperError { step: "kinship", .. })); + assert_eq!(err.to_string(), "gemma-wrapper (kinship) reported an error: boom"); + } + + #[test] + fn kinship_args_is_loco_with_gk_and_no_lmm() { + let args = kinship_args("/cache", "geno.txt.gz", "pheno.txt", "snps.txt"); + assert_eq!( + args, + vec![ + "--json", "--cache-dir", "/cache", "--loco", "--", "-g", "geno.txt.gz", "-p", + "pheno.txt", "-a", "snps.txt", "-gk" + ] + ); + } + + #[test] + fn gwa_args_is_loco_with_lmm_and_no_gk() { + let args = gwa_args("/cache", "geno.txt.gz", "pheno.txt", "snps.txt", "9", "/cache/k.json"); + assert_eq!( + args, + vec![ + "--json", "--cache-dir", "/cache", "--loco", "--input", "/cache/k.json", "--", + "-g", "geno.txt.gz", "-p", "pheno.txt", "-a", "snps.txt", "-lmm", "9" + ] + ); + assert!(!args.contains(&"-gk"), "the GWA step must never carry -gk alongside -lmm"); + } + + #[test] + fn is_valid_run_id_accepts_alnum_dash_underscore() { + for ok in ["abc123", "a-b_c", "A1", "x"] { + assert!(is_valid_run_id(ok), "{ok}"); + } + } + + #[test] + fn is_valid_run_id_rejects_path_separators_and_empty() { + for bad in ["", "../etc/passwd", "a/b", "a.b", "a b", &"x".repeat(65)] { + assert!(!is_valid_run_id(bad), "{bad}"); + } + } + + #[test] + fn status_reads_from_peek_and_returns_a_new_pointer() { + let path = std::env::temp_dir().join(format!("gn4-gemma-status-test-{}", std::process::id())); + let cache_dir = path.to_str().unwrap(); + std::fs::create_dir_all(cache_dir).unwrap(); + std::fs::write(Path::new(cache_dir).join("run1.log"), "line one\nline two\n").unwrap(); + + let (lines, pointer) = status(cache_dir, "run1", 0).unwrap(); + assert_eq!(lines, vec!["line one", "line two"]); + + std::fs::OpenOptions::new() + .append(true) + .open(Path::new(cache_dir).join("run1.log")) + .unwrap() + .write_all(b"line three\n") + .unwrap(); + let (more, _) = status(cache_dir, "run1", pointer).unwrap(); + assert_eq!(more, vec!["line three"]); + + std::fs::remove_dir_all(cache_dir).unwrap(); + } + + #[test] + fn status_unknown_run_id_is_not_found() { + let err = status("/tmp", "no-such-gn4-gemma-run-id", 0).unwrap_err(); + assert!(matches!(err, StatusError::NotFound)); + } +} diff --git a/src/lib.rs b/src/lib.rs index 6be5cff..50db2b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,7 @@ mod db; pub mod embedded; mod error; pub mod indexer; +mod gemma; pub mod response; mod routes; pub mod search; @@ -77,7 +78,9 @@ pub fn app() -> Router { .routes(routes!(routes::traits::trait_values)) .routes(routes!(routes::publications::publication_traits)) .routes(routes!(routes::search::search)) - .routes(routes!(routes::correlation::correlation)); + .routes(routes!(routes::correlation::correlation)) + .routes(routes!(routes::gemma::compute_gemma)) + .routes(routes!(routes::gemma::gemma_status)); // Registered -- and so reachable and listed in /openapi.json -- unless // GENENETWORK4_TEST_TOKEN switches it off; when registered it decides // per request whether the caller is local. See src/routes/dev.rs. diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs new file mode 100644 index 0000000..c79c589 --- /dev/null +++ b/src/routes/gemma.rs @@ -0,0 +1,230 @@ +//! Compute domain: /compute/gemma invokes gemma-wrapper (and the real +//! GEMMA binary it wraps) to map a QTL from a genotype file, a SNP +//! annotation file and a list of trait values. `GET +//! /compute/gemma/status/{run_id}` tails that run's progress while it +//! is (or was) in flight, the same shape GeneNetwork2/3 use for R/qtl +//! (gn3/computations/streaming.py, gn3/api/streaming.py): the caller +//! picks a `run_id`, passes it as `?id=` on the POST, and polls the +//! status endpoint with `?peek=` (the byte offset already read) on a +//! second connection while the first is still blocked on the POST. See +//! `crate::gemma` for the invocation and log-writing itself. + +use axum::Json; +use axum::extract::{Path, Query}; +use axum::http::StatusCode; +use serde::{Deserialize, Serialize}; + +use crate::auth::AuthToken; +use crate::config::Config; +use crate::error::{ApiError, Unauthorized}; +use crate::gemma::{self, GemmaError, StatusError}; +use crate::response::{Ctx, Envelope, RequestInfo}; + +/// `?id=` on `POST /compute/gemma`: an optional, caller-chosen run id +/// to log this run's progress under and poll back via `GET +/// /compute/gemma/status/{id}`. Omitted entirely, no log is written. +/// Must be unique: a collision with an existing log (another run still +/// in progress, or one finished and never polled/cleaned up) errors out +/// rather than overwriting it -- see `gemma::GemmaError::RunIdInUse`. +#[derive(Deserialize, utoipa::IntoParams)] +pub struct ComputeGemmaParams { + pub id: Option<String>, +} + +#[derive(Deserialize, utoipa::IntoParams)] +pub struct GemmaStatusParams { + /// Byte offset already read; 0 to read from the start. Echoed back + /// (grown) as `pointer` in the response -- pass that next time. + pub peek: Option<u64>, +} + +#[derive(Serialize, utoipa::ToSchema)] +pub struct GemmaStatus { + pub run_id: String, + /// New lines of gemma-wrapper's stderr (its progress output under + /// --json) since `peek`. The last line, once the run has finished, + /// is the completion marker (`gemma::COMPLETION_MARKER`). + pub lines: Vec<String>, + /// The new byte offset; pass this back as `peek` next poll. + pub pointer: u64, +} + +#[derive(Deserialize, utoipa::ToSchema)] +pub struct GemmaRequest { + /// Path to a BIMBAM genotype file (plain text or gzipped), passed + /// as GEMMA's `-g`. + pub genotype_file: String, + /// Path to a SNP annotation file (rs id, position, chromosome), + /// passed as GEMMA's `-a`. + pub snps_file: String, + /// One phenotype value per strain, in the genotype file's strain + /// order; `null` for a missing value. + pub trait_values: Vec<Option<f64>>, + /// GEMMA's `-lmm` model selector: stock values are 1 = Wald, + /// 2 = LRT, 3 = score, 4 = all three. Defaults to 9 when omitted -- + /// not a stock value, but the one GeneNetwork2 + /// (wqflask/marker_regression/gemma_mapping.py) and gn-guile + /// (gn/runner/gemma.scm) both use, and the only value this has been + /// run against in the --loco shape (see `crate::gemma`). Confirmed + /// by actually running it: `gemma-gn2` accepts it and the output + /// carries `p_lrt`, not `p_wald` -- some other mapping than the + /// stock 1-4 scale, not independently verified here. + pub lmm: Option<u8>, +} + +#[utoipa::path(post, path = "/compute/gemma", + summary = "Map a QTL from a genotype file and trait values using GEMMA", + description = "Runs gemma-wrapper's K-matrix step, then its LMM \ +association step, over the given genotype and SNP annotation files and \ +trait values, and returns the resulting rows of GEMMA's own \ +.assoc.txt output. Pass ?id= to log progress for GET \ +/compute/gemma/status/{id} to poll concurrently.", + params(ComputeGemmaParams), + request_body = GemmaRequest, + responses( + (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as a JSON array of row objects", body = Envelope<serde_json::Value>), + Unauthorized, + (status = 409, description = "id is already in use by another run", body = ApiError), + (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9), or id not alnum/-/_ (max 64 chars)", body = ApiError), + (status = 502, description = "gemma-wrapper or GEMMA failed", body = ApiError) + ))] +pub async fn compute_gemma( + ctx: Ctx, + _token: AuthToken, + Query(params): Query<ComputeGemmaParams>, + Json(request): Json<GemmaRequest>, +) -> Result<Json<Envelope<serde_json::Value>>, ApiError> { + let rid = ctx.request_info(); + let lmm = request.lmm.unwrap_or(9); + // Only the values actually run against this GEMMA build (see the + // `lmm` field doc above) are allowed; 5-8 are stock-invalid and + // untested here, so they're rejected rather than guessed at. + if !matches!(lmm, 1 | 2 | 3 | 4 | 9) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_lmm", + "Invalid lmm", + format!( + "lmm must be one of 1, 2, 3, 4, 9 (got {lmm}); 1=Wald, 2=LRT, 3=score, 4=all three, 9=GeneNetwork's convention (observed to behave like LRT)." + ), + "compute_gemma", + ) + .with_request_id(rid)); + } + if let Some(id) = ¶ms.id { + if !gemma::is_valid_run_id(id) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_run_id", + "Invalid id", + "id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), + "compute_gemma", + ) + .with_request_id(rid)); + } + } + + let config = Config::load(); + gemma::compute( + &config.gemma_wrapper_cmd, + &config.gemma_cache_dir, + &request.genotype_file, + &request.snps_file, + &request.trait_values, + lmm, + params.id.as_deref(), + ) + .await + .map(|output| { + let mut envelope = ctx.envelope(output.rows).with_note( + "Rows come from GEMMA's own .assoc.txt output; column names depend on the lmm value used.", + ); + for note in &output.notes { + envelope = envelope.with_note(note); + } + Json(envelope) + }) + .map_err(|e| match e { + GemmaError::RunIdInUse(id) => ApiError::new( + StatusCode::CONFLICT, + "run_id_in_use", + "Run id already in use", + format!("id '{id}' is already in use; pick a different id."), + "compute_gemma", + ) + .with_request_id(rid.clone()), + other => gemma_error_to_api_error(other, rid.clone()), + }) +} + +#[utoipa::path(get, path = "/compute/gemma/status/{run_id}", + summary = "Poll a running (or finished) /compute/gemma call's progress", + description = "Returns the lines gemma-wrapper has written to stderr \ +for this run_id since byte offset `peek`, and a new `pointer` to pass as \ +`peek` next time. Call this on a second connection while the POST is \ +still in flight, the same tail-the-log-file shape GeneNetwork2/3 use for \ +R/qtl.", + params(("run_id" = String, Path, description = "The id passed as ?id= on the POST"), GemmaStatusParams), + responses( + (status = 200, description = "New log lines and the new pointer", body = Envelope<GemmaStatus>), + Unauthorized, + (status = 404, description = "No run with this id (unknown, or not started yet)", body = ApiError), + (status = 422, description = "run_id not alnum/-/_ (max 64 chars)", body = ApiError) + ))] +pub async fn gemma_status( + ctx: Ctx, + _token: AuthToken, + Path(run_id): Path<String>, + Query(params): Query<GemmaStatusParams>, +) -> Result<Json<Envelope<GemmaStatus>>, ApiError> { + let rid = ctx.request_info(); + if !gemma::is_valid_run_id(&run_id) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_run_id", + "Invalid run_id", + "run_id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), + "gemma_status", + ) + .with_request_id(rid)); + } + let config = Config::load(); + let peek = params.peek.unwrap_or(0); + gemma::status(&config.gemma_cache_dir, &run_id, peek) + .map(|(lines, pointer)| { + Json(ctx.envelope(GemmaStatus { + run_id: run_id.clone(), + lines, + pointer, + })) + }) + .map_err(|e| match e { + StatusError::NotFound => ApiError::new( + StatusCode::NOT_FOUND, + "run_not_found", + "Run not found", + format!("No run with id '{run_id}' (unknown, or not started yet)."), + "gemma_status", + ) + .with_request_id(rid), + StatusError::Io(err) => ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "internal_error", + "Internal error", + err.to_string(), + "gemma_status", + ) + .with_request_id(rid), + }) +} + +fn gemma_error_to_api_error(error: GemmaError, rid: RequestInfo) -> ApiError { + ApiError::new( + StatusCode::BAD_GATEWAY, + "gemma_failed", + "GEMMA computation failed", + error.to_string(), + "compute_gemma", + ) + .with_request_id(rid) +} diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 55227e0..3a9f3e3 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -7,6 +7,7 @@ pub mod accessions; pub mod correlation; pub mod datasets; pub mod dev; +pub mod gemma; pub mod groups; pub mod mapping_methods; pub mod publications; |
