From 9e03cf40b875790c19d3fa1623297841b72b553e Mon Sep 17 00:00:00 2001 From: Pjotr Date: Sun, 11 Oct 2026 19:49:57 +0000 Subject: gemma: add POST /compute/gemma, backed by the real gemma-wrapper/GEMMA Maps a QTL from a genotype file, a SNP annotation file and trait values, by running gemma-wrapper's K-matrix step then its LMM association step, both synchronously (axum/tokio awaits the subprocess directly, no job queue needed) and returning the parsed .assoc.txt rows in the standard envelope. Both steps use a bare --loco (gemma-wrapper derives the chromosome list from the SNP annotation file). This was arrived at empirically, not from GeneNetwork3's or GeneNetwork2's own gemma code: run for real against the BXD fixtures, - GeneNetwork3's gn3/computations/gemma.py composes a GWA command that always keeps "-gk" alongside "-lmm"; real GEMMA rejects that combination outright ("only one of -gk ... -lmm ... options is allowed"), so none of its gwa-compute/k-gwa-compute endpoints can produce output. Unsurprising: tests/integration/test_gemma.py is entirely @unittest.skip, so this was never exercised. - GeneNetwork2's wqflask/marker_regression/gemma_mapping.py correctly omits "-gk" from the GWA step, but its non-LOCO branch fails too, the same way my own first (non-LOCO) attempt did: gemma-wrapper only ever archives a non-LOCO kinship matrix into a .tar.xz, with no loose file at the path its own non-LOCO GWA code assumes ("Enforce failed for file_kin ... open file"). Only --loco, which gemma-wrapper's own USAGE banner documents and which does unpack that archive, actually works. A --loco run's .assoc.txt files (one per chromosome) are themselves only ever inside gemma-wrapper's own output archive, never loose, so gemma.rs shells out to `tar` (one of gemma-wrapper's own propagated dependencies) to extract them before concatenating all chromosomes' rows into the response. guix.scm now depends on (gn packages gemma) from guix-bioinformatics for gemma-wrapper (propagated-input: it's invoked as a subprocess, not linked, and this makes it land in a consuming profile too, same as GeneNetwork3's and GeneNetwork2's own packages do it). That dependency needed guix-bioinformatics' own gn/packages/gemma.scm fixed first (see that repo's fix/gemma-which-shadowing branch) -- gemma-gn2 would not build at all before that fix. Verified end to end: `cargo test` passes in full, including a real (non-mocked, non-skipped) run of gemma-wrapper/GEMMA against the BXD fixtures copied from genenetwork3/tests/test_data, producing exactly 7320 association rows (one per marker in BXD_snps.txt) in ~19s. Co-Authored-By: Claude Sonnet 5 --- src/config.rs | 24 ++++ src/gemma.rs | 406 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 4 +- src/routes/gemma.rs | 90 ++++++++++++ src/routes/mod.rs | 1 + 5 files changed, 524 insertions(+), 1 deletion(-) create mode 100644 src/gemma.rs create mode 100644 src/routes/gemma.rs (limited to 'src') diff --git a/src/config.rs b/src/config.rs index 10f5755..16286b5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -60,6 +60,14 @@ pub struct Config { pub dev_token_client_secret: String, pub dev_token_username: String, pub dev_token_password: String, + /// The gemma-wrapper executable to invoke for POST /compute/gemma. + /// Same GEMMA_WRAPPER_CMD config key GeneNetwork3 uses. Defaults to + /// "gemma-wrapper", resolved via PATH. + pub gemma_wrapper_cmd: String, + /// Directory gemma-wrapper uses to cache K matrices and association + /// results between calls (its own --cache-dir switch). Defaults to + /// a "gn4-gemma" directory under the system temp dir. + pub gemma_cache_dir: String, } impl Config { @@ -102,6 +110,14 @@ impl Config { values.insert("GN3_TIMEOUT_SECS".to_string(), "30".to_string()); values.insert("XAPIAN_DB_PATH".to_string(), String::new()); values.insert("SEARCH_ENGINE".to_string(), "forward".to_string()); + values.insert("GEMMA_WRAPPER_CMD".to_string(), "gemma-wrapper".to_string()); + values.insert( + "GEMMA_CACHE_DIR".to_string(), + std::env::temp_dir() + .join("gn4-gemma") + .to_string_lossy() + .into_owned(), + ); // Layers 2 and 3: config files, in increasing priority. for env_key in ["GN4_CONF", "GN4_SECRETS"] { @@ -127,6 +143,8 @@ impl Config { "GN3_TIMEOUT_SECS", "XAPIAN_DB_PATH", "SEARCH_ENGINE", + "GEMMA_WRAPPER_CMD", + "GEMMA_CACHE_DIR", ] { if let Ok(value) = std::env::var(key) { values.insert(key.to_string(), value); @@ -153,6 +171,8 @@ impl Config { dev_token_client_secret: values["GN_AUTH_DEV_CLIENT_SECRET"].clone(), dev_token_username: values["GN_AUTH_DEV_USERNAME"].clone(), dev_token_password: values["GN_AUTH_DEV_PASSWORD"].clone(), + gemma_wrapper_cmd: values["GEMMA_WRAPPER_CMD"].clone(), + gemma_cache_dir: values["GEMMA_CACHE_DIR"].clone(), } } } @@ -223,6 +243,8 @@ mod tests { "GN3_TIMEOUT_SECS", "XAPIAN_DB_PATH", "SEARCH_ENGINE", + "GEMMA_WRAPPER_CMD", + "GEMMA_CACHE_DIR", ] { if std::env::var(key).is_ok() { eprintln!("skipping: {key} is set in the environment"); @@ -232,6 +254,8 @@ mod tests { let config = Config::load(); assert!(config.sql_uri.contains("db_webqtl")); assert_eq!(config.port, 3000); + assert_eq!(config.gemma_wrapper_cmd, "gemma-wrapper"); + assert!(config.gemma_cache_dir.ends_with("gn4-gemma")); assert_eq!(config.auth_server_url, "http://127.0.0.1:8081/"); assert_eq!(config.bind, "0.0.0.0"); assert!(config.test_token_enabled, "on by default"); diff --git a/src/gemma.rs b/src/gemma.rs new file mode 100644 index 0000000..b1f17ed --- /dev/null +++ b/src/gemma.rs @@ -0,0 +1,406 @@ +//! Core logic for POST /compute/gemma: compose and run gemma-wrapper +//! against a genotype file, a SNP annotation file and trait values, and +//! parse the resulting GEMMA association output into QTL mapping rows. +//! +//! Mirrors gn3/computations/gemma.py and gn3/api/gemma.py +//! (GeneNetwork3), but synchronously: axum/tokio can await an external +//! process directly, so there is no need for GeneNetwork3's Redis job +//! queue. The actual invocation also differs from GeneNetwork3's in two +//! respects, both found by running the real gemma-wrapper/GEMMA binaries +//! against the BXD fixtures rather than trusting GeneNetwork3's own +//! (entirely @unittest.skip'd) integration tests -- see `compute_gwa` +//! and `compute` below. + +use std::io; +use std::path::Path; +use std::process::Output; + +use serde_json::Value; +use tokio::process::Command; + +/// One phenotype value per strain (genotype-file order), or `None` for +/// a missing ("NA") sample. +pub type TraitValues = Vec>; + +#[derive(Debug)] +pub enum GemmaError { + /// Failed to spawn or run a child process. + Io(io::Error), + /// gemma-wrapper exited non-zero; carries its stderr. + CommandFailed { step: &'static str, stderr: String }, + /// gemma-wrapper's own --json stdout record was not valid JSON, or + /// reported an error (its own `errno` field was non-zero). + WrapperError { step: &'static str, detail: String }, + /// The real GEMMA `.assoc.txt` file gemma-wrapper pointed to could + /// not be found or read. + AssocFile(io::Error), + /// gemma-wrapper's GWA output archive did not list any files. + NoAssocFiles, +} + +impl std::fmt::Display for GemmaError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + GemmaError::Io(e) => write!(f, "failed to run gemma-wrapper: {e}"), + GemmaError::CommandFailed { step, stderr } => { + write!(f, "gemma-wrapper ({step}) failed: {stderr}") + } + GemmaError::WrapperError { step, detail } => { + write!(f, "gemma-wrapper ({step}) reported an error: {detail}") + } + GemmaError::AssocFile(e) => write!(f, "could not read GEMMA association output: {e}"), + GemmaError::NoAssocFiles => { + write!(f, "gemma-wrapper's response listed no association output files") + } + } + } +} + +/// Write `values` as a GEMMA/BIMBAM phenotype file: one value per line, +/// "NA" for a missing sample. Mirrors +/// gn3.computations.gemma.generate_pheno_txt_file, minus its hash-based +/// cache filename (the caller picks the path; see `compute`). +pub fn write_pheno_file(path: &Path, values: &TraitValues) -> io::Result<()> { + let mut contents = String::new(); + for value in values { + match value { + Some(v) => contents.push_str(&v.to_string()), + None => contents.push_str("NA"), + } + contents.push('\n'); + } + std::fs::write(path, contents) +} + +async fn run(cmd: &str, args: &[&str]) -> io::Result { + Command::new(cmd).args(args).output().await +} + +fn parse_wrapper_output(step: &'static str, output: Output) -> Result { + if !output.status.success() { + return Err(GemmaError::CommandFailed { + step, + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + }); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let record: Value = serde_json::from_str(&stdout).map_err(|e| GemmaError::WrapperError { + step, + detail: format!("could not parse gemma-wrapper's --json output: {e}"), + })?; + if record.get("errno").and_then(Value::as_i64).unwrap_or(0) != 0 { + return Err(GemmaError::WrapperError { + step, + detail: record + .get("error") + .and_then(Value::as_str) + .unwrap_or("gemma-wrapper reported a non-zero errno") + .to_string(), + }); + } + Ok(record) +} + +/// Run gemma-wrapper's `--json` K-matrix step and return its stdout +/// record (parsed JSON). +/// +/// `--loco` (bare, no chromosome list) is required here: confirmed by +/// actually running gemma-wrapper against the BXD fixtures, a plain +/// (non-LOCO) `-gk` only ever leaves its kinship matrix inside a +/// `.tar.xz` archive in `--cache-dir`, with no loose file at the path +/// the non-LOCO branch of gemma-wrapper's own GWA step later assumes -- +/// that combination errors out (`gemma: ... open file ... CheckParam`). +/// `--loco` is the combination gemma-wrapper's own USAGE banner +/// documents and the one whose GWA-side code path actually unpacks the +/// archive it needs (see `compute_gwa` and `compute`). Passing it with +/// no chromosome list makes gemma-wrapper derive the full chromosome +/// list from the SNP annotation file itself. +async fn compute_kinship( + gemma_wrapper_cmd: &str, + cache_dir: &str, + genotype_file: &str, + pheno_file: &str, + snps_file: &str, +) -> Result { + let output = run( + gemma_wrapper_cmd, + &[ + "--json", + "--cache-dir", + cache_dir, + "--loco", + "--", + "-g", + genotype_file, + "-p", + pheno_file, + "-a", + snps_file, + "-gk", + ], + ) + .await + .map_err(GemmaError::Io)?; + parse_wrapper_output("kinship", output) +} + +/// Run gemma-wrapper's `--json` GWA (association) step, given the +/// K-step's JSON record written to `k_json_path` (gemma-wrapper's +/// `--input` takes a file path, not stdin), and return its stdout +/// record. `--loco` is required here for the same reason as in +/// `compute_kinship`. +/// +/// Note: unlike GeneNetwork3's `generate_gemma_cmd` +/// (gn3/computations/gemma.py), which always appends a literal "-gk" to +/// the forwarded GEMMA args regardless of whether an LMM run was +/// requested, this passes `-lmm` without `-gk`, matching gemma-wrapper's +/// own documented usage (its USAGE banner in bin/gemma-wrapper). +/// gemma-wrapper decides K-only vs. GWA purely by whether "-gk" is +/// present in those args (`DO_COMPUTE_KINSHIP = +/// gemma_args.include?("-gk")`); passing both flags together, as +/// GeneNetwork3 does, makes every one of its calls a K-only run, and the +/// `-lmm` value is silently ignored. That GeneNetwork3 code path has no +/// non-skipped test covering it (tests/integration/test_gemma.py is +/// `@unittest.skip`), so it is not knowingly replicated here. +async fn compute_gwa( + gemma_wrapper_cmd: &str, + cache_dir: &str, + genotype_file: &str, + pheno_file: &str, + snps_file: &str, + lmm: u8, + k_json_path: &str, +) -> Result { + let lmm_str = lmm.to_string(); + let output = run( + gemma_wrapper_cmd, + &[ + "--json", + "--cache-dir", + cache_dir, + "--loco", + "--input", + k_json_path, + "--", + "-g", + genotype_file, + "-p", + pheno_file, + "-a", + snps_file, + "-lmm", + &lmm_str, + ], + ) + .await + .map_err(GemmaError::Io)?; + parse_wrapper_output("gwa", output) +} + +/// The association output basenames gemma-wrapper's GWA-step JSON +/// record points at: `record["files"]` is a list of +/// `[chr, log_basename, data_basename]`, one entry per chromosome in a +/// `--loco` run (see `compute`). +fn assoc_file_basenames(record: &Value) -> Vec<&str> { + record + .get("files") + .and_then(Value::as_array) + .map(|files| { + files + .iter() + .filter_map(|entry| entry.as_array()?.get(2)?.as_str()) + .collect() + }) + .unwrap_or_default() +} + +/// One row of a GEMMA `.assoc.txt` file, keyed by its header column +/// names. Columns vary with `-lmm` (1/2/3 run one test; 4 runs all +/// three), so this stays a generic map rather than a fixed struct. +pub type AssocRow = serde_json::Map; + +/// Parse a GEMMA `.assoc.txt` file (tab-separated, header row first) +/// into one row per marker. A value that parses as a number is kept +/// numeric; "NA" and anything else stays a string. +fn parse_assoc_file(contents: &str) -> Vec { + let mut lines = contents.lines(); + let Some(header) = lines.next() else { + return Vec::new(); + }; + let columns: Vec<&str> = header.split('\t').collect(); + lines + .filter(|line| !line.is_empty()) + .map(|line| { + let mut row = AssocRow::new(); + for (column, value) in columns.iter().zip(line.split('\t')) { + let json_value = value + .parse::() + .map(|n| serde_json::json!(n)) + .unwrap_or_else(|_| Value::String(value.to_string())); + row.insert((*column).to_string(), json_value); + } + row + }) + .collect() +} + +/// Extract `archive` (a `.tar.xz`, relative to `cache_dir`) into +/// `dest_dir`. Confirmed necessary by actually running the real +/// binaries: a `--loco` GWA run's `.assoc.txt` files (one per +/// chromosome) only ever end up inside this archive, never as loose +/// files in `cache_dir` -- unlike the kinship step's `.cXX.txt` files, +/// which do also appear loose (gemma-wrapper itself is inconsistent +/// about this between the two steps). +async fn extract_archive(cache_dir: &str, archive: &str, dest_dir: &str) -> Result<(), GemmaError> { + let archive_path = Path::new(cache_dir).join(archive); + let output = run( + "tar", + &[ + "xJf", + &archive_path.to_string_lossy(), + "-C", + dest_dir, + ], + ) + .await + .map_err(GemmaError::Io)?; + if !output.status.success() { + return Err(GemmaError::CommandFailed { + step: "extract-archive", + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + }); + } + Ok(()) +} + +fn nanos_since_epoch() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) +} + +/// Run the full gemma-wrapper flow -- compute kinship, then run the LMM +/// association test, both `--loco` -- and return the mapped QTL as a +/// JSON array of rows (each a JSON object), concatenated across every +/// chromosome's `.assoc.txt` from the real GEMMA output. The result is +/// one `Value::Array`, not a `Vec`: utoipa (for the OpenAPI +/// schema) implements `ToSchema` for `serde_json::Value` itself, but +/// `Vec` needs `Value: ComposeSchema`, which it does +/// not implement in this utoipa version. +pub async fn compute( + gemma_wrapper_cmd: &str, + cache_dir: &str, + genotype_file: &str, + snps_file: &str, + trait_values: &TraitValues, + lmm: u8, +) -> Result { + std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?; + let unique = format!("{}-{}", std::process::id(), nanos_since_epoch()); + let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt")); + write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?; + let pheno_file = pheno_path.to_string_lossy().into_owned(); + + let k_record = compute_kinship( + gemma_wrapper_cmd, + cache_dir, + genotype_file, + &pheno_file, + snps_file, + ) + .await?; + let k_json_path = Path::new(cache_dir).join(format!("{unique}-k.json")); + std::fs::write(&k_json_path, k_record.to_string()).map_err(GemmaError::Io)?; + + let gwa_record = compute_gwa( + gemma_wrapper_cmd, + cache_dir, + genotype_file, + &pheno_file, + snps_file, + lmm, + &k_json_path.to_string_lossy(), + ) + .await?; + + let basenames = assoc_file_basenames(&gwa_record); + if basenames.is_empty() { + return Err(GemmaError::NoAssocFiles); + } + let archive = gwa_record + .get("archive") + .and_then(Value::as_str) + .ok_or_else(|| GemmaError::WrapperError { + step: "gwa", + detail: "gemma-wrapper's response did not list an output archive".to_string(), + })?; + let extract_dir = Path::new(cache_dir).join(format!("{unique}-gwa-extract")); + std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?; + extract_archive(cache_dir, archive, &extract_dir.to_string_lossy()).await?; + + let mut rows = Vec::new(); + for basename in basenames { + let contents = std::fs::read_to_string(extract_dir.join(basename)) + .map_err(GemmaError::AssocFile)?; + rows.extend(parse_assoc_file(&contents).into_iter().map(Value::Object)); + } + let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless + Ok(Value::Array(rows)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn write_pheno_file_writes_one_value_per_line_and_na_for_missing() { + let path = std::env::temp_dir().join(format!("gn4-gemma-test-{}", std::process::id())); + write_pheno_file(&path, &vec![Some(9.944), None, Some(8.572)]).unwrap(); + let contents = std::fs::read_to_string(&path).unwrap(); + std::fs::remove_file(&path).unwrap(); + assert_eq!(contents, "9.944\nNA\n8.572\n"); + } + + #[test] + fn parse_assoc_file_parses_header_and_numeric_rows() { + let contents = "chr\trs\tps\tp_wald\n1\trs001\t3010274\t0.05\n1\trs002\t3492195\tNA\n"; + let rows = parse_assoc_file(contents); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0]["chr"], serde_json::json!(1.0)); + assert_eq!(rows[0]["rs"], serde_json::json!("rs001")); + assert_eq!(rows[0]["p_wald"], serde_json::json!(0.05)); + assert_eq!(rows[1]["p_wald"], serde_json::json!("NA")); + } + + #[test] + fn parse_assoc_file_empty_contents_yields_no_rows() { + assert_eq!(parse_assoc_file("").len(), 0); + } + + #[test] + fn assoc_file_basenames_reads_the_third_element_of_every_file_entry() { + let record = serde_json::json!({ + "files": [["1", "a.log.txt", "a.assoc.txt"], ["2", "b.log.txt", "b.assoc.txt"]] + }); + assert_eq!(assoc_file_basenames(&record), vec!["a.assoc.txt", "b.assoc.txt"]); + } + + #[test] + fn assoc_file_basenames_missing_files_yields_empty() { + assert_eq!(assoc_file_basenames(&serde_json::json!({})), Vec::<&str>::new()); + } + + #[test] + fn parse_wrapper_output_rejects_non_zero_errno() { + use std::os::unix::process::ExitStatusExt; + let output = Output { + status: std::process::ExitStatus::from_raw(0), + stdout: br#"{"errno": 1, "error": "boom"}"#.to_vec(), + stderr: Vec::new(), + }; + let err = parse_wrapper_output("kinship", output).unwrap_err(); + assert!(matches!(err, GemmaError::WrapperError { step: "kinship", .. })); + assert_eq!(err.to_string(), "gemma-wrapper (kinship) reported an error: boom"); + } +} diff --git a/src/lib.rs b/src/lib.rs index cfe2c59..ab88fff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ pub mod config; mod db; pub mod embedded; mod error; +mod gemma; pub mod response; mod routes; pub mod search; @@ -76,7 +77,8 @@ pub fn app() -> Router { .routes(routes!(routes::traits::trait_values)) .routes(routes!(routes::publications::publication_traits)) .routes(routes!(routes::search::search)) - .routes(routes!(routes::correlation::correlation)); + .routes(routes!(routes::correlation::correlation)) + .routes(routes!(routes::gemma::compute_gemma)); // Registered -- and so reachable and listed in /openapi.json -- unless // GENENETWORK4_TEST_TOKEN switches it off; when registered it decides // per request whether the caller is local. See src/routes/dev.rs. diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs new file mode 100644 index 0000000..1506f1b --- /dev/null +++ b/src/routes/gemma.rs @@ -0,0 +1,90 @@ +//! Compute domain: /compute/gemma invokes gemma-wrapper (and the real +//! GEMMA binary it wraps) to map a QTL from a genotype file, a SNP +//! annotation file and a list of trait values. See `crate::gemma` for +//! the invocation itself. + +use axum::Json; +use axum::http::StatusCode; +use serde::Deserialize; + +use crate::auth::AuthToken; +use crate::config::Config; +use crate::error::{ApiError, Unauthorized}; +use crate::gemma::{self, GemmaError}; +use crate::response::{Ctx, Envelope, RequestInfo}; + +#[derive(Deserialize, utoipa::ToSchema)] +pub struct GemmaRequest { + /// Path to a BIMBAM genotype file (plain text or gzipped), passed + /// as GEMMA's `-g`. + pub genotype_file: String, + /// Path to a SNP annotation file (rs id, position, chromosome), + /// passed as GEMMA's `-a`. + pub snps_file: String, + /// One phenotype value per strain, in the genotype file's strain + /// order; `null` for a missing value. + pub trait_values: Vec>, + /// GEMMA's `-lmm` model selector: 1 = Wald, 2 = LRT, 3 = score, + /// 4 = all three. Defaults to 4 when omitted. + pub lmm: Option, +} + +#[utoipa::path(post, path = "/compute/gemma", + summary = "Map a QTL from a genotype file and trait values using GEMMA", + description = "Runs gemma-wrapper's K-matrix step, then its LMM \ +association step, over the given genotype and SNP annotation files and \ +trait values, and returns the resulting rows of GEMMA's own \ +.assoc.txt output.", + request_body = GemmaRequest, + responses( + (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as a JSON array of row objects", body = Envelope), + Unauthorized, + (status = 422, description = "lmm out of range (must be 1-4)", body = ApiError), + (status = 502, description = "gemma-wrapper or GEMMA failed", body = ApiError) + ))] +pub async fn compute_gemma( + ctx: Ctx, + _token: AuthToken, + Json(request): Json, +) -> Result>, ApiError> { + let rid = ctx.request_info(); + let lmm = request.lmm.unwrap_or(4); + if !(1..=4).contains(&lmm) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_lmm", + "Invalid lmm", + format!("lmm must be 1-4 (got {lmm}); 1=Wald, 2=LRT, 3=score, 4=all three."), + "compute_gemma", + ) + .with_request_id(rid)); + } + + let config = Config::load(); + gemma::compute( + &config.gemma_wrapper_cmd, + &config.gemma_cache_dir, + &request.genotype_file, + &request.snps_file, + &request.trait_values, + lmm, + ) + .await + .map(|rows| { + Json(ctx.envelope(rows).with_note( + "Rows come from GEMMA's own .assoc.txt output; column names depend on the lmm value used.", + )) + }) + .map_err(|e| gemma_error_to_api_error(e, rid)) +} + +fn gemma_error_to_api_error(error: GemmaError, rid: RequestInfo) -> ApiError { + ApiError::new( + StatusCode::BAD_GATEWAY, + "gemma_failed", + "GEMMA computation failed", + error.to_string(), + "compute_gemma", + ) + .with_request_id(rid) +} diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 55227e0..3a9f3e3 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -7,6 +7,7 @@ pub mod accessions; pub mod correlation; pub mod datasets; pub mod dev; +pub mod gemma; pub mod groups; pub mod mapping_methods; pub mod publications; -- cgit 1.4.1 From 53ae1b4cbf8bd68278e7a59cd41a4c145f8362ab Mon Sep 17 00:00:00 2001 From: Pjotr Date: Sun, 11 Oct 2026 20:07:40 +0000 Subject: gemma: default lmm to 9, matching GeneNetwork's own convention GeneNetwork2 (wqflask/marker_regression/gemma_mapping.py) and gn-guile (gn/runner/gemma.scm) both always use -lmm 9 in their working --loco invocations. Checked by running it: gemma-gn2 accepts 9 fine, and the resulting .assoc.txt carries p_lrt, not p_wald -- some mapping other than stock GEMMA's 1-4 scale, not independently verified here, but consistent with what both of those codebases rely on in production. Widened validation from a 1-4 range to the explicit set {1, 2, 3, 4, 9}: 9 is now known-good, but 5-8 remain untested and stock-invalid, so they're still rejected rather than guessed at. Updated the real end-to-end test to assert on p_lrt (what lmm=9 actually produces) instead of p_wald, and moved the out-of-range-lmm test from 9 (now valid) to 5. Verified: cargo test passes in full, including the real gemma-wrapper/ GEMMA run against the BXD fixtures with the new default. Co-Authored-By: Claude Sonnet 5 --- src/gemma.rs | 6 ++++-- src/routes/gemma.rs | 24 ++++++++++++++++++------ tests/gemma.rs | 8 ++++++-- 3 files changed, 28 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/gemma.rs b/src/gemma.rs index b1f17ed..eee16b6 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -215,8 +215,10 @@ fn assoc_file_basenames(record: &Value) -> Vec<&str> { } /// One row of a GEMMA `.assoc.txt` file, keyed by its header column -/// names. Columns vary with `-lmm` (1/2/3 run one test; 4 runs all -/// three), so this stays a generic map rather than a fixed struct. +/// names. Columns vary with `-lmm` (stock 1/2/3 run one test, 4 runs +/// all three; this build's `9`, GeneNetwork's convention, was observed +/// to carry `p_lrt`), so this stays a generic map rather than a fixed +/// struct. pub type AssocRow = serde_json::Map; /// Parse a GEMMA `.assoc.txt` file (tab-separated, header row first) diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs index 1506f1b..03b96bc 100644 --- a/src/routes/gemma.rs +++ b/src/routes/gemma.rs @@ -24,8 +24,15 @@ pub struct GemmaRequest { /// One phenotype value per strain, in the genotype file's strain /// order; `null` for a missing value. pub trait_values: Vec>, - /// GEMMA's `-lmm` model selector: 1 = Wald, 2 = LRT, 3 = score, - /// 4 = all three. Defaults to 4 when omitted. + /// GEMMA's `-lmm` model selector: stock values are 1 = Wald, + /// 2 = LRT, 3 = score, 4 = all three. Defaults to 9 when omitted -- + /// not a stock value, but the one GeneNetwork2 + /// (wqflask/marker_regression/gemma_mapping.py) and gn-guile + /// (gn/runner/gemma.scm) both use, and the only value this has been + /// run against in the --loco shape (see `crate::gemma`). Confirmed + /// by actually running it: `gemma-gn2` accepts it and the output + /// carries `p_lrt`, not `p_wald` -- some other mapping than the + /// stock 1-4 scale, not independently verified here. pub lmm: Option, } @@ -39,7 +46,7 @@ trait values, and returns the resulting rows of GEMMA's own \ responses( (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as a JSON array of row objects", body = Envelope), Unauthorized, - (status = 422, description = "lmm out of range (must be 1-4)", body = ApiError), + (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9)", body = ApiError), (status = 502, description = "gemma-wrapper or GEMMA failed", body = ApiError) ))] pub async fn compute_gemma( @@ -48,13 +55,18 @@ pub async fn compute_gemma( Json(request): Json, ) -> Result>, ApiError> { let rid = ctx.request_info(); - let lmm = request.lmm.unwrap_or(4); - if !(1..=4).contains(&lmm) { + let lmm = request.lmm.unwrap_or(9); + // Only the values actually run against this GEMMA build (see the + // `lmm` field doc above) are allowed; 5-8 are stock-invalid and + // untested here, so they're rejected rather than guessed at. + if !matches!(lmm, 1 | 2 | 3 | 4 | 9) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_lmm", "Invalid lmm", - format!("lmm must be 1-4 (got {lmm}); 1=Wald, 2=LRT, 3=score, 4=all three."), + format!( + "lmm must be one of 1, 2, 3, 4, 9 (got {lmm}); 1=Wald, 2=LRT, 3=score, 4=all three, 9=GeneNetwork's convention (observed to behave like LRT)." + ), "compute_gemma", ) .with_request_id(rid)); diff --git a/tests/gemma.rs b/tests/gemma.rs index 2529174..d08a088 100644 --- a/tests/gemma.rs +++ b/tests/gemma.rs @@ -66,7 +66,9 @@ async fn compute_gemma_maps_a_qtl_from_the_bxd_fixtures() { let first = &rows[0]; assert!(first.get("chr").is_some(), "{first}"); assert!(first.get("rs").is_some(), "{first}"); - assert!(first.get("p_wald").is_some(), "lmm defaults to 4 (all tests): {first}"); + // lmm defaults to 9 (GeneNetwork's convention); observed to behave + // like LRT, carrying p_lrt rather than p_wald. + assert!(first.get("p_lrt").is_some(), "{first}"); } #[tokio::test] @@ -80,7 +82,9 @@ async fn compute_gemma_rejects_an_out_of_range_lmm() { "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": [1.0, 2.0], - "lmm": 9, + // 9 is now accepted (GeneNetwork's convention); 5-8 are stock- + // invalid and untested, so they're still rejected. + "lmm": 5, }); let response = common::post_json_with_token( "POST /compute/gemma (bad lmm)", -- cgit 1.4.1 From a60d3a1bc21e355b5342747be0ede3654d2cce41 Mon Sep 17 00:00:00 2001 From: Pjotr Date: Mon, 12 Oct 2026 06:38:08 +0000 Subject: gemma: document the endpoint in README; put the actual gemma-wrapper commands in the response notes README: added /compute/gemma to the endpoint table and a dedicated section with a real example response, and the explicit caveat the user asked for -- not feasible past roughly 100,000 markers at this point (synchronous, sequential, no parallel LOCO jobs; the BXD fixtures, 7,320 markers, already take ~19s, and real datasets go up to ~1M markers). notes: the response now carries the exact gemma-wrapper invocations used for the kinship and GWA steps ("Kinship: gemma-wrapper ... -gk", "GWA: gemma-wrapper ... -lmm 9"), not just a generic description. kinship_args/gwa_args build the argument list once and feed both the actual subprocess call and the note text, so the two can't drift apart. Added unit tests asserting -gk and -lmm never appear together, and extended the real end-to-end test to check both notes' shape. Verified: cargo test passes in full, including the real gemma-wrapper/ GEMMA run, with the notes field inspected directly against the actual command lines used. Co-Authored-By: Claude Sonnet 5 --- README.md | 59 ++++++++++++++++++ src/gemma.rs | 170 ++++++++++++++++++++++++++++++++++++++-------------- src/routes/gemma.rs | 10 +++- tests/gemma.rs | 15 +++++ 4 files changed, 207 insertions(+), 47 deletions(-) (limited to 'src') diff --git a/README.md b/README.md index 4cde289..66de220 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ A Rust/Axum-based API server for GeneNetwork. | GET | `/traits/{dataset}/{trait}` | One trait's metadata, ProbeSet or Publish. | | GET | `/traits/{dataset}/{trait}/values` | One trait's per-strain values, either kind. | | POST | `/compute/correlation` | Correlate a trait (STUB: canned data). | +| POST | `/compute/gemma` | Map a QTL with GEMMA (real; not feasible yet past ~100K markers -- see below). | | POST | `/get-test-token` | Local callers only: get a bearer token to test with. | | GET | `/search` | Search traits (free-text; xapian query syntax). | | GET | `/openapi.json` | OpenAPI 3.1 document for all of the above. | @@ -879,6 +880,64 @@ accepts: the `PublishFreeze`'s own `Name`/`ShortName`/`Id` (e.g. - `GET /openapi.json` - returns an OpenAPI 3.1 document describing the endpoints above; see [OpenAPI](#openapi) below. +### `/compute/gemma` + +Maps a QTL by running the real `gemma-wrapper`/GEMMA binaries: a +K-matrix step, then an LMM association step, both `--loco`, over a +genotype file, a SNP annotation file and a list of trait values. See +`src/gemma.rs` for the invocation itself and why it's shaped this way +(found by running GeneNetwork2's and GeneNetwork3's own gemma code for +real against the BXD fixtures -- GeneNetwork3's always errors out; +GeneNetwork2's non-LOCO branch does too; only `--loco` works). + +This runs synchronously inside the request -- there is no job queue -- +and it runs sequentially, not per-chromosome in parallel. **Not +feasible past roughly 100,000 markers at this point.** The BXD +fixtures (7,320 markers, LOCO across 20 chromosomes) already take +about 19 seconds; some real GeneNetwork datasets have upwards of 1 +million markers, which this endpoint cannot handle yet -- the request +would simply run for a very long time inside one HTTP call. Making +that feasible needs either parallelising the per-chromosome LOCO jobs +(`gemma-wrapper --parallel`, which gn-guile's own precompute script +already uses) or moving the computation out of the request/response +cycle entirely (a job queue, or an offline precompute step, the way +gn-guile's own `scripts/precompute/run-gemma.scm` already does it). + +```json +{ + "data": [ + {"chr": 1.0, "rs": "rs31443144", "ps": 3010274.0, "n_miss": 0.0, + "allele1": "X", "allele0": "Y", "af": 0.276, "beta": 0.1533063, + "se": 0.1255921, "l_mle": 0.00001, "p_lrt": 0.2178407}, + ... + {"chr": 1.0, "rs": "rs8253327", "ps": 191908118.0, "n_miss": 0.0, + "allele1": "X", "allele0": "Y", "af": 0.284, "beta": 0.4281751, + "se": 0.1158946, "l_mle": 0.00001, "p_lrt": 0.0003520103} + ], + "meta": { + "state": "complete", + "took_ms": 18456, + "request_id": "ade51981-4417-47cd-93d7-70dbd9da1ecc" + }, + "notes": [ + "Rows come from GEMMA's own .assoc.txt output; column names depend on the lmm value used.", + "Kinship: gemma-wrapper --json --cache-dir /tmp/gn4-gemma --loco -- -g .../BXD_geno.txt.gz -p /tmp/gn4-gemma/492514-...-pheno.txt -a .../BXD_snps.txt -gk", + "GWA: gemma-wrapper --json --cache-dir /tmp/gn4-gemma --loco --input /tmp/gn4-gemma/492514-...-k.json -- -g .../BXD_geno.txt.gz -p /tmp/gn4-gemma/492514-...-pheno.txt -a .../BXD_snps.txt -lmm 9" + ], + "next": [] +} +``` + +(`data` has one row per marker -- 7,320 rows in this real run against +the BXD fixtures, trimmed to two here. The last two `notes` are the +exact gemma-wrapper invocations used (paths shortened here for +readability) -- `-gk` only on the kinship step, `-lmm` only on the GWA +step, never both together; see `src/gemma.rs`'s `compute_gwa` doc +comment for why that distinction matters. `lmm` defaults to 9, matching +GeneNetwork2's and gn-guile's own convention; see the field's own doc +comment in `src/routes/gemma.rs` for what's actually been verified +about it.) + ## OpenAPI `GET /openapi.json` serves an OpenAPI 3.1 document covering every route. diff --git a/src/gemma.rs b/src/gemma.rs index eee16b6..9c74b30 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -122,25 +122,10 @@ async fn compute_kinship( pheno_file: &str, snps_file: &str, ) -> Result { - let output = run( - gemma_wrapper_cmd, - &[ - "--json", - "--cache-dir", - cache_dir, - "--loco", - "--", - "-g", - genotype_file, - "-p", - pheno_file, - "-a", - snps_file, - "-gk", - ], - ) - .await - .map_err(GemmaError::Io)?; + let args = kinship_args(cache_dir, genotype_file, pheno_file, snps_file); + let output = run(gemma_wrapper_cmd, &args) + .await + .map_err(GemmaError::Io)?; parse_wrapper_output("kinship", output) } @@ -172,28 +157,17 @@ async fn compute_gwa( k_json_path: &str, ) -> Result { let lmm_str = lmm.to_string(); - let output = run( - gemma_wrapper_cmd, - &[ - "--json", - "--cache-dir", - cache_dir, - "--loco", - "--input", - k_json_path, - "--", - "-g", - genotype_file, - "-p", - pheno_file, - "-a", - snps_file, - "-lmm", - &lmm_str, - ], - ) - .await - .map_err(GemmaError::Io)?; + let args = gwa_args( + cache_dir, + genotype_file, + pheno_file, + snps_file, + &lmm_str, + k_json_path, + ); + let output = run(gemma_wrapper_cmd, &args) + .await + .map_err(GemmaError::Io)?; parse_wrapper_output("gwa", output) } @@ -282,10 +256,73 @@ fn nanos_since_epoch() -> u128 { .unwrap_or(0) } +/// The mapped QTL rows, plus the exact gemma-wrapper invocations used +/// to produce them (see `compute`'s `notes` below), so a caller can see +/// precisely what was run without reading server logs. +pub struct GemmaOutput { + pub rows: Value, + pub notes: Vec, +} + +/// The gemma-wrapper arguments for the kinship step, shared between the +/// actual invocation (`compute_kinship`) and the note describing it +/// (`compute`), so the two can never drift apart. +fn kinship_args<'a>( + cache_dir: &'a str, + genotype_file: &'a str, + pheno_file: &'a str, + snps_file: &'a str, +) -> Vec<&'a str> { + vec![ + "--json", + "--cache-dir", + cache_dir, + "--loco", + "--", + "-g", + genotype_file, + "-p", + pheno_file, + "-a", + snps_file, + "-gk", + ] +} + +/// The gemma-wrapper arguments for the GWA step, same pairing as +/// `kinship_args`. +fn gwa_args<'a>( + cache_dir: &'a str, + genotype_file: &'a str, + pheno_file: &'a str, + snps_file: &'a str, + lmm_str: &'a str, + k_json_path: &'a str, +) -> Vec<&'a str> { + vec![ + "--json", + "--cache-dir", + cache_dir, + "--loco", + "--input", + k_json_path, + "--", + "-g", + genotype_file, + "-p", + pheno_file, + "-a", + snps_file, + "-lmm", + lmm_str, + ] +} + /// Run the full gemma-wrapper flow -- compute kinship, then run the LMM /// association test, both `--loco` -- and return the mapped QTL as a /// JSON array of rows (each a JSON object), concatenated across every -/// chromosome's `.assoc.txt` from the real GEMMA output. The result is +/// chromosome's `.assoc.txt` from the real GEMMA output, plus the two +/// gemma-wrapper invocations used (`GemmaOutput::notes`). The rows are /// one `Value::Array`, not a `Vec`: utoipa (for the OpenAPI /// schema) implements `ToSchema` for `serde_json::Value` itself, but /// `Vec` needs `Value: ComposeSchema`, which it does @@ -297,13 +334,17 @@ pub async fn compute( snps_file: &str, trait_values: &TraitValues, lmm: u8, -) -> Result { +) -> Result { std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?; let unique = format!("{}-{}", std::process::id(), nanos_since_epoch()); let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt")); write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?; let pheno_file = pheno_path.to_string_lossy().into_owned(); + let kinship_note = format!( + "Kinship: {gemma_wrapper_cmd} {}", + kinship_args(cache_dir, genotype_file, &pheno_file, snps_file).join(" ") + ); let k_record = compute_kinship( gemma_wrapper_cmd, cache_dir, @@ -315,6 +356,19 @@ pub async fn compute( let k_json_path = Path::new(cache_dir).join(format!("{unique}-k.json")); std::fs::write(&k_json_path, k_record.to_string()).map_err(GemmaError::Io)?; + let lmm_str = lmm.to_string(); + let gwa_note = format!( + "GWA: {gemma_wrapper_cmd} {}", + gwa_args( + cache_dir, + genotype_file, + &pheno_file, + snps_file, + &lmm_str, + &k_json_path.to_string_lossy(), + ) + .join(" ") + ); let gwa_record = compute_gwa( gemma_wrapper_cmd, cache_dir, @@ -348,7 +402,10 @@ pub async fn compute( rows.extend(parse_assoc_file(&contents).into_iter().map(Value::Object)); } let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless - Ok(Value::Array(rows)) + Ok(GemmaOutput { + rows: Value::Array(rows), + notes: vec![kinship_note, gwa_note], + }) } #[cfg(test)] @@ -405,4 +462,29 @@ mod tests { assert!(matches!(err, GemmaError::WrapperError { step: "kinship", .. })); assert_eq!(err.to_string(), "gemma-wrapper (kinship) reported an error: boom"); } + + #[test] + fn kinship_args_is_loco_with_gk_and_no_lmm() { + let args = kinship_args("/cache", "geno.txt.gz", "pheno.txt", "snps.txt"); + assert_eq!( + args, + vec![ + "--json", "--cache-dir", "/cache", "--loco", "--", "-g", "geno.txt.gz", "-p", + "pheno.txt", "-a", "snps.txt", "-gk" + ] + ); + } + + #[test] + fn gwa_args_is_loco_with_lmm_and_no_gk() { + let args = gwa_args("/cache", "geno.txt.gz", "pheno.txt", "snps.txt", "9", "/cache/k.json"); + assert_eq!( + args, + vec![ + "--json", "--cache-dir", "/cache", "--loco", "--input", "/cache/k.json", "--", + "-g", "geno.txt.gz", "-p", "pheno.txt", "-a", "snps.txt", "-lmm", "9" + ] + ); + assert!(!args.contains(&"-gk"), "the GWA step must never carry -gk alongside -lmm"); + } } diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs index 03b96bc..3df86a7 100644 --- a/src/routes/gemma.rs +++ b/src/routes/gemma.rs @@ -82,10 +82,14 @@ pub async fn compute_gemma( lmm, ) .await - .map(|rows| { - Json(ctx.envelope(rows).with_note( + .map(|output| { + let mut envelope = ctx.envelope(output.rows).with_note( "Rows come from GEMMA's own .assoc.txt output; column names depend on the lmm value used.", - )) + ); + for note in &output.notes { + envelope = envelope.with_note(note); + } + Json(envelope) }) .map_err(|e| gemma_error_to_api_error(e, rid)) } diff --git a/tests/gemma.rs b/tests/gemma.rs index d08a088..9aa58d7 100644 --- a/tests/gemma.rs +++ b/tests/gemma.rs @@ -69,6 +69,21 @@ async fn compute_gemma_maps_a_qtl_from_the_bxd_fixtures() { // lmm defaults to 9 (GeneNetwork's convention); observed to behave // like LRT, carrying p_lrt rather than p_wald. assert!(first.get("p_lrt").is_some(), "{first}"); + + // The notes carry the exact gemma-wrapper invocations used, not + // just a generic description. + let envelope: Value = serde_json::from_slice(&bytes).unwrap(); + let notes = envelope["notes"].as_array().unwrap(); + let kinship_note = notes.iter().find(|n| n.as_str().unwrap().starts_with("Kinship:")); + let gwa_note = notes.iter().find(|n| n.as_str().unwrap().starts_with("GWA:")); + assert!(kinship_note.is_some(), "{notes:?}"); + assert!(gwa_note.is_some(), "{notes:?}"); + let kinship_note = kinship_note.unwrap().as_str().unwrap(); + let gwa_note = gwa_note.unwrap().as_str().unwrap(); + assert!(kinship_note.contains("--loco") && kinship_note.contains("-gk"), "{kinship_note}"); + assert!(!kinship_note.contains("-lmm"), "{kinship_note}"); + assert!(gwa_note.contains("--loco") && gwa_note.contains("-lmm 9"), "{gwa_note}"); + assert!(!gwa_note.contains("-gk"), "{gwa_note}"); } #[tokio::test] -- cgit 1.4.1 From b04db9ea05a73074afcf88300f8b6aa8c6e28e64 Mon Sep 17 00:00:00 2001 From: Pjotr Date: Mon, 12 Oct 2026 12:19:12 +0000 Subject: gemma: stream gemma-wrapper's progress; add GET /compute/gemma/status /compute/gemma can genuinely take minutes (see the README's ~100K marker caveat), and until now gave no visibility into a run already in flight. Same shape GeneNetwork2/3 use for R/qtl (gn3/computations/streaming.py, gn3/api/streaming.py): the caller picks a run id, passes it as POST /compute/gemma?id=, and polls GET /compute/gemma/status/{id}?peak= on a second connection while the first is still blocked. What's actually tailed is gemma-wrapper's stderr, not stdout: under --json, all of its human-readable progress (banner, chromosome list, "Executing: ...", per-step info) goes to stderr, and stdout carries only the one final JSON record, printed once at the very end -- so there is nothing to stream on stdout. run_streaming() replaces the old buffered Command::output() call with one that reads stderr line by line as the process runs, appending each line to a log file if one was given, across both the kinship and GWA steps into a single continuous log; stdout is still read in full (unchanged, one JSON blob). A completion marker ("GN4: computation complete") is appended once the whole request finishes, the same convention GeneNetwork3 uses ("Done with GN3 Computation" in gn3/api/rqtl2.py) under this project's own name. run_id is validated (1-64 ASCII letters/digits/-/_) before it's ever joined into a path, on both the POST's ?id= and the status endpoint's path segment -- GeneNetwork3's own /api/stream/{run_id} does no such check on its equivalent id, which is not knowingly replicated here. Verified end to end, not just unit tested: the integration test spawns the POST on its own task and polls the status endpoint from this one while the ~19s real computation is still running, asserting it actually observed non-empty progress lines mid-run (not merely after the fact), then checks the finished log for both real gemma-wrapper progress text and the completion marker. cargo test passes in full. Co-Authored-By: Claude Sonnet 5 --- Cargo.toml | 2 +- README.md | 45 +++++++++++++ src/gemma.rs | 182 ++++++++++++++++++++++++++++++++++++++++++++++++---- src/lib.rs | 3 +- src/routes/gemma.rs | 122 +++++++++++++++++++++++++++++++++-- tests/gemma.rs | 154 ++++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 481 insertions(+), 27 deletions(-) (limited to 'src') diff --git a/Cargo.toml b/Cargo.toml index bef2027..805ab58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls" rusqlite = "0.40.2" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" -tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "process"] } +tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "process", "time"] } tower-http = { version = "0.7.1", features = ["request-id", "util"] } ureq = { version = "3.2.1", features = ["json"] } url = "2" diff --git a/README.md b/README.md index 66de220..a8cc7e2 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ A Rust/Axum-based API server for GeneNetwork. | GET | `/traits/{dataset}/{trait}/values` | One trait's per-strain values, either kind. | | POST | `/compute/correlation` | Correlate a trait (STUB: canned data). | | POST | `/compute/gemma` | Map a QTL with GEMMA (real; not feasible yet past ~100K markers -- see below). | +| GET | `/compute/gemma/status/{run_id}` | Poll a `/compute/gemma` call's progress while it's running. | | POST | `/get-test-token` | Local callers only: get a bearer token to test with. | | GET | `/search` | Search traits (free-text; xapian query syntax). | | GET | `/openapi.json` | OpenAPI 3.1 document for all of the above. | @@ -938,6 +939,50 @@ GeneNetwork2's and gn-guile's own convention; see the field's own doc comment in `src/routes/gemma.rs` for what's actually been verified about it.) +#### `/compute/gemma/status/{run_id}` -- polling a run's progress + +Given the endpoint above can genuinely take minutes, pass `?id=` on +the `POST` and poll this on a second connection while it's in flight +to show progress, the same log-tailing shape GeneNetwork2/3 use for +R/qtl (`gn3/computations/streaming.py`, `gn3/api/streaming.py`): the +run's `id` becomes a log file (`{cache_dir}/{id}.log`); gemma-wrapper's +own stderr (its `--json`-mode progress banner, chromosome list, +"Executing: ...", per-step info -- its stdout carries only the one +final JSON record, so there's nothing to stream there) is appended to +it line by line as both the kinship and GWA steps run; once the whole +request finishes, `"GN4: computation complete"` is appended as a +sentinel so a poller can tell "still running" from "done". `run_id` +must be 1-64 ASCII letters/digits/`-`/`_` -- it's a path component, so +this is enforced on both this endpoint and the `?id=` that names it. + +``` +POST /compute/gemma?id=my-run-1 (on one connection; blocks until done) +GET /compute/gemma/status/my-run-1?peak=0 (on another, while the first is in flight) +``` + +```json +{ + "data": { + "run_id": "my-run-1", + "lines": [ + "gemma-wrapper 1.00-pre1 (Ruby 3.3.9) by Pjotr Prins 2017-2024", + "Using GEMMA 0.98.6 (2022-08-05) by Xiang Zhou, Pjotr Prins and team (C) 2012-2022", + "Executing: time -v .../gemma -o ... -g ... -p ... -a ... -gk -outdir ..." + ], + "pointer": 612 + }, + "meta": {"state": "complete", "took_ms": 1, "request_id": "..."}, + "notes": [], + "next": [] +} +``` + +Pass `pointer` back as `?peak=` on the next poll; `lines` is empty +(not an error) when nothing new has been written since. An unknown +`run_id` (never started, or the main request hasn't written its log +yet) is a 404, not an empty response -- genuinely different from +"nothing new yet". + ## OpenAPI `GET /openapi.json` serves an OpenAPI 3.1 document covering every route. diff --git a/src/gemma.rs b/src/gemma.rs index 9c74b30..a42c201 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -12,10 +12,12 @@ //! and `compute` below. use std::io; +use std::io::{Read, Seek, SeekFrom, Write}; use std::path::Path; -use std::process::Output; +use std::process::{Output, Stdio}; use serde_json::Value; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; use tokio::process::Command; /// One phenotype value per strain (genotype-file order), or `None` for @@ -72,8 +74,54 @@ pub fn write_pheno_file(path: &Path, values: &TraitValues) -> io::Result<()> { std::fs::write(path, contents) } -async fn run(cmd: &str, args: &[&str]) -> io::Result { - Command::new(cmd).args(args).output().await +/// Run `cmd`, optionally tailing its stderr into `log_path` one line at +/// a time as it runs, and return the same `Output` shape +/// `Command::output()` would have (full stdout, full stderr, exit +/// status) -- callers that don't pass `log_path` see identical +/// behaviour to the plain, non-streaming call it replaces. +/// +/// stderr, not stdout, is what's tailed: gemma-wrapper's own `--json` +/// mode sends all of its human-readable progress (banner, chromosome +/// list, "Executing: ...", per-step info/debug lines) to stderr, and +/// reserves stdout for the one final JSON record, printed once at +/// the very end (`OUTPUT = (options[:json] ? $stderr : $stdout)` in +/// bin/gemma-wrapper) -- so stdout has nothing to stream. +async fn run_streaming(cmd: &str, args: &[&str], log_path: Option<&Path>) -> io::Result { + let mut child = Command::new(cmd) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let stdout_handle = child.stdout.take().expect("stdout was piped"); + let stderr_handle = child.stderr.take().expect("stderr was piped"); + + let stdout_task = tokio::spawn(async move { + let mut buf = Vec::new(); + let mut reader = stdout_handle; + let _ = reader.read_to_end(&mut buf).await; + buf + }); + + let log_path = log_path.map(Path::to_path_buf); + let stderr_task = tokio::spawn(async move { + let mut buf = Vec::new(); + let mut lines = BufReader::new(stderr_handle).lines(); + while let Ok(Some(line)) = lines.next_line().await { + if let Some(path) = &log_path { + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(path) { + let _ = writeln!(f, "{line}"); + } + } + buf.extend_from_slice(line.as_bytes()); + buf.push(b'\n'); + } + buf + }); + + let status = child.wait().await?; + let stdout = stdout_task.await.unwrap_or_default(); + let stderr = stderr_task.await.unwrap_or_default(); + Ok(Output { status, stdout, stderr }) } fn parse_wrapper_output(step: &'static str, output: Output) -> Result { @@ -121,9 +169,10 @@ async fn compute_kinship( genotype_file: &str, pheno_file: &str, snps_file: &str, + log_path: Option<&Path>, ) -> Result { let args = kinship_args(cache_dir, genotype_file, pheno_file, snps_file); - let output = run(gemma_wrapper_cmd, &args) + let output = run_streaming(gemma_wrapper_cmd, &args, log_path) .await .map_err(GemmaError::Io)?; parse_wrapper_output("kinship", output) @@ -155,6 +204,7 @@ async fn compute_gwa( snps_file: &str, lmm: u8, k_json_path: &str, + log_path: Option<&Path>, ) -> Result { let lmm_str = lmm.to_string(); let args = gwa_args( @@ -165,7 +215,7 @@ async fn compute_gwa( &lmm_str, k_json_path, ); - let output = run(gemma_wrapper_cmd, &args) + let output = run_streaming(gemma_wrapper_cmd, &args, log_path) .await .map_err(GemmaError::Io)?; parse_wrapper_output("gwa", output) @@ -229,14 +279,10 @@ fn parse_assoc_file(contents: &str) -> Vec { /// about this between the two steps). async fn extract_archive(cache_dir: &str, archive: &str, dest_dir: &str) -> Result<(), GemmaError> { let archive_path = Path::new(cache_dir).join(archive); - let output = run( + let output = run_streaming( "tar", - &[ - "xJf", - &archive_path.to_string_lossy(), - "-C", - dest_dir, - ], + &["xJf", &archive_path.to_string_lossy(), "-C", dest_dir], + None, ) .await .map_err(GemmaError::Io)?; @@ -327,6 +373,27 @@ fn gwa_args<'a>( /// schema) implements `ToSchema` for `serde_json::Value` itself, but /// `Vec` needs `Value: ComposeSchema`, which it does /// not implement in this utoipa version. +/// `run_id` must be a short token of ASCII letters, digits, `-` or `_` +/// (no path separators or `.`): it is joined straight onto `cache_dir` +/// to name the log file `status` reads back, so an unchecked value +/// would let a caller write/read outside `cache_dir` (the same +/// unchecked-`run_id`-as-path-component GeneNetwork3 does for its own +/// `/api/stream/{run_id}` -- not knowingly replicated here). +pub fn is_valid_run_id(run_id: &str) -> bool { + !run_id.is_empty() + && run_id.len() <= 64 + && run_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') +} + +/// The line `compute` appends to the log file once it finishes +/// successfully, so a `status` poller can tell "still running" apart +/// from "done" without needing a separate field -- same convention +/// GeneNetwork3 uses ("Done with GN3 Computation" in gn3/api/rqtl2.py), +/// under this project's own name. +pub const COMPLETION_MARKER: &str = "GN4: computation complete"; + pub async fn compute( gemma_wrapper_cmd: &str, cache_dir: &str, @@ -334,6 +401,7 @@ pub async fn compute( snps_file: &str, trait_values: &TraitValues, lmm: u8, + run_id: Option<&str>, ) -> Result { std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?; let unique = format!("{}-{}", std::process::id(), nanos_since_epoch()); @@ -341,6 +409,14 @@ pub async fn compute( write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?; let pheno_file = pheno_path.to_string_lossy().into_owned(); + // One log file for the whole request (both steps), truncated fresh + // at the start of each run so re-using an id doesn't append stale + // lines from a previous request ahead of this one's. + let log_path = run_id.map(|id| Path::new(cache_dir).join(format!("{id}.log"))); + if let Some(path) = &log_path { + std::fs::write(path, "").map_err(GemmaError::Io)?; + } + let kinship_note = format!( "Kinship: {gemma_wrapper_cmd} {}", kinship_args(cache_dir, genotype_file, &pheno_file, snps_file).join(" ") @@ -351,6 +427,7 @@ pub async fn compute( genotype_file, &pheno_file, snps_file, + log_path.as_deref(), ) .await?; let k_json_path = Path::new(cache_dir).join(format!("{unique}-k.json")); @@ -377,6 +454,7 @@ pub async fn compute( snps_file, lmm, &k_json_path.to_string_lossy(), + log_path.as_deref(), ) .await?; @@ -402,12 +480,50 @@ pub async fn compute( rows.extend(parse_assoc_file(&contents).into_iter().map(Value::Object)); } let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless + + if let Some(path) = &log_path { + if let Ok(mut f) = std::fs::OpenOptions::new().append(true).open(path) { + let _ = writeln!(f, "{COMPLETION_MARKER}"); + } + } + Ok(GemmaOutput { rows: Value::Array(rows), notes: vec![kinship_note, gwa_note], }) } +/// Why `status` (below) couldn't read a run's log. +#[derive(Debug)] +pub enum StatusError { + /// No log file for this `run_id` (unknown id, or the run hasn't + /// written anything yet -- `compute` creates the file before its + /// first subprocess call, but there is a brief window before that). + NotFound, + Io(io::Error), +} + +/// Lines appended to `run_id`'s log since byte offset `peak`, and the +/// new offset to pass as `peak` next time. Mirrors GeneNetwork3's +/// `GET /api/stream/{run_id}?peak=` (gn3/api/streaming.py), stateless +/// in the same way: the caller tracks `peak`, not the server. +pub fn status(cache_dir: &str, run_id: &str, peak: u64) -> Result<(Vec, u64), StatusError> { + let path = Path::new(cache_dir).join(format!("{run_id}.log")); + let mut file = std::fs::File::open(&path).map_err(|e| { + if e.kind() == io::ErrorKind::NotFound { + StatusError::NotFound + } else { + StatusError::Io(e) + } + })?; + file.seek(SeekFrom::Start(peak)).map_err(StatusError::Io)?; + let mut buf = String::new(); + file.read_to_string(&mut buf).map_err(StatusError::Io)?; + let pointer = peak + buf.len() as u64; + let lines = buf.lines().map(str::to_string).collect(); + Ok((lines, pointer)) +} + #[cfg(test)] mod tests { use super::*; @@ -487,4 +603,46 @@ mod tests { ); assert!(!args.contains(&"-gk"), "the GWA step must never carry -gk alongside -lmm"); } + + #[test] + fn is_valid_run_id_accepts_alnum_dash_underscore() { + for ok in ["abc123", "a-b_c", "A1", "x"] { + assert!(is_valid_run_id(ok), "{ok}"); + } + } + + #[test] + fn is_valid_run_id_rejects_path_separators_and_empty() { + for bad in ["", "../etc/passwd", "a/b", "a.b", "a b", &"x".repeat(65)] { + assert!(!is_valid_run_id(bad), "{bad}"); + } + } + + #[test] + fn status_reads_from_peak_and_returns_a_new_pointer() { + let path = std::env::temp_dir().join(format!("gn4-gemma-status-test-{}", std::process::id())); + let cache_dir = path.to_str().unwrap(); + std::fs::create_dir_all(cache_dir).unwrap(); + std::fs::write(Path::new(cache_dir).join("run1.log"), "line one\nline two\n").unwrap(); + + let (lines, pointer) = status(cache_dir, "run1", 0).unwrap(); + assert_eq!(lines, vec!["line one", "line two"]); + + std::fs::OpenOptions::new() + .append(true) + .open(Path::new(cache_dir).join("run1.log")) + .unwrap() + .write_all(b"line three\n") + .unwrap(); + let (more, _) = status(cache_dir, "run1", pointer).unwrap(); + assert_eq!(more, vec!["line three"]); + + std::fs::remove_dir_all(cache_dir).unwrap(); + } + + #[test] + fn status_unknown_run_id_is_not_found() { + let err = status("/tmp", "no-such-gn4-gemma-run-id", 0).unwrap_err(); + assert!(matches!(err, StatusError::NotFound)); + } } diff --git a/src/lib.rs b/src/lib.rs index ab88fff..4d90785 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -78,7 +78,8 @@ pub fn app() -> Router { .routes(routes!(routes::publications::publication_traits)) .routes(routes!(routes::search::search)) .routes(routes!(routes::correlation::correlation)) - .routes(routes!(routes::gemma::compute_gemma)); + .routes(routes!(routes::gemma::compute_gemma)) + .routes(routes!(routes::gemma::gemma_status)); // Registered -- and so reachable and listed in /openapi.json -- unless // GENENETWORK4_TEST_TOKEN switches it off; when registered it decides // per request whether the caller is local. See src/routes/dev.rs. diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs index 3df86a7..18423b1 100644 --- a/src/routes/gemma.rs +++ b/src/routes/gemma.rs @@ -1,18 +1,51 @@ //! Compute domain: /compute/gemma invokes gemma-wrapper (and the real //! GEMMA binary it wraps) to map a QTL from a genotype file, a SNP -//! annotation file and a list of trait values. See `crate::gemma` for -//! the invocation itself. +//! annotation file and a list of trait values. `GET +//! /compute/gemma/status/{run_id}` tails that run's progress while it +//! is (or was) in flight, the same shape GeneNetwork2/3 use for R/qtl +//! (gn3/computations/streaming.py, gn3/api/streaming.py): the caller +//! picks a `run_id`, passes it as `?id=` on the POST, and polls the +//! status endpoint with `?peak=` (the byte offset already read) on a +//! second connection while the first is still blocked on the POST. See +//! `crate::gemma` for the invocation and log-writing itself. use axum::Json; +use axum::extract::{Path, Query}; use axum::http::StatusCode; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::auth::AuthToken; use crate::config::Config; use crate::error::{ApiError, Unauthorized}; -use crate::gemma::{self, GemmaError}; +use crate::gemma::{self, GemmaError, StatusError}; use crate::response::{Ctx, Envelope, RequestInfo}; +/// `?id=` on `POST /compute/gemma`: an optional, caller-chosen run id +/// to log this run's progress under and poll back via `GET +/// /compute/gemma/status/{id}`. Omitted entirely, no log is written. +#[derive(Deserialize, utoipa::IntoParams)] +pub struct ComputeGemmaParams { + pub id: Option, +} + +#[derive(Deserialize, utoipa::IntoParams)] +pub struct GemmaStatusParams { + /// Byte offset already read; 0 to read from the start. Echoed back + /// (grown) as `pointer` in the response -- pass that next time. + pub peak: Option, +} + +#[derive(Serialize, utoipa::ToSchema)] +pub struct GemmaStatus { + pub run_id: String, + /// New lines of gemma-wrapper's stderr (its progress output under + /// --json) since `peak`. The last line, once the run has finished, + /// is the completion marker (`gemma::COMPLETION_MARKER`). + pub lines: Vec, + /// The new byte offset; pass this back as `peak` next poll. + pub pointer: u64, +} + #[derive(Deserialize, utoipa::ToSchema)] pub struct GemmaRequest { /// Path to a BIMBAM genotype file (plain text or gzipped), passed @@ -41,17 +74,20 @@ pub struct GemmaRequest { description = "Runs gemma-wrapper's K-matrix step, then its LMM \ association step, over the given genotype and SNP annotation files and \ trait values, and returns the resulting rows of GEMMA's own \ -.assoc.txt output.", +.assoc.txt output. Pass ?id= to log progress for GET \ +/compute/gemma/status/{id} to poll concurrently.", + params(ComputeGemmaParams), request_body = GemmaRequest, responses( (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as a JSON array of row objects", body = Envelope), Unauthorized, - (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9)", body = ApiError), + (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9), or id not alnum/-/_ (max 64 chars)", body = ApiError), (status = 502, description = "gemma-wrapper or GEMMA failed", body = ApiError) ))] pub async fn compute_gemma( ctx: Ctx, _token: AuthToken, + Query(params): Query, Json(request): Json, ) -> Result>, ApiError> { let rid = ctx.request_info(); @@ -71,6 +107,18 @@ pub async fn compute_gemma( ) .with_request_id(rid)); } + if let Some(id) = ¶ms.id { + if !gemma::is_valid_run_id(id) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_run_id", + "Invalid id", + "id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), + "compute_gemma", + ) + .with_request_id(rid)); + } + } let config = Config::load(); gemma::compute( @@ -80,6 +128,7 @@ pub async fn compute_gemma( &request.snps_file, &request.trait_values, lmm, + params.id.as_deref(), ) .await .map(|output| { @@ -94,6 +143,67 @@ pub async fn compute_gemma( .map_err(|e| gemma_error_to_api_error(e, rid)) } +#[utoipa::path(get, path = "/compute/gemma/status/{run_id}", + summary = "Poll a running (or finished) /compute/gemma call's progress", + description = "Returns the lines gemma-wrapper has written to stderr \ +for this run_id since byte offset `peak`, and a new `pointer` to pass as \ +`peak` next time. Call this on a second connection while the POST is \ +still in flight, the same tail-the-log-file shape GeneNetwork2/3 use for \ +R/qtl.", + params(("run_id" = String, Path, description = "The id passed as ?id= on the POST"), GemmaStatusParams), + responses( + (status = 200, description = "New log lines and the new pointer", body = Envelope), + Unauthorized, + (status = 404, description = "No run with this id (unknown, or not started yet)", body = ApiError), + (status = 422, description = "run_id not alnum/-/_ (max 64 chars)", body = ApiError) + ))] +pub async fn gemma_status( + ctx: Ctx, + _token: AuthToken, + Path(run_id): Path, + Query(params): Query, +) -> Result>, ApiError> { + let rid = ctx.request_info(); + if !gemma::is_valid_run_id(&run_id) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_run_id", + "Invalid run_id", + "run_id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), + "gemma_status", + ) + .with_request_id(rid)); + } + let config = Config::load(); + let peak = params.peak.unwrap_or(0); + gemma::status(&config.gemma_cache_dir, &run_id, peak) + .map(|(lines, pointer)| { + Json(ctx.envelope(GemmaStatus { + run_id: run_id.clone(), + lines, + pointer, + })) + }) + .map_err(|e| match e { + StatusError::NotFound => ApiError::new( + StatusCode::NOT_FOUND, + "run_not_found", + "Run not found", + format!("No run with id '{run_id}' (unknown, or not started yet)."), + "gemma_status", + ) + .with_request_id(rid), + StatusError::Io(err) => ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "internal_error", + "Internal error", + err.to_string(), + "gemma_status", + ) + .with_request_id(rid), + }) +} + fn gemma_error_to_api_error(error: GemmaError, rid: RequestInfo) -> ApiError { ApiError::new( StatusCode::BAD_GATEWAY, diff --git a/tests/gemma.rs b/tests/gemma.rs index 9aa58d7..0eef57a 100644 --- a/tests/gemma.rs +++ b/tests/gemma.rs @@ -42,22 +42,93 @@ async fn compute_gemma_maps_a_qtl_from_the_bxd_fixtures() { return; }; + let run_id = format!( + "test-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); let body = serde_json::json!({ "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": bxd_trait_values(), }); - let response = common::post_json_with_token( - "POST /compute/gemma", - "/compute/gemma", - &token, - &body, - ) - .await; + + // Run the POST (which blocks for the whole ~19s computation) on its + // own task, so this test can poll the status endpoint on a second + // connection while it is still in flight -- the real shape a + // browser would use, not just a read of the finished log. + let post_token = token.clone(); + let post_run_id = run_id.clone(); + let post_handle = tokio::spawn(async move { + common::post_json_with_token( + "POST /compute/gemma", + &format!("/compute/gemma?id={post_run_id}"), + &post_token, + &body, + ) + .await + }); + + // Poll until something besides the empty initial log file shows up, + // or the main request finishes first (a fast machine might not + // catch it mid-run; either way, a fresh log with the kinship step's + // output should exist very soon after the POST starts). + let mut saw_progress_while_running = false; + let mut peak = 0u64; + for _ in 0..40 { + if post_handle.is_finished() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + let response = + common::get_with_token("GET /compute/gemma/status", &format!("/compute/gemma/status/{run_id}?peak={peak}"), &token) + .await; + if response.status() != StatusCode::OK { + continue; // log file not created yet + } + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let status: Value = common::data(&bytes); + peak = status["pointer"].as_u64().unwrap_or(peak); + if !status["lines"].as_array().unwrap().is_empty() { + saw_progress_while_running = true; + } + } + assert!( + saw_progress_while_running, + "expected at least one non-empty status poll while the computation was still running" + ); + + let response = post_handle.await.unwrap(); let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); + // After completion, the log carries the completion marker, and + // real gemma-wrapper progress text (not just the one line written + // at log creation). + let final_response = common::get_with_token( + "GET /compute/gemma/status (final)", + &format!("/compute/gemma/status/{run_id}?peak=0"), + &token, + ) + .await; + assert_eq!(final_response.status(), StatusCode::OK); + let final_bytes = final_response.into_body().collect().await.unwrap().to_bytes(); + let final_status: Value = common::data(&final_bytes); + let lines: Vec<&str> = final_status["lines"] + .as_array() + .unwrap() + .iter() + .map(|l| l.as_str().unwrap()) + .collect(); + assert!(lines.contains(&"GN4: computation complete"), "{lines:?}"); + assert!( + lines.iter().any(|l| l.contains("GEMMA")), + "expected real gemma-wrapper progress text in the log: {lines:?}" + ); + let rows: Value = common::data(&bytes); let rows = rows.as_array().expect("data is a JSON array of rows"); assert!(!rows.is_empty(), "expected at least one mapped marker"); @@ -135,3 +206,72 @@ async fn compute_gemma_requires_a_token() { .unwrap(); assert_eq!(response.status(), StatusCode::UNAUTHORIZED); } + +#[tokio::test] +async fn compute_gemma_rejects_an_invalid_run_id() { + let Some(token) = common::auth_token().await else { + eprintln!("skipping compute_gemma_rejects_an_invalid_run_id: no token (see docs/gn-auth.md)"); + return; + }; + + let body = serde_json::json!({ + "genotype_file": test_data_path("BXD_geno.txt.gz"), + "snps_file": test_data_path("BXD_snps.txt"), + "trait_values": [1.0, 2.0], + }); + let response = common::post_json_with_token( + "POST /compute/gemma (bad id)", + "/compute/gemma?id=../etc/passwd", + &token, + &body, + ) + .await; + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); +} + +#[tokio::test] +async fn gemma_status_unknown_run_id_returns_404() { + let Some(token) = common::auth_token().await else { + eprintln!("skipping gemma_status_unknown_run_id_returns_404: no token (see docs/gn-auth.md)"); + return; + }; + let response = common::get_with_token( + "GET /compute/gemma/status (unknown)", + "/compute/gemma/status/no-such-gn4-gemma-run?peak=0", + &token, + ) + .await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn gemma_status_rejects_an_invalid_run_id() { + let Some(token) = common::auth_token().await else { + eprintln!("skipping gemma_status_rejects_an_invalid_run_id: no token (see docs/gn-auth.md)"); + return; + }; + let response = common::get_with_token( + "GET /compute/gemma/status (bad id)", + "/compute/gemma/status/..%2Fetc%2Fpasswd?peak=0", + &token, + ) + .await; + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); +} + +#[tokio::test] +async fn gemma_status_requires_a_token() { + let request = axum::http::Request::builder() + .method(axum::http::Method::GET) + .uri("/compute/gemma/status/some-run?peak=0") + .extension(axum::extract::ConnectInfo(std::net::SocketAddr::from(( + [127, 0, 0, 1], + 40000, + )))) + .body(axum::body::Body::empty()) + .unwrap(); + let response = tower::ServiceExt::oneshot(genenetwork4::app(), request) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} -- cgit 1.4.1 From 9cc1631324216d564aca4448cbe71f49dbf2a4af Mon Sep 17 00:00:00 2001 From: Pjotr Date: Mon, 12 Oct 2026 15:57:46 +0000 Subject: gemma: rename peak to peek; reject a reused run id with 409 peak -> peek: the status endpoint's byte-offset query parameter and everywhere it's documented (code comments, the GemmaStatusParams field, tests, README). GeneNetwork3's own equivalent (gn3/api/streaming.py) is spelled "peak"; noted where this now deliberately differs, not touching the unrelated "peak marker" wording elsewhere in the README. Run id uniqueness: the log file is now created with OpenOptions::create_new(true) instead of a plain truncating write, so it fails atomically (no separate check-then-write race) if a log already exists for that id -- another run still in progress, or a finished one never polled/cleaned up. Added GemmaError::RunIdInUse, mapped to 409 Conflict in the route, caught before any subprocess runs. Previously a reused id would have silently truncated whatever was already there. Verified: the new compute_gemma_rejects_a_run_id_already_in_use test pre-seeds a log file for an id and confirms the POST is refused (this needs no gemma-wrapper, since the check runs before any subprocess call). cargo test passes in full. Co-Authored-By: Claude Sonnet 5 --- README.md | 7 +++++-- src/gemma.rs | 40 ++++++++++++++++++++++++++------------ src/routes/gemma.rs | 32 +++++++++++++++++++++--------- tests/gemma.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++------- 4 files changed, 105 insertions(+), 30 deletions(-) (limited to 'src') diff --git a/README.md b/README.md index a8cc7e2..f7c50ce 100644 --- a/README.md +++ b/README.md @@ -954,10 +954,13 @@ request finishes, `"GN4: computation complete"` is appended as a sentinel so a poller can tell "still running" from "done". `run_id` must be 1-64 ASCII letters/digits/`-`/`_` -- it's a path component, so this is enforced on both this endpoint and the `?id=` that names it. +`id` must also be unique: if a log already exists for it (another run +still in progress, or a finished one never polled/cleaned up), the +`POST` is refused with 409 rather than silently overwriting it. ``` POST /compute/gemma?id=my-run-1 (on one connection; blocks until done) -GET /compute/gemma/status/my-run-1?peak=0 (on another, while the first is in flight) +GET /compute/gemma/status/my-run-1?peek=0 (on another, while the first is in flight) ``` ```json @@ -977,7 +980,7 @@ GET /compute/gemma/status/my-run-1?peak=0 (on another, while the first is } ``` -Pass `pointer` back as `?peak=` on the next poll; `lines` is empty +Pass `pointer` back as `?peek=` on the next poll; `lines` is empty (not an error) when nothing new has been written since. An unknown `run_id` (never started, or the main request hasn't written its log yet) is a 404, not an empty response -- genuinely different from diff --git a/src/gemma.rs b/src/gemma.rs index a42c201..c855673 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -38,6 +38,11 @@ pub enum GemmaError { AssocFile(io::Error), /// gemma-wrapper's GWA output archive did not list any files. NoAssocFiles, + /// `run_id` already has a log file (a run with that id is in + /// progress, or finished and was never polled/cleaned up). Caught + /// before any subprocess runs, so this never silently truncates + /// another run's in-progress log. + RunIdInUse(String), } impl std::fmt::Display for GemmaError { @@ -54,6 +59,9 @@ impl std::fmt::Display for GemmaError { GemmaError::NoAssocFiles => { write!(f, "gemma-wrapper's response listed no association output files") } + GemmaError::RunIdInUse(id) => { + write!(f, "run id '{id}' is already in use; pick a different id") + } } } } @@ -409,12 +417,19 @@ pub async fn compute( write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?; let pheno_file = pheno_path.to_string_lossy().into_owned(); - // One log file for the whole request (both steps), truncated fresh - // at the start of each run so re-using an id doesn't append stale - // lines from a previous request ahead of this one's. + // One log file for the whole request (both steps). create_new + // fails if it already exists, atomically: run_id must be unique, + // so a collision errors out here rather than silently truncating + // another run's (possibly still in-progress) log. let log_path = run_id.map(|id| Path::new(cache_dir).join(format!("{id}.log"))); if let Some(path) = &log_path { - std::fs::write(path, "").map_err(GemmaError::Io)?; + match std::fs::OpenOptions::new().write(true).create_new(true).open(path) { + Ok(_) => {} + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + return Err(GemmaError::RunIdInUse(run_id.unwrap().to_string())); + } + Err(e) => return Err(GemmaError::Io(e)), + } } let kinship_note = format!( @@ -503,11 +518,12 @@ pub enum StatusError { Io(io::Error), } -/// Lines appended to `run_id`'s log since byte offset `peak`, and the -/// new offset to pass as `peak` next time. Mirrors GeneNetwork3's -/// `GET /api/stream/{run_id}?peak=` (gn3/api/streaming.py), stateless -/// in the same way: the caller tracks `peak`, not the server. -pub fn status(cache_dir: &str, run_id: &str, peak: u64) -> Result<(Vec, u64), StatusError> { +/// Lines appended to `run_id`'s log since byte offset `peek`, and the +/// new offset to pass as `peek` next time. Mirrors GeneNetwork3's +/// `GET /api/stream/{run_id}?peak=` (gn3/api/streaming.py, spelled +/// "peak" there), stateless in the same way: the caller tracks `peek`, +/// not the server. +pub fn status(cache_dir: &str, run_id: &str, peek: u64) -> Result<(Vec, u64), StatusError> { let path = Path::new(cache_dir).join(format!("{run_id}.log")); let mut file = std::fs::File::open(&path).map_err(|e| { if e.kind() == io::ErrorKind::NotFound { @@ -516,10 +532,10 @@ pub fn status(cache_dir: &str, run_id: &str, peak: u64) -> Result<(Vec, StatusError::Io(e) } })?; - file.seek(SeekFrom::Start(peak)).map_err(StatusError::Io)?; + file.seek(SeekFrom::Start(peek)).map_err(StatusError::Io)?; let mut buf = String::new(); file.read_to_string(&mut buf).map_err(StatusError::Io)?; - let pointer = peak + buf.len() as u64; + let pointer = peek + buf.len() as u64; let lines = buf.lines().map(str::to_string).collect(); Ok((lines, pointer)) } @@ -619,7 +635,7 @@ mod tests { } #[test] - fn status_reads_from_peak_and_returns_a_new_pointer() { + fn status_reads_from_peek_and_returns_a_new_pointer() { let path = std::env::temp_dir().join(format!("gn4-gemma-status-test-{}", std::process::id())); let cache_dir = path.to_str().unwrap(); std::fs::create_dir_all(cache_dir).unwrap(); diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs index 18423b1..c79c589 100644 --- a/src/routes/gemma.rs +++ b/src/routes/gemma.rs @@ -5,7 +5,7 @@ //! is (or was) in flight, the same shape GeneNetwork2/3 use for R/qtl //! (gn3/computations/streaming.py, gn3/api/streaming.py): the caller //! picks a `run_id`, passes it as `?id=` on the POST, and polls the -//! status endpoint with `?peak=` (the byte offset already read) on a +//! status endpoint with `?peek=` (the byte offset already read) on a //! second connection while the first is still blocked on the POST. See //! `crate::gemma` for the invocation and log-writing itself. @@ -23,6 +23,9 @@ use crate::response::{Ctx, Envelope, RequestInfo}; /// `?id=` on `POST /compute/gemma`: an optional, caller-chosen run id /// to log this run's progress under and poll back via `GET /// /compute/gemma/status/{id}`. Omitted entirely, no log is written. +/// Must be unique: a collision with an existing log (another run still +/// in progress, or one finished and never polled/cleaned up) errors out +/// rather than overwriting it -- see `gemma::GemmaError::RunIdInUse`. #[derive(Deserialize, utoipa::IntoParams)] pub struct ComputeGemmaParams { pub id: Option, @@ -32,17 +35,17 @@ pub struct ComputeGemmaParams { pub struct GemmaStatusParams { /// Byte offset already read; 0 to read from the start. Echoed back /// (grown) as `pointer` in the response -- pass that next time. - pub peak: Option, + pub peek: Option, } #[derive(Serialize, utoipa::ToSchema)] pub struct GemmaStatus { pub run_id: String, /// New lines of gemma-wrapper's stderr (its progress output under - /// --json) since `peak`. The last line, once the run has finished, + /// --json) since `peek`. The last line, once the run has finished, /// is the completion marker (`gemma::COMPLETION_MARKER`). pub lines: Vec, - /// The new byte offset; pass this back as `peak` next poll. + /// The new byte offset; pass this back as `peek` next poll. pub pointer: u64, } @@ -81,6 +84,7 @@ trait values, and returns the resulting rows of GEMMA's own \ responses( (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as a JSON array of row objects", body = Envelope), Unauthorized, + (status = 409, description = "id is already in use by another run", body = ApiError), (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9), or id not alnum/-/_ (max 64 chars)", body = ApiError), (status = 502, description = "gemma-wrapper or GEMMA failed", body = ApiError) ))] @@ -140,14 +144,24 @@ pub async fn compute_gemma( } Json(envelope) }) - .map_err(|e| gemma_error_to_api_error(e, rid)) + .map_err(|e| match e { + GemmaError::RunIdInUse(id) => ApiError::new( + StatusCode::CONFLICT, + "run_id_in_use", + "Run id already in use", + format!("id '{id}' is already in use; pick a different id."), + "compute_gemma", + ) + .with_request_id(rid.clone()), + other => gemma_error_to_api_error(other, rid.clone()), + }) } #[utoipa::path(get, path = "/compute/gemma/status/{run_id}", summary = "Poll a running (or finished) /compute/gemma call's progress", description = "Returns the lines gemma-wrapper has written to stderr \ -for this run_id since byte offset `peak`, and a new `pointer` to pass as \ -`peak` next time. Call this on a second connection while the POST is \ +for this run_id since byte offset `peek`, and a new `pointer` to pass as \ +`peek` next time. Call this on a second connection while the POST is \ still in flight, the same tail-the-log-file shape GeneNetwork2/3 use for \ R/qtl.", params(("run_id" = String, Path, description = "The id passed as ?id= on the POST"), GemmaStatusParams), @@ -175,8 +189,8 @@ pub async fn gemma_status( .with_request_id(rid)); } let config = Config::load(); - let peak = params.peak.unwrap_or(0); - gemma::status(&config.gemma_cache_dir, &run_id, peak) + let peek = params.peek.unwrap_or(0); + gemma::status(&config.gemma_cache_dir, &run_id, peek) .map(|(lines, pointer)| { Json(ctx.envelope(GemmaStatus { run_id: run_id.clone(), diff --git a/tests/gemma.rs b/tests/gemma.rs index 0eef57a..0e0d313 100644 --- a/tests/gemma.rs +++ b/tests/gemma.rs @@ -76,21 +76,21 @@ async fn compute_gemma_maps_a_qtl_from_the_bxd_fixtures() { // catch it mid-run; either way, a fresh log with the kinship step's // output should exist very soon after the POST starts). let mut saw_progress_while_running = false; - let mut peak = 0u64; + let mut peek = 0u64; for _ in 0..40 { if post_handle.is_finished() { break; } tokio::time::sleep(std::time::Duration::from_millis(500)).await; let response = - common::get_with_token("GET /compute/gemma/status", &format!("/compute/gemma/status/{run_id}?peak={peak}"), &token) + common::get_with_token("GET /compute/gemma/status", &format!("/compute/gemma/status/{run_id}?peek={peek}"), &token) .await; if response.status() != StatusCode::OK { continue; // log file not created yet } let bytes = response.into_body().collect().await.unwrap().to_bytes(); let status: Value = common::data(&bytes); - peak = status["pointer"].as_u64().unwrap_or(peak); + peek = status["pointer"].as_u64().unwrap_or(peek); if !status["lines"].as_array().unwrap().is_empty() { saw_progress_while_running = true; } @@ -110,7 +110,7 @@ async fn compute_gemma_maps_a_qtl_from_the_bxd_fixtures() { // at log creation). let final_response = common::get_with_token( "GET /compute/gemma/status (final)", - &format!("/compute/gemma/status/{run_id}?peak=0"), + &format!("/compute/gemma/status/{run_id}?peek=0"), &token, ) .await; @@ -237,7 +237,7 @@ async fn gemma_status_unknown_run_id_returns_404() { }; let response = common::get_with_token( "GET /compute/gemma/status (unknown)", - "/compute/gemma/status/no-such-gn4-gemma-run?peak=0", + "/compute/gemma/status/no-such-gn4-gemma-run?peek=0", &token, ) .await; @@ -252,7 +252,7 @@ async fn gemma_status_rejects_an_invalid_run_id() { }; let response = common::get_with_token( "GET /compute/gemma/status (bad id)", - "/compute/gemma/status/..%2Fetc%2Fpasswd?peak=0", + "/compute/gemma/status/..%2Fetc%2Fpasswd?peek=0", &token, ) .await; @@ -263,7 +263,7 @@ async fn gemma_status_rejects_an_invalid_run_id() { async fn gemma_status_requires_a_token() { let request = axum::http::Request::builder() .method(axum::http::Method::GET) - .uri("/compute/gemma/status/some-run?peak=0") + .uri("/compute/gemma/status/some-run?peek=0") .extension(axum::extract::ConnectInfo(std::net::SocketAddr::from(( [127, 0, 0, 1], 40000, @@ -275,3 +275,45 @@ async fn gemma_status_requires_a_token() { .unwrap(); assert_eq!(response.status(), StatusCode::UNAUTHORIZED); } + +#[tokio::test] +async fn compute_gemma_rejects_a_run_id_already_in_use() { + let Some(token) = common::auth_token().await else { + eprintln!("skipping compute_gemma_rejects_a_run_id_already_in_use: no token (see docs/gn-auth.md)"); + return; + }; + + // The uniqueness check happens before any subprocess runs, so this + // doesn't need gemma-wrapper on PATH: just pre-seed a log file for + // an id and confirm the POST refuses to reuse it. + let cache_dir = genenetwork4::config::Config::load().gemma_cache_dir; + std::fs::create_dir_all(&cache_dir).unwrap(); + let run_id = format!( + "dup-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + std::fs::write( + std::path::Path::new(&cache_dir).join(format!("{run_id}.log")), + "a previous (or still running) call already owns this id\n", + ) + .unwrap(); + + let body = serde_json::json!({ + "genotype_file": test_data_path("BXD_geno.txt.gz"), + "snps_file": test_data_path("BXD_snps.txt"), + "trait_values": [1.0, 2.0], + }); + let response = common::post_json_with_token( + "POST /compute/gemma (duplicate id)", + &format!("/compute/gemma?id={run_id}"), + &token, + &body, + ) + .await; + assert_eq!(response.status(), StatusCode::CONFLICT); + + std::fs::remove_file(std::path::Path::new(&cache_dir).join(format!("{run_id}.log"))).unwrap(); +} -- cgit 1.4.1