diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-12 21:36:45 +0200 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-12 21:36:45 +0200 |
| commit | ba00d2650ddbb3c9b73e01b6537aa53ac04f3cc3 (patch) | |
| tree | 1c82d1b789b8937cbcb6da6b33ac55ae58759ac9 /src | |
| parent | ad3bb1e37e5feb22d3a8eb9e99e072f613fa186c (diff) | |
| parent | 2456d66fc2c14345ab0730ef0a4ae07458af8ec6 (diff) | |
| download | genenetwork4-ba00d2650ddbb3c9b73e01b6537aa53ac04f3cc3.tar.gz | |
Merge branch 'gemma'
Diffstat (limited to 'src')
| -rw-r--r-- | src/config.rs | 28 | ||||
| -rw-r--r-- | src/gemma.rs | 194 | ||||
| -rw-r--r-- | src/lib.rs | 6 | ||||
| -rw-r--r-- | src/routes/gemma.rs | 39 | ||||
| -rw-r--r-- | src/routes/mod.rs | 1 | ||||
| -rw-r--r-- | src/routes/rqtl.rs | 210 | ||||
| -rw-r--r-- | src/rqtl.rs | 433 | ||||
| -rw-r--r-- | src/streaming.rs | 248 |
8 files changed, 958 insertions, 201 deletions
diff --git a/src/config.rs b/src/config.rs index 16286b5..5030576 100644 --- a/src/config.rs +++ b/src/config.rs @@ -68,6 +68,15 @@ pub struct Config { /// results between calls (its own --cache-dir switch). Defaults to /// a "gn4-gemma" directory under the system temp dir. pub gemma_cache_dir: String, + /// Path to scripts/rqtl_wrapper.R (copied here from GeneNetwork3) + /// for POST /compute/rqtl. Defaults to that path relative to the + /// working directory the server is started from. + pub rqtl_wrapper_path: String, + /// Directory POST /compute/rqtl uses for phenotype files, run logs + /// and the "gn3" subdirectory the R script itself writes its + /// output into (see src/rqtl.rs). Defaults to a "gn4-rqtl" + /// directory under the system temp dir. + pub rqtl_cache_dir: String, } impl Config { @@ -118,6 +127,17 @@ impl Config { .to_string_lossy() .into_owned(), ); + values.insert( + "RQTL_WRAPPER_PATH".to_string(), + "scripts/rqtl_wrapper.R".to_string(), + ); + values.insert( + "RQTL_CACHE_DIR".to_string(), + std::env::temp_dir() + .join("gn4-rqtl") + .to_string_lossy() + .into_owned(), + ); // Layers 2 and 3: config files, in increasing priority. for env_key in ["GN4_CONF", "GN4_SECRETS"] { @@ -145,6 +165,8 @@ impl Config { "SEARCH_ENGINE", "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", + "RQTL_WRAPPER_PATH", + "RQTL_CACHE_DIR", ] { if let Ok(value) = std::env::var(key) { values.insert(key.to_string(), value); @@ -173,6 +195,8 @@ impl Config { dev_token_password: values["GN_AUTH_DEV_PASSWORD"].clone(), gemma_wrapper_cmd: values["GEMMA_WRAPPER_CMD"].clone(), gemma_cache_dir: values["GEMMA_CACHE_DIR"].clone(), + rqtl_wrapper_path: values["RQTL_WRAPPER_PATH"].clone(), + rqtl_cache_dir: values["RQTL_CACHE_DIR"].clone(), } } } @@ -245,6 +269,8 @@ mod tests { "SEARCH_ENGINE", "GEMMA_WRAPPER_CMD", "GEMMA_CACHE_DIR", + "RQTL_WRAPPER_PATH", + "RQTL_CACHE_DIR", ] { if std::env::var(key).is_ok() { eprintln!("skipping: {key} is set in the environment"); @@ -256,6 +282,8 @@ mod tests { assert_eq!(config.port, 3000); assert_eq!(config.gemma_wrapper_cmd, "gemma-wrapper"); assert!(config.gemma_cache_dir.ends_with("gn4-gemma")); + assert_eq!(config.rqtl_wrapper_path, "scripts/rqtl_wrapper.R"); + assert!(config.rqtl_cache_dir.ends_with("gn4-rqtl")); assert_eq!(config.auth_server_url, "http://127.0.0.1:8081/"); assert_eq!(config.bind, "0.0.0.0"); assert!(config.test_token_enabled, "on by default"); diff --git a/src/gemma.rs b/src/gemma.rs index c855673..9563131 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -9,16 +9,16 @@ //! respects, both found by running the real gemma-wrapper/GEMMA binaries //! against the BXD fixtures rather than trusting GeneNetwork3's own //! (entirely @unittest.skip'd) integration tests -- see `compute_gwa` -//! and `compute` below. +//! and `compute` below. Progress-streaming (`run_id`/`status`) is +//! shared infrastructure; see `crate::streaming`. use std::io; -use std::io::{Read, Seek, SeekFrom, Write}; use std::path::Path; -use std::process::{Output, Stdio}; +use std::process::Output; use serde_json::Value; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; -use tokio::process::Command; + +use crate::streaming::{self, RunLogError}; /// One phenotype value per strain (genotype-file order), or `None` for /// a missing ("NA") sample. @@ -66,6 +66,15 @@ impl std::fmt::Display for GemmaError { } } +impl From<RunLogError> for GemmaError { + fn from(e: RunLogError) -> Self { + match e { + RunLogError::AlreadyInUse(id) => GemmaError::RunIdInUse(id), + RunLogError::Io(e) => GemmaError::Io(e), + } + } +} + /// Write `values` as a GEMMA/BIMBAM phenotype file: one value per line, /// "NA" for a missing sample. Mirrors /// gn3.computations.gemma.generate_pheno_txt_file, minus its hash-based @@ -82,56 +91,6 @@ pub fn write_pheno_file(path: &Path, values: &TraitValues) -> io::Result<()> { std::fs::write(path, contents) } -/// Run `cmd`, optionally tailing its stderr into `log_path` one line at -/// a time as it runs, and return the same `Output` shape -/// `Command::output()` would have (full stdout, full stderr, exit -/// status) -- callers that don't pass `log_path` see identical -/// behaviour to the plain, non-streaming call it replaces. -/// -/// stderr, not stdout, is what's tailed: gemma-wrapper's own `--json` -/// mode sends all of its human-readable progress (banner, chromosome -/// list, "Executing: ...", per-step info/debug lines) to stderr, and -/// reserves stdout for the one final JSON record, printed once at -/// the very end (`OUTPUT = (options[:json] ? $stderr : $stdout)` in -/// bin/gemma-wrapper) -- so stdout has nothing to stream. -async fn run_streaming(cmd: &str, args: &[&str], log_path: Option<&Path>) -> io::Result<Output> { - let mut child = Command::new(cmd) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - let stdout_handle = child.stdout.take().expect("stdout was piped"); - let stderr_handle = child.stderr.take().expect("stderr was piped"); - - let stdout_task = tokio::spawn(async move { - let mut buf = Vec::new(); - let mut reader = stdout_handle; - let _ = reader.read_to_end(&mut buf).await; - buf - }); - - let log_path = log_path.map(Path::to_path_buf); - let stderr_task = tokio::spawn(async move { - let mut buf = Vec::new(); - let mut lines = BufReader::new(stderr_handle).lines(); - while let Ok(Some(line)) = lines.next_line().await { - if let Some(path) = &log_path { - if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(path) { - let _ = writeln!(f, "{line}"); - } - } - buf.extend_from_slice(line.as_bytes()); - buf.push(b'\n'); - } - buf - }); - - let status = child.wait().await?; - let stdout = stdout_task.await.unwrap_or_default(); - let stderr = stderr_task.await.unwrap_or_default(); - Ok(Output { status, stdout, stderr }) -} - fn parse_wrapper_output(step: &'static str, output: Output) -> Result<Value, GemmaError> { if !output.status.success() { return Err(GemmaError::CommandFailed { @@ -180,7 +139,7 @@ async fn compute_kinship( log_path: Option<&Path>, ) -> Result<Value, GemmaError> { let args = kinship_args(cache_dir, genotype_file, pheno_file, snps_file); - let output = run_streaming(gemma_wrapper_cmd, &args, log_path) + let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path) .await .map_err(GemmaError::Io)?; parse_wrapper_output("kinship", output) @@ -223,7 +182,7 @@ async fn compute_gwa( &lmm_str, k_json_path, ); - let output = run_streaming(gemma_wrapper_cmd, &args, log_path) + let output = streaming::run_streaming(gemma_wrapper_cmd, &args, log_path) .await .map_err(GemmaError::Io)?; parse_wrapper_output("gwa", output) @@ -287,7 +246,7 @@ fn parse_assoc_file(contents: &str) -> Vec<AssocRow> { /// about this between the two steps). async fn extract_archive(cache_dir: &str, archive: &str, dest_dir: &str) -> Result<(), GemmaError> { let archive_path = Path::new(cache_dir).join(archive); - let output = run_streaming( + let output = streaming::run_streaming( "tar", &["xJf", &archive_path.to_string_lossy(), "-C", dest_dir], None, @@ -381,27 +340,6 @@ fn gwa_args<'a>( /// schema) implements `ToSchema` for `serde_json::Value` itself, but /// `Vec<serde_json::Value>` needs `Value: ComposeSchema`, which it does /// not implement in this utoipa version. -/// `run_id` must be a short token of ASCII letters, digits, `-` or `_` -/// (no path separators or `.`): it is joined straight onto `cache_dir` -/// to name the log file `status` reads back, so an unchecked value -/// would let a caller write/read outside `cache_dir` (the same -/// unchecked-`run_id`-as-path-component GeneNetwork3 does for its own -/// `/api/stream/{run_id}` -- not knowingly replicated here). -pub fn is_valid_run_id(run_id: &str) -> bool { - !run_id.is_empty() - && run_id.len() <= 64 - && run_id - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') -} - -/// The line `compute` appends to the log file once it finishes -/// successfully, so a `status` poller can tell "still running" apart -/// from "done" without needing a separate field -- same convention -/// GeneNetwork3 uses ("Done with GN3 Computation" in gn3/api/rqtl2.py), -/// under this project's own name. -pub const COMPLETION_MARKER: &str = "GN4: computation complete"; - pub async fn compute( gemma_wrapper_cmd: &str, cache_dir: &str, @@ -417,20 +355,10 @@ pub async fn compute( write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?; let pheno_file = pheno_path.to_string_lossy().into_owned(); - // One log file for the whole request (both steps). create_new - // fails if it already exists, atomically: run_id must be unique, - // so a collision errors out here rather than silently truncating - // another run's (possibly still in-progress) log. - let log_path = run_id.map(|id| Path::new(cache_dir).join(format!("{id}.log"))); - if let Some(path) = &log_path { - match std::fs::OpenOptions::new().write(true).create_new(true).open(path) { - Ok(_) => {} - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { - return Err(GemmaError::RunIdInUse(run_id.unwrap().to_string())); - } - Err(e) => return Err(GemmaError::Io(e)), - } - } + // One log file for the whole request (both steps); see + // crate::streaming::create_run_log for why this can't silently + // collide with another run's. + let log_path = streaming::create_run_log(cache_dir, run_id)?; let kinship_note = format!( "Kinship: {gemma_wrapper_cmd} {}", @@ -496,11 +424,7 @@ pub async fn compute( } let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless - if let Some(path) = &log_path { - if let Ok(mut f) = std::fs::OpenOptions::new().append(true).open(path) { - let _ = writeln!(f, "{COMPLETION_MARKER}"); - } - } + streaming::mark_complete(&log_path); Ok(GemmaOutput { rows: Value::Array(rows), @@ -508,38 +432,6 @@ pub async fn compute( }) } -/// Why `status` (below) couldn't read a run's log. -#[derive(Debug)] -pub enum StatusError { - /// No log file for this `run_id` (unknown id, or the run hasn't - /// written anything yet -- `compute` creates the file before its - /// first subprocess call, but there is a brief window before that). - NotFound, - Io(io::Error), -} - -/// Lines appended to `run_id`'s log since byte offset `peek`, and the -/// new offset to pass as `peek` next time. Mirrors GeneNetwork3's -/// `GET /api/stream/{run_id}?peak=` (gn3/api/streaming.py, spelled -/// "peak" there), stateless in the same way: the caller tracks `peek`, -/// not the server. -pub fn status(cache_dir: &str, run_id: &str, peek: u64) -> Result<(Vec<String>, u64), StatusError> { - let path = Path::new(cache_dir).join(format!("{run_id}.log")); - let mut file = std::fs::File::open(&path).map_err(|e| { - if e.kind() == io::ErrorKind::NotFound { - StatusError::NotFound - } else { - StatusError::Io(e) - } - })?; - file.seek(SeekFrom::Start(peek)).map_err(StatusError::Io)?; - let mut buf = String::new(); - file.read_to_string(&mut buf).map_err(StatusError::Io)?; - let pointer = peek + buf.len() as u64; - let lines = buf.lines().map(str::to_string).collect(); - Ok((lines, pointer)) -} - #[cfg(test)] mod tests { use super::*; @@ -619,46 +511,4 @@ mod tests { ); assert!(!args.contains(&"-gk"), "the GWA step must never carry -gk alongside -lmm"); } - - #[test] - fn is_valid_run_id_accepts_alnum_dash_underscore() { - for ok in ["abc123", "a-b_c", "A1", "x"] { - assert!(is_valid_run_id(ok), "{ok}"); - } - } - - #[test] - fn is_valid_run_id_rejects_path_separators_and_empty() { - for bad in ["", "../etc/passwd", "a/b", "a.b", "a b", &"x".repeat(65)] { - assert!(!is_valid_run_id(bad), "{bad}"); - } - } - - #[test] - fn status_reads_from_peek_and_returns_a_new_pointer() { - let path = std::env::temp_dir().join(format!("gn4-gemma-status-test-{}", std::process::id())); - let cache_dir = path.to_str().unwrap(); - std::fs::create_dir_all(cache_dir).unwrap(); - std::fs::write(Path::new(cache_dir).join("run1.log"), "line one\nline two\n").unwrap(); - - let (lines, pointer) = status(cache_dir, "run1", 0).unwrap(); - assert_eq!(lines, vec!["line one", "line two"]); - - std::fs::OpenOptions::new() - .append(true) - .open(Path::new(cache_dir).join("run1.log")) - .unwrap() - .write_all(b"line three\n") - .unwrap(); - let (more, _) = status(cache_dir, "run1", pointer).unwrap(); - assert_eq!(more, vec!["line three"]); - - std::fs::remove_dir_all(cache_dir).unwrap(); - } - - #[test] - fn status_unknown_run_id_is_not_found() { - let err = status("/tmp", "no-such-gn4-gemma-run-id", 0).unwrap_err(); - assert!(matches!(err, StatusError::NotFound)); - } } diff --git a/src/lib.rs b/src/lib.rs index 50db2b5..75746d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,8 @@ mod gemma; pub mod response; mod routes; pub mod search; +mod streaming; +mod rqtl; use error::ApiError; use response::{Ctx, Envelope}; @@ -80,7 +82,9 @@ pub fn app() -> Router { .routes(routes!(routes::search::search)) .routes(routes!(routes::correlation::correlation)) .routes(routes!(routes::gemma::compute_gemma)) - .routes(routes!(routes::gemma::gemma_status)); + .routes(routes!(routes::gemma::gemma_status)) + .routes(routes!(routes::rqtl::compute_rqtl)) + .routes(routes!(routes::rqtl::rqtl_status)); // Registered -- and so reachable and listed in /openapi.json -- unless // GENENETWORK4_TEST_TOKEN switches it off; when registered it decides // per request whether the caller is local. See src/routes/dev.rs. diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs index c79c589..5ed3336 100644 --- a/src/routes/gemma.rs +++ b/src/routes/gemma.rs @@ -12,13 +12,14 @@ use axum::Json; use axum::extract::{Path, Query}; use axum::http::StatusCode; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use crate::auth::AuthToken; use crate::config::Config; use crate::error::{ApiError, Unauthorized}; -use crate::gemma::{self, GemmaError, StatusError}; +use crate::gemma::{self, GemmaError}; use crate::response::{Ctx, Envelope, RequestInfo}; +use crate::streaming::{self, RunStatus, RunStatusParams, StatusError}; /// `?id=` on `POST /compute/gemma`: an optional, caller-chosen run id /// to log this run's progress under and poll back via `GET @@ -31,24 +32,6 @@ pub struct ComputeGemmaParams { pub id: Option<String>, } -#[derive(Deserialize, utoipa::IntoParams)] -pub struct GemmaStatusParams { - /// Byte offset already read; 0 to read from the start. Echoed back - /// (grown) as `pointer` in the response -- pass that next time. - pub peek: Option<u64>, -} - -#[derive(Serialize, utoipa::ToSchema)] -pub struct GemmaStatus { - pub run_id: String, - /// New lines of gemma-wrapper's stderr (its progress output under - /// --json) since `peek`. The last line, once the run has finished, - /// is the completion marker (`gemma::COMPLETION_MARKER`). - pub lines: Vec<String>, - /// The new byte offset; pass this back as `peek` next poll. - pub pointer: u64, -} - #[derive(Deserialize, utoipa::ToSchema)] pub struct GemmaRequest { /// Path to a BIMBAM genotype file (plain text or gzipped), passed @@ -112,7 +95,7 @@ pub async fn compute_gemma( .with_request_id(rid)); } if let Some(id) = ¶ms.id { - if !gemma::is_valid_run_id(id) { + if !streaming::is_valid_run_id(id) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_run_id", @@ -164,9 +147,9 @@ for this run_id since byte offset `peek`, and a new `pointer` to pass as \ `peek` next time. Call this on a second connection while the POST is \ still in flight, the same tail-the-log-file shape GeneNetwork2/3 use for \ R/qtl.", - params(("run_id" = String, Path, description = "The id passed as ?id= on the POST"), GemmaStatusParams), + params(("run_id" = String, Path, description = "The id passed as ?id= on the POST"), RunStatusParams), responses( - (status = 200, description = "New log lines and the new pointer", body = Envelope<GemmaStatus>), + (status = 200, description = "New log lines and the new pointer", body = Envelope<RunStatus>), Unauthorized, (status = 404, description = "No run with this id (unknown, or not started yet)", body = ApiError), (status = 422, description = "run_id not alnum/-/_ (max 64 chars)", body = ApiError) @@ -175,10 +158,10 @@ pub async fn gemma_status( ctx: Ctx, _token: AuthToken, Path(run_id): Path<String>, - Query(params): Query<GemmaStatusParams>, -) -> Result<Json<Envelope<GemmaStatus>>, ApiError> { + Query(params): Query<RunStatusParams>, +) -> Result<Json<Envelope<RunStatus>>, ApiError> { let rid = ctx.request_info(); - if !gemma::is_valid_run_id(&run_id) { + if !streaming::is_valid_run_id(&run_id) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_run_id", @@ -190,9 +173,9 @@ pub async fn gemma_status( } let config = Config::load(); let peek = params.peek.unwrap_or(0); - gemma::status(&config.gemma_cache_dir, &run_id, peek) + streaming::status(&config.gemma_cache_dir, &run_id, peek) .map(|(lines, pointer)| { - Json(ctx.envelope(GemmaStatus { + Json(ctx.envelope(RunStatus { run_id: run_id.clone(), lines, pointer, diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 3a9f3e3..d050616 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -11,6 +11,7 @@ pub mod gemma; pub mod groups; pub mod mapping_methods; pub mod publications; +pub mod rqtl; pub mod search; pub mod species; pub mod traits; diff --git a/src/routes/rqtl.rs b/src/routes/rqtl.rs new file mode 100644 index 0000000..0cc7d23 --- /dev/null +++ b/src/routes/rqtl.rs @@ -0,0 +1,210 @@ +//! Compute domain: /compute/rqtl runs GeneNetwork3's own +//! `scripts/rqtl_wrapper.R` (via `Rscript`) to map a QTL from a +//! genotype file and trait values, the same shape as `/compute/gemma` +//! including its progress-streaming `?id=`/`GET .../status/{run_id}` +//! pair -- see `crate::streaming` for that and `crate::rqtl` for the +//! invocation and its documented scope (single scan + permutations; +//! no covariates, interval mapping or pair-scan). + +use axum::Json; +use axum::extract::{Path, Query}; +use axum::http::StatusCode; +use serde::Deserialize; + +use crate::auth::AuthToken; +use crate::config::Config; +use crate::error::{ApiError, Unauthorized}; +use crate::response::{Ctx, Envelope, RequestInfo}; +use crate::rqtl::{self, RqtlError, Sample}; +use crate::streaming::{self, RunStatus, RunStatusParams, StatusError}; + +/// `?id=` on `POST /compute/rqtl`: same as `/compute/gemma`'s -- an +/// optional, caller-chosen, unique run id to log progress under and +/// poll back via `GET /compute/rqtl/status/{id}`. +#[derive(Deserialize, utoipa::IntoParams)] +pub struct ComputeRqtlParams { + pub id: Option<String>, +} + +#[derive(Deserialize, utoipa::ToSchema)] +pub struct RqtlSample { + /// Must match the genotype file's sample-column order exactly; see + /// `crate::rqtl`'s module doc for why (R/qtl's cross format has no + /// per-sample name matching of its own). + pub name: String, + pub value: Option<f64>, +} + +#[derive(Deserialize, utoipa::ToSchema)] +pub struct RqtlRequest { + /// Path to an R/qtl-format `.geno` file (riset/F2/4-way; `@type:` + /// in its header). + pub genotype_file: String, + pub trait_name: String, + pub samples: Vec<RqtlSample>, + /// R/qtl's `--model`: "normal" or "np" (non-parametric). Defaults + /// to "normal". + pub model: Option<String>, + /// R/qtl's `--method`: hk, ehk, mr, em or imp. Defaults to "hk" + /// (Haley-Knott), the R script's own default. + pub method: Option<String>, + /// Number of permutations; 0 (default) skips permutation testing + /// entirely. + pub nperm: Option<u32>, + /// R/qtl's `--scale`: "mb" or "cm". Defaults to "mb", the R + /// script's own default; note the script's `geno_to_csvr` call site + /// does not actually forward this option to its mapping-scale + /// parameter (always "Mb" regardless) -- a latent inconsistency in + /// the script itself, not fixed here since it's GeneNetwork3's own + /// script, copied verbatim. + pub scale: Option<String>, + /// A marker name (from the genotype file) to use as a covariate. + pub control: Option<String>, +} + +#[utoipa::path(post, path = "/compute/rqtl", + summary = "Map a QTL from a genotype file and trait values using R/qtl", + description = "Runs GeneNetwork3's scripts/rqtl_wrapper.R (scanone, \ +plus permutations if nperm > 0) and returns the parsed marker results. \ +Pass ?id= to log progress for GET /compute/rqtl/status/{id} to poll \ +concurrently -- same shape as /compute/gemma.", + params(ComputeRqtlParams), + request_body = RqtlRequest, + responses( + (status = 200, description = "Mapped QTL markers (and, if requested, permutation thresholds)", body = Envelope<serde_json::Value>), + Unauthorized, + (status = 409, description = "id is already in use by another run", body = ApiError), + (status = 422, description = "id not alnum/-/_ (max 64 chars)", body = ApiError), + (status = 502, description = "Rscript or rqtl_wrapper.R failed", body = ApiError) + ))] +pub async fn compute_rqtl( + ctx: Ctx, + _token: AuthToken, + Query(params): Query<ComputeRqtlParams>, + Json(request): Json<RqtlRequest>, +) -> Result<Json<Envelope<serde_json::Value>>, ApiError> { + let rid = ctx.request_info(); + if let Some(id) = ¶ms.id { + if !streaming::is_valid_run_id(id) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_run_id", + "Invalid id", + "id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), + "compute_rqtl", + ) + .with_request_id(rid)); + } + } + + let config = Config::load(); + let samples: Vec<Sample> = request + .samples + .into_iter() + .map(|s| Sample { name: s.name, value: s.value }) + .collect(); + rqtl::compute( + &config.rqtl_wrapper_path, + &config.rqtl_cache_dir, + &request.genotype_file, + &request.trait_name, + &samples, + request.model.as_deref().unwrap_or("normal"), + request.method.as_deref().unwrap_or("hk"), + request.nperm.unwrap_or(0), + request.scale.as_deref().unwrap_or("mb"), + request.control.as_deref(), + params.id.as_deref(), + ) + .await + .map(|output| { + let notes = output.notes.clone(); + let mut envelope = ctx.envelope(rqtl::to_json(output)).with_note( + "markers come from R/qtl's scanone output; permutation is null unless nperm > 0 was requested.", + ); + for note in ¬es { + envelope = envelope.with_note(note); + } + Json(envelope) + }) + .map_err(|e| match e { + RqtlError::RunIdInUse(id) => ApiError::new( + StatusCode::CONFLICT, + "run_id_in_use", + "Run id already in use", + format!("id '{id}' is already in use; pick a different id."), + "compute_rqtl", + ) + .with_request_id(rid.clone()), + other => rqtl_error_to_api_error(other, rid.clone()), + }) +} + +#[utoipa::path(get, path = "/compute/rqtl/status/{run_id}", + summary = "Poll a running (or finished) /compute/rqtl call's progress", + description = "Same as GET /compute/gemma/status/{run_id}, against \ +the run_id passed as ?id= on POST /compute/rqtl.", + params(("run_id" = String, Path, description = "The id passed as ?id= on the POST"), RunStatusParams), + responses( + (status = 200, description = "New log lines and the new pointer", body = Envelope<RunStatus>), + Unauthorized, + (status = 404, description = "No run with this id (unknown, or not started yet)", body = ApiError), + (status = 422, description = "run_id not alnum/-/_ (max 64 chars)", body = ApiError) + ))] +pub async fn rqtl_status( + ctx: Ctx, + _token: AuthToken, + Path(run_id): Path<String>, + Query(params): Query<RunStatusParams>, +) -> Result<Json<Envelope<RunStatus>>, ApiError> { + let rid = ctx.request_info(); + if !streaming::is_valid_run_id(&run_id) { + return Err(ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_run_id", + "Invalid run_id", + "run_id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), + "rqtl_status", + ) + .with_request_id(rid)); + } + let config = Config::load(); + let peek = params.peek.unwrap_or(0); + streaming::status(&config.rqtl_cache_dir, &run_id, peek) + .map(|(lines, pointer)| { + Json(ctx.envelope(RunStatus { + run_id: run_id.clone(), + lines, + pointer, + })) + }) + .map_err(|e| match e { + StatusError::NotFound => ApiError::new( + StatusCode::NOT_FOUND, + "run_not_found", + "Run not found", + format!("No run with id '{run_id}' (unknown, or not started yet)."), + "rqtl_status", + ) + .with_request_id(rid), + StatusError::Io(err) => ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "internal_error", + "Internal error", + err.to_string(), + "rqtl_status", + ) + .with_request_id(rid), + }) +} + +fn rqtl_error_to_api_error(error: RqtlError, rid: RequestInfo) -> ApiError { + ApiError::new( + StatusCode::BAD_GATEWAY, + "rqtl_failed", + "R/qtl computation failed", + error.to_string(), + "compute_rqtl", + ) + .with_request_id(rid) +} diff --git a/src/rqtl.rs b/src/rqtl.rs new file mode 100644 index 0000000..d68f7e8 --- /dev/null +++ b/src/rqtl.rs @@ -0,0 +1,433 @@ +//! Core logic for POST /compute/rqtl: run GeneNetwork3's own +//! `scripts/rqtl_wrapper.R` (copied here verbatim) via `Rscript` against +//! a genotype file and trait values, and parse the resulting R/qtl +//! `scanone` (and, if requested, permutation) output into QTL mapping +//! rows. +//! +//! Mirrors gn3/computations/rqtl.py and gn3/api/rqtl.py (GeneNetwork3), +//! but synchronously, for the same reason as `crate::gemma`: axum/tokio +//! can await the subprocess directly, no job queue needed. +//! Progress-streaming (`run_id`/`status`) is the shared +//! `crate::streaming` infrastructure. +//! +//! Scope: a single scan (`scanone`) plus, if `nperm > 0`, its +//! permutation thresholds -- the common case, and the one actually +//! exercised against the real script below. Covariates (`addcovar`, +//! `covarstruct`), interval mapping, permutation strata and pair-scan +//! (`scantwo`, GeneNetwork3's `process_rqtl_pairscan`) are not +//! implemented: pair-scan in particular needs cross-referencing the +//! genotype file for nearest-marker lookups and a second (`MAP_`) +//! output file, a materially bigger port than this covers. +//! +//! One fix relative to GeneNetwork3, found by reading the code (not by +//! hitting it live -- this path has no test covering the real caller's +//! keys): `generate_rqtl_cmd`'s content hash only ever looks at kwargs +//! literally keyed `"g"`/`"p"` (`k in ["g", "p"]`), but +//! `gn3/api/rqtl.py`'s real caller builds its kwargs dict with keys +//! `"geno"`/`"pheno"` -- the two never match, so the genotype/phenotype +//! file *contents* never actually enter the uniqueness hash in +//! production, only the other options (model, method, ...). Two +//! different phenotypes mapped with otherwise-identical options would +//! get the same cached output filename and silently reuse each other's +//! result. Fixed here: the genotype and phenotype file contents are +//! always hashed, unconditionally. +//! +//! Important, easily-missed correctness requirement inherited from the +//! R script itself, not something this wrapper can check: R/qtl's +//! `csvr` cross format has no per-sample name matching at all -- +//! phenotype rows are joined to genotype columns purely by position. +//! `samples` here must list values in exactly the genotype file's +//! sample-column order, or results will be silently misassigned to the +//! wrong individual. + +use std::io; +use std::path::Path; + +use serde::Serialize; +use serde_json::Value; + +use crate::streaming::{self, RunLogError}; + +#[derive(Debug)] +pub enum RqtlError { + Io(io::Error), + /// Rscript exited non-zero; carries its stderr. + CommandFailed(String), + /// The main or permutation output CSV could not be read or parsed. + OutputFile(io::Error), + /// `run_id` already has a log file; see `crate::gemma::GemmaError`'s + /// variant of the same name for why this is caught before any + /// subprocess runs. + RunIdInUse(String), +} + +impl std::fmt::Display for RqtlError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RqtlError::Io(e) => write!(f, "failed to run Rscript: {e}"), + RqtlError::CommandFailed(stderr) => write!(f, "rqtl_wrapper.R failed: {stderr}"), + RqtlError::OutputFile(e) => write!(f, "could not read R/qtl output: {e}"), + RqtlError::RunIdInUse(id) => { + write!(f, "run id '{id}' is already in use; pick a different id") + } + } + } +} + +impl From<RunLogError> for RqtlError { + fn from(e: RunLogError) -> Self { + match e { + RunLogError::AlreadyInUse(id) => RqtlError::RunIdInUse(id), + RunLogError::Io(e) => RqtlError::Io(e), + } + } +} + +/// One sample's name and trait value (`None` for missing -- written as +/// the literal string `x`, the R script's own `na.strings`, not `NA` +/// like `crate::gemma`'s pheno file convention). +pub struct Sample { + pub name: String, + pub value: Option<f64>, +} + +/// Write the phenotype file R/qtl's wrapper script expects: +/// `read.table(..., na.strings = "x", header = TRUE)`, a `Sample` +/// column plus one named trait column. +pub fn write_pheno_file(path: &Path, trait_name: &str, samples: &[Sample]) -> io::Result<String> { + let mut contents = format!("Sample\t{trait_name}\n"); + for sample in samples { + contents.push_str(&sample.name); + contents.push('\t'); + match sample.value { + Some(v) => contents.push_str(&v.to_string()), + None => contents.push('x'), + } + contents.push('\n'); + } + std::fs::write(path, &contents)?; + Ok(contents) +} + +/// A cheap, dependency-free (std `DefaultHasher`/`SipHash`) stable hash +/// for a cache-key filename -- not cryptographic, and not meant to be; +/// GeneNetwork3's own `generate_hash_of_string` uses md5 for the same +/// non-adversarial purpose. +fn hash_str(s: &str) -> String { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let mut h = DefaultHasher::new(); + s.hash(&mut h); + format!("{:016x}", h.finish()) +} + +/// The `Rscript` arguments for one `scripts/rqtl_wrapper.R` invocation, +/// shared between the actual call and the note describing it (see +/// `compute`), so the two can never drift apart. +fn rqtl_args<'a>( + wrapper_path: &'a str, + genotype_file: &'a str, + pheno_file: &'a str, + model: &'a str, + method: &'a str, + nperm_str: &'a str, + scale: &'a str, + control: Option<&'a str>, + outdir: &'a str, + filename: &'a str, +) -> Vec<&'a str> { + let mut args = vec![ + wrapper_path, + "--geno", + genotype_file, + "--pheno", + pheno_file, + "--model", + model, + "--method", + method, + "--nperm", + nperm_str, + "--scale", + scale, + ]; + if let Some(control) = control { + args.push("--control"); + args.push(control); + } + args.push("--outdir"); + args.push(outdir); + args.push("--filename"); + args.push(filename); + args +} + +/// One row of R/qtl's main scanone output CSV (`name`, `chr`, `pos`, +/// `lod`). GeneNetwork3's `process_rqtl_mapping` also handles an +/// extended 4-way-cross shape with four extra mean/SE effect columns; +/// not reproduced here since this port's test fixture (and the common +/// case) is a 2-allele riset/F2 cross -- flagged, not silently dropped, +/// if ever hit (`extra` carries anything past the four known columns). +#[derive(Debug, Serialize, utoipa::ToSchema, PartialEq)] +pub struct MarkerResult { + pub name: String, + pub chr: String, + pub pos: f64, + pub lod: f64, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub extra: Vec<String>, +} + +/// Parse R/qtl's main output CSV: `"","chr","pos","lod"` header, then +/// `"name","chr",pos,lod[,...]` rows (name and chr quoted; pos, lod and +/// any extra columns are not). Confirmed by actually running +/// `scripts/rqtl_wrapper.R` against the BXD riset fixture. +fn parse_mapping_output(contents: &str) -> Vec<MarkerResult> { + contents + .lines() + .skip(1) // header + .filter(|line| !line.is_empty()) + .filter_map(|line| { + let fields: Vec<&str> = line.split(',').collect(); + if fields.len() < 4 { + return None; + } + let unquote = |s: &str| s.trim_matches('"').to_string(); + Some(MarkerResult { + name: unquote(fields[0]), + chr: unquote(fields[1]), + pos: fields[2].parse().ok()?, + lod: fields[3].parse().ok()?, + extra: fields[4..].iter().map(|s| unquote(s)).collect(), + }) + }) + .collect() +} + +/// R/qtl's permutation thresholds: every permutation's LOD, and the +/// suggestive (67th percentile) / significant (95th percentile) +/// thresholds GeneNetwork3 computes from them +/// (`gn3.computations.rqtl.process_perm_output`, via `numpy.percentile` +/// with its default linear interpolation, reproduced in `percentile` +/// below). +#[derive(Debug, Serialize, utoipa::ToSchema, PartialEq)] +pub struct PermutationResult { + pub values: Vec<f64>, + pub suggestive: f64, + pub significant: f64, +} + +/// Parse R/qtl's permutation output CSV: `"","lod"` header, then +/// `"1",value` rows. Confirmed by actually running +/// `scripts/rqtl_wrapper.R --nperm 50` against the BXD riset fixture. +fn parse_permutation_output(contents: &str) -> Vec<f64> { + contents + .lines() + .skip(1) + .filter(|line| !line.is_empty()) + .filter_map(|line| line.split(',').nth(1)?.parse().ok()) + .collect() +} + +/// `numpy.percentile`'s default ("linear") interpolation: for a +/// 0-indexed sorted array of length `n`, the percentile-`p` position is +/// `(n-1) * p/100`; a fractional position interpolates linearly between +/// its floor and ceiling. +fn percentile(sorted: &[f64], p: f64) -> f64 { + match sorted.len() { + 0 => f64::NAN, + 1 => sorted[0], + n => { + let idx = (n - 1) as f64 * p / 100.0; + let lo = idx.floor() as usize; + let hi = idx.ceil() as usize; + if lo == hi { + sorted[lo] + } else { + sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo as f64) + } + } + } +} + +pub struct RqtlOutput { + pub markers: Vec<MarkerResult>, + pub permutation: Option<PermutationResult>, + pub notes: Vec<String>, +} + +/// Run `scripts/rqtl_wrapper.R` (`scanone`, plus permutations if +/// `nperm > 0`) and return the parsed results. +#[allow(clippy::too_many_arguments)] +pub async fn compute( + r_wrapper_path: &str, + cache_dir: &str, + genotype_file: &str, + trait_name: &str, + samples: &[Sample], + model: &str, + method: &str, + nperm: u32, + scale: &str, + control: Option<&str>, + run_id: Option<&str>, +) -> Result<RqtlOutput, RqtlError> { + // The R script hardcodes its own output directory as + // "{outdir}/gn3/..." (both the cross file it builds and every + // result file it writes), so that subdirectory has to exist first, + // same as GeneNetwork3's own api/rqtl.py (`os.mkdir(outdir)`) does. + let outdir = Path::new(cache_dir).join("gn3"); + std::fs::create_dir_all(&outdir).map_err(RqtlError::Io)?; + + let unique = format!( + "{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + ); + let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt")); + let pheno_contents = + write_pheno_file(&pheno_path, trait_name, samples).map_err(RqtlError::Io)?; + let pheno_file = pheno_path.to_string_lossy().into_owned(); + + let geno_contents = std::fs::read(genotype_file).map_err(RqtlError::Io)?; + let options_key = format!("{model},{method},{nperm},{scale},{control:?}"); + let filename = format!( + "{}-output.csv", + hash_str(&format!( + "{}{pheno_contents}{options_key}", + String::from_utf8_lossy(&geno_contents) + )) + ); + + let log_path = streaming::create_run_log(cache_dir, run_id)?; + + let nperm_str = nperm.to_string(); + let args = rqtl_args( + r_wrapper_path, + genotype_file, + &pheno_file, + model, + method, + &nperm_str, + scale, + control, + cache_dir, + &filename, + ); + let note = format!("Rscript {}", args.join(" ")); + let output = streaming::run_streaming("Rscript", &args, log_path.as_deref()) + .await + .map_err(RqtlError::Io)?; + if !output.status.success() { + return Err(RqtlError::CommandFailed( + String::from_utf8_lossy(&output.stderr).into_owned(), + )); + } + + let main_contents = std::fs::read_to_string(outdir.join(&filename)).map_err(RqtlError::OutputFile)?; + let markers = parse_mapping_output(&main_contents); + + let permutation = if nperm > 0 { + let perm_contents = std::fs::read_to_string(outdir.join(format!("PERM_{filename}"))) + .map_err(RqtlError::OutputFile)?; + let values = parse_permutation_output(&perm_contents); + let mut sorted = values.clone(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + Some(PermutationResult { + suggestive: percentile(&sorted, 67.0), + significant: percentile(&sorted, 95.0), + values, + }) + } else { + None + }; + + streaming::mark_complete(&log_path); + + Ok(RqtlOutput { + markers, + permutation, + notes: vec![note], + }) +} + +/// `RqtlOutput` as the single JSON value the route returns -- see +/// `crate::gemma`'s own note on why `serde_json::Value`, not a `Vec`, +/// is what utoipa needs here. +pub fn to_json(output: RqtlOutput) -> Value { + serde_json::json!({ + "markers": output.markers, + "permutation": output.permutation, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn write_pheno_file_uses_x_for_missing_and_a_named_trait_column() { + let path = std::env::temp_dir().join(format!("gn4-rqtl-test-{}", std::process::id())); + let samples = vec![ + Sample { name: "BXD1".to_string(), value: Some(9.5) }, + Sample { name: "BXD2".to_string(), value: None }, + ]; + write_pheno_file(&path, "mytrait", &samples).unwrap(); + let contents = std::fs::read_to_string(&path).unwrap(); + std::fs::remove_file(&path).unwrap(); + assert_eq!(contents, "Sample\tmytrait\nBXD1\t9.5\nBXD2\tx\n"); + } + + #[test] + fn parse_mapping_output_strips_quotes_from_name_and_chr_only() { + let contents = "\"\",\"chr\",\"pos\",\"lod\"\n\"rs1\",\"1\",3.01,0.092\n\"rs2\",\"2\",5.01,0.013\n"; + let rows = parse_mapping_output(contents); + assert_eq!( + rows, + vec![ + MarkerResult { name: "rs1".into(), chr: "1".into(), pos: 3.01, lod: 0.092, extra: vec![] }, + MarkerResult { name: "rs2".into(), chr: "2".into(), pos: 5.01, lod: 0.013, extra: vec![] }, + ] + ); + } + + #[test] + fn parse_mapping_output_keeps_extra_effect_columns_for_4way_crosses() { + let contents = "\"\",\"chr\",\"pos\",\"lod\",\"AC\",\"AD\"\n\"rs1\",\"1\",3.01,0.092,\"5 ± 1\",\"6 ± 2\"\n"; + let rows = parse_mapping_output(contents); + assert_eq!(rows[0].extra, vec!["5 \u{b1} 1", "6 \u{b1} 2"]); + } + + #[test] + fn parse_permutation_output_skips_header_and_reads_the_second_column() { + let contents = "\"\",\"lod\"\n\"1\",0.54\n\"2\",0.10\n"; + assert_eq!(parse_permutation_output(contents), vec![0.54, 0.10]); + } + + #[test] + fn percentile_matches_numpy_linear_interpolation() { + let sorted = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + assert_eq!(percentile(&sorted, 0.0), 1.0); + assert_eq!(percentile(&sorted, 100.0), 5.0); + assert_eq!(percentile(&sorted, 50.0), 3.0); + // index = (5-1)*25/100 = 1.0 exactly -> sorted[1] = 2.0 + assert_eq!(percentile(&sorted, 25.0), 2.0); + // index = (5-1)*67/100 = 2.68 -> interpolate sorted[2]..sorted[3] + assert!((percentile(&sorted, 67.0) - 3.68).abs() < 1e-9); + } + + #[test] + fn rqtl_args_includes_control_only_when_given() { + let with_control = rqtl_args( + "wrapper.R", "g.geno", "p.txt", "normal", "hk", "0", "cM", Some("rs1"), "/out", "f.csv", + ); + assert!(with_control.contains(&"--control")); + let without = rqtl_args( + "wrapper.R", "g.geno", "p.txt", "normal", "hk", "0", "cM", None, "/out", "f.csv", + ); + assert!(!without.contains(&"--control")); + } +} diff --git a/src/streaming.rs b/src/streaming.rs new file mode 100644 index 0000000..90ce0f4 --- /dev/null +++ b/src/streaming.rs @@ -0,0 +1,248 @@ +//! Shared infrastructure for compute endpoints that run a long external +//! process and want to show progress while it's still running: tail a +//! subprocess's stderr into a per-run log file as it runs, and let a +//! second connection poll that log while the first is still blocked. +//! Used by `crate::gemma` and `crate::rqtl`, which both call a +//! different external tool (gemma-wrapper, Rscript) but need exactly +//! this same shape. Mirrors GeneNetwork2/3's own equivalent +//! (gn3/computations/streaming.py, gn3/api/streaming.py), used there +//! for R/qtl. + +use std::io; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Output, Stdio}; + +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; +use tokio::process::Command; + +/// `?peek=` on a status-polling endpoint: the byte offset already +/// read; 0 to read from the start. +#[derive(Deserialize, utoipa::IntoParams)] +pub struct RunStatusParams { + pub peek: Option<u64>, +} + +/// A status-polling endpoint's response: new log lines since `peek`, +/// and the new offset to pass as `peek` next time. +#[derive(Serialize, utoipa::ToSchema)] +pub struct RunStatus { + pub run_id: String, + /// New lines of the subprocess's stderr since `peek`. The last + /// line, once the run has finished, is `COMPLETION_MARKER`. + pub lines: Vec<String>, + /// The new byte offset; pass this back as `peek` next poll. + pub pointer: u64, +} + +/// A short token of ASCII letters, digits, `-` or `_` (no path +/// separators or `.`). `run_id` is joined straight onto a cache +/// directory to name the log file `status` reads back, so an unchecked +/// value would let a caller write/read outside it -- the same +/// unchecked-`run_id`-as-path-component GeneNetwork3 does for its own +/// `/api/stream/{run_id}`, not knowingly replicated here. +pub fn is_valid_run_id(run_id: &str) -> bool { + !run_id.is_empty() + && run_id.len() <= 64 + && run_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') +} + +/// The line appended to a run's log once it finishes successfully, so +/// a `status` poller can tell "still running" apart from "done" without +/// a separate field -- same convention GeneNetwork3 uses ("Done with +/// GN3 Computation" in gn3/api/rqtl2.py), under this project's own +/// name. +pub const COMPLETION_MARKER: &str = "GN4: computation complete"; + +/// Why `create_run_log` couldn't start a run's log. +#[derive(Debug)] +pub enum RunLogError { + /// `run_id` already has a log file: a run with that id is in + /// progress, or finished and was never polled/cleaned up. + AlreadyInUse(String), + Io(io::Error), +} + +/// Atomically create a fresh, empty log file for `run_id` under +/// `cache_dir` (`None` if `run_id` is `None` -- no streaming wanted), +/// returning its path. `create_new` fails if the file already exists, +/// so a reused id errors out here rather than silently truncating +/// another run's (possibly still in-progress) log -- there is no +/// separate check-then-write step to race. +pub fn create_run_log(cache_dir: &str, run_id: Option<&str>) -> Result<Option<PathBuf>, RunLogError> { + let Some(id) = run_id else { return Ok(None) }; + let path = Path::new(cache_dir).join(format!("{id}.log")); + match std::fs::OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(_) => Ok(Some(path)), + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + Err(RunLogError::AlreadyInUse(id.to_string())) + } + Err(e) => Err(RunLogError::Io(e)), + } +} + +/// Append `COMPLETION_MARKER` to `log_path`, if there is one. Best +/// effort: a failure here (e.g. the log was deleted mid-run) is not +/// worth failing an otherwise-successful computation over. +pub fn mark_complete(log_path: &Option<PathBuf>) { + if let Some(path) = log_path { + if let Ok(mut f) = std::fs::OpenOptions::new().append(true).open(path) { + let _ = writeln!(f, "{COMPLETION_MARKER}"); + } + } +} + +/// Run `cmd`, optionally tailing its stderr into `log_path` one line at +/// a time as it runs, and return the same `Output` shape +/// `Command::output()` would have (full stdout, full stderr, exit +/// status) -- callers that don't pass `log_path` see identical +/// behaviour to the plain, non-streaming call it replaces. +/// +/// stderr, not stdout, is what's tailed. For gemma-wrapper, all of its +/// human-readable `--json`-mode progress goes to stderr, with stdout +/// reserved for the one final JSON record (see `crate::gemma`'s own +/// note on this). R/qtl's wrapper script is simpler: its `cat(...)` and +/// `verbose_print(...)` calls just go to stdout by default, but run +/// under `Rscript` with nothing redirecting it, so stderr still carries +/// R's own progress/warning output (package load messages, warnings) -- +/// tailing stderr here is a deliberate, uniform choice across both +/// tools' call sites, not a guarantee every line of every tool's +/// progress ends up in the log. +pub async fn run_streaming(cmd: &str, args: &[&str], log_path: Option<&Path>) -> io::Result<Output> { + let mut child = Command::new(cmd) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let stdout_handle = child.stdout.take().expect("stdout was piped"); + let stderr_handle = child.stderr.take().expect("stderr was piped"); + + let stdout_task = tokio::spawn(async move { + let mut buf = Vec::new(); + let mut reader = stdout_handle; + let _ = reader.read_to_end(&mut buf).await; + buf + }); + + let log_path = log_path.map(Path::to_path_buf); + let stderr_task = tokio::spawn(async move { + let mut buf = Vec::new(); + let mut lines = BufReader::new(stderr_handle).lines(); + while let Ok(Some(line)) = lines.next_line().await { + if let Some(path) = &log_path { + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(path) { + let _ = writeln!(f, "{line}"); + } + } + buf.extend_from_slice(line.as_bytes()); + buf.push(b'\n'); + } + buf + }); + + let status = child.wait().await?; + let stdout = stdout_task.await.unwrap_or_default(); + let stderr = stderr_task.await.unwrap_or_default(); + Ok(Output { status, stdout, stderr }) +} + +/// Why `status` couldn't read a run's log. +#[derive(Debug)] +pub enum StatusError { + /// No log file for this `run_id` (unknown id, or the run hasn't + /// written anything yet -- there is a brief window between + /// `create_run_log` and the first subprocess call). + NotFound, + Io(io::Error), +} + +/// Lines appended to `run_id`'s log since byte offset `peek`, and the +/// new offset to pass as `peek` next time. Mirrors GeneNetwork3's +/// `GET /api/stream/{run_id}?peak=` (gn3/api/streaming.py, spelled +/// "peak" there), stateless in the same way: the caller tracks `peek`, +/// not the server. +pub fn status(cache_dir: &str, run_id: &str, peek: u64) -> Result<(Vec<String>, u64), StatusError> { + let path = Path::new(cache_dir).join(format!("{run_id}.log")); + let mut file = std::fs::File::open(&path).map_err(|e| { + if e.kind() == io::ErrorKind::NotFound { + StatusError::NotFound + } else { + StatusError::Io(e) + } + })?; + file.seek(SeekFrom::Start(peek)).map_err(StatusError::Io)?; + let mut buf = String::new(); + file.read_to_string(&mut buf).map_err(StatusError::Io)?; + let pointer = peek + buf.len() as u64; + let lines = buf.lines().map(str::to_string).collect(); + Ok((lines, pointer)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_valid_run_id_accepts_alnum_dash_underscore() { + for ok in ["abc123", "a-b_c", "A1", "x"] { + assert!(is_valid_run_id(ok), "{ok}"); + } + } + + #[test] + fn is_valid_run_id_rejects_path_separators_and_empty() { + for bad in ["", "../etc/passwd", "a/b", "a.b", "a b", &"x".repeat(65)] { + assert!(!is_valid_run_id(bad), "{bad}"); + } + } + + #[test] + fn create_run_log_none_run_id_yields_no_path() { + assert!(matches!(create_run_log("/tmp", None), Ok(None))); + } + + #[test] + fn create_run_log_rejects_a_reused_id() { + let cache_dir = std::env::temp_dir().join(format!("gn4-streaming-test-{}", std::process::id())); + let cache_dir = cache_dir.to_str().unwrap(); + std::fs::create_dir_all(cache_dir).unwrap(); + let id = "dup-run"; + + assert!(matches!(create_run_log(cache_dir, Some(id)), Ok(Some(_)))); + let err = create_run_log(cache_dir, Some(id)).unwrap_err(); + assert!(matches!(err, RunLogError::AlreadyInUse(ref i) if i == id)); + + std::fs::remove_dir_all(cache_dir).unwrap(); + } + + #[test] + fn status_reads_from_peek_and_returns_a_new_pointer() { + let path = std::env::temp_dir().join(format!("gn4-streaming-status-test-{}", std::process::id())); + let cache_dir = path.to_str().unwrap(); + std::fs::create_dir_all(cache_dir).unwrap(); + std::fs::write(Path::new(cache_dir).join("run1.log"), "line one\nline two\n").unwrap(); + + let (lines, pointer) = status(cache_dir, "run1", 0).unwrap(); + assert_eq!(lines, vec!["line one", "line two"]); + + std::fs::OpenOptions::new() + .append(true) + .open(Path::new(cache_dir).join("run1.log")) + .unwrap() + .write_all(b"line three\n") + .unwrap(); + let (more, _) = status(cache_dir, "run1", pointer).unwrap(); + assert_eq!(more, vec!["line three"]); + + std::fs::remove_dir_all(cache_dir).unwrap(); + } + + #[test] + fn status_unknown_run_id_is_not_found() { + let err = status("/tmp", "no-such-gn4-streaming-run-id", 0).unwrap_err(); + assert!(matches!(err, StatusError::NotFound)); + } +} |
