//! Core logic for POST /compute/rqtl: run GeneNetwork3's own //! `scripts/rqtl_wrapper.R` (copied here verbatim) via `Rscript` against //! a genotype file and trait values, and parse the resulting R/qtl //! `scanone` (and, if requested, permutation) output into QTL mapping //! rows. //! //! Mirrors gn3/computations/rqtl.py and gn3/api/rqtl.py (GeneNetwork3), //! but synchronously, for the same reason as `crate::gemma`: axum/tokio //! can await the subprocess directly, no job queue needed. //! Progress-streaming (`run_id`/`status`) is the shared //! `crate::streaming` infrastructure. //! //! Scope: a single scan (`scanone`) plus, if `nperm > 0`, its //! permutation thresholds -- the common case, and the one actually //! exercised against the real script below. Covariates (`addcovar`, //! `covarstruct`), interval mapping, permutation strata and pair-scan //! (`scantwo`, GeneNetwork3's `process_rqtl_pairscan`) are not //! implemented: pair-scan in particular needs cross-referencing the //! genotype file for nearest-marker lookups and a second (`MAP_`) //! output file, a materially bigger port than this covers. //! //! Two fixes relative to GeneNetwork3's script, plus one found by reading //! its Rust-side caller: //! //! The script itself (see the comments in `scripts/rqtl_wrapper.R`): it now //! picks the map column from whichever of `Mb`/`cM` the file has, and starts //! the samples right after the position columns. GeneNetwork3's copy assumes //! `Chr, Locus, cM, Mb` and begins the samples at column 5, which loses the //! first individual of a cM-only file (44 of this deployment's `.geno` files //! are cM-only) and fails the whole scan with "number of columns of result is //! not a multiple of vector length". //! //! One fix found by reading the code (not by //! hitting it live -- this path has no test covering the real caller's //! keys): `generate_rqtl_cmd`'s content hash only ever looks at kwargs //! literally keyed `"g"`/`"p"` (`k in ["g", "p"]`), but //! `gn3/api/rqtl.py`'s real caller builds its kwargs dict with keys //! `"geno"`/`"pheno"` -- the two never match, so the genotype/phenotype //! file *contents* never actually enter the uniqueness hash in //! production, only the other options (model, method, ...). Two //! different phenotypes mapped with otherwise-identical options would //! get the same cached output filename and silently reuse each other's //! result. Fixed here: the genotype and phenotype file contents are //! always hashed, unconditionally. //! //! Important, easily-missed correctness requirement inherited from the //! R script itself, not something this wrapper can check: R/qtl's //! `csvr` cross format has no per-sample name matching at all -- //! phenotype rows are joined to genotype columns purely by position. //! `samples` here must list values in exactly the genotype file's //! sample-column order, or results will be silently misassigned to the //! wrong individual. use std::io; use std::path::Path; use crate::covariates::{Covariate, write_covarstruct_file}; #[cfg(test)] use crate::covariates::CovariateKind; use serde::Serialize; use serde_json::Value; use crate::streaming::{self, RunLogError}; #[derive(Debug)] pub enum RqtlError { Io(io::Error), /// Rscript exited non-zero; carries its stderr. CommandFailed(String), /// The main or permutation output CSV could not be read or parsed. OutputFile(io::Error), /// `run_id` already has a log file; see `crate::gemma::GemmaError`'s /// variant of the same name for why this is caught before any /// subprocess runs. RunIdInUse(String), } impl std::fmt::Display for RqtlError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { RqtlError::Io(e) => write!(f, "failed to run Rscript: {e}"), RqtlError::CommandFailed(stderr) => write!(f, "rqtl_wrapper.R failed: {stderr}"), RqtlError::OutputFile(e) => write!(f, "could not read R/qtl output: {e}"), RqtlError::RunIdInUse(id) => { write!(f, "run id '{id}' is already in use; pick a different id") } } } } impl From for RqtlError { fn from(e: RunLogError) -> Self { match e { RunLogError::AlreadyInUse(id) => RqtlError::RunIdInUse(id), RunLogError::Io(e) => RqtlError::Io(e), } } } /// One sample's name and trait value (`None` for missing -- written as /// the literal string `x`, the R script's own `na.strings`, not `NA` /// like `crate::gemma`'s pheno file convention). #[derive(Debug, Clone)] pub struct Sample { pub name: String, pub value: Option, } /// Write the phenotype file R/qtl's wrapper script expects: /// `read.table(..., na.strings = "x", header = TRUE)`, a `Sample` /// column plus one named trait column. pub fn write_pheno_file( path: &Path, trait_name: &str, samples: &[Sample], covariates: &[Covariate], ) -> io::Result { let mut contents = format!("Sample\t{trait_name}"); for covariate in covariates { contents.push('\t'); contents.push_str(&covariate.name); } contents.push('\n'); for (index, sample) in samples.iter().enumerate() { contents.push_str(&sample.name); contents.push('\t'); match sample.value { Some(v) => contents.push_str(&v.to_string()), None => contents.push('x'), } for covariate in covariates { contents.push('\t'); match covariate.values.get(index).copied().flatten() { Some(v) => contents.push_str(&v.to_string()), None => contents.push('x'), } } contents.push('\n'); } std::fs::write(path, &contents)?; Ok(contents) } /// A cheap, dependency-free (std `DefaultHasher`/`SipHash`) stable hash /// for a cache-key filename -- not cryptographic, and not meant to be; /// GeneNetwork3's own `generate_hash_of_string` uses md5 for the same /// non-adversarial purpose. fn hash_str(s: &str) -> String { use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; let mut h = DefaultHasher::new(); s.hash(&mut h); format!("{:016x}", h.finish()) } /// The R/qtl run's output filename. The content hash alone would be /// deterministic across processes: two concurrent identical requests /// (an AI client retrying after a timeout; two agents on the same /// trait) would share one output file, and R's `write.csv` truncates /// on open, so one request can read the other's half-written results /// and silently drop the damaged lines. Folding in the per-call /// `unique` (pid + nanos, already computed for the pheno file) gives /// every request its own output file. fn output_filename( unique: &str, geno_contents: &str, pheno_contents: &str, options_key: &str, ) -> String { format!( "{unique}-{}-output.csv", hash_str(&format!("{geno_contents}{pheno_contents}{options_key}")) ) } /// The `Rscript` arguments for one `scripts/rqtl_wrapper.R` invocation, /// shared between the actual call and the note describing it (see /// `compute`), so the two can never drift apart. fn rqtl_args<'a>( wrapper_path: &'a str, genotype_file: &'a str, pheno_file: &'a str, model: &'a str, method: &'a str, nperm_str: &'a str, scale: &'a str, control: Option<&'a str>, covarstruct: Option<&'a str>, outdir: &'a str, filename: &'a str, ) -> Vec<&'a str> { let mut args = vec![ wrapper_path, "--geno", genotype_file, "--pheno", pheno_file, "--model", model, "--method", method, "--nperm", nperm_str, "--scale", scale, ]; if let Some(control) = control { args.push("--control"); args.push(control); } if let Some(covarstruct) = covarstruct { args.push("--addcovar"); args.push("--covarstruct"); args.push(covarstruct); } args.push("--outdir"); args.push(outdir); args.push("--filename"); args.push(filename); args } /// One row of R/qtl's main scanone output CSV (`name`, `chr`, `pos`, /// `lod`). GeneNetwork3's `process_rqtl_mapping` also handles an /// extended 4-way-cross shape with four extra mean/SE effect columns; /// not reproduced here since this port's test fixture (and the common /// case) is a 2-allele riset/F2 cross -- flagged, not silently dropped, /// if ever hit (`extra` carries anything past the four known columns). #[derive(Debug, Serialize, utoipa::ToSchema, PartialEq)] pub struct MarkerResult { pub name: String, pub chr: String, pub pos: f64, pub lod: f64, #[serde(skip_serializing_if = "Vec::is_empty")] pub extra: Vec, } /// Parse R/qtl's main output CSV: `"","chr","pos","lod"` header, then /// `"name","chr",pos,lod[,...]` rows (name and chr quoted; pos, lod and /// any extra columns are not). Confirmed by actually running /// `scripts/rqtl_wrapper.R` against the BXD riset fixture. fn parse_mapping_output(contents: &str) -> Vec { contents .lines() .skip(1) // header .filter(|line| !line.is_empty()) .filter_map(|line| { let fields: Vec<&str> = line.split(',').collect(); if fields.len() < 4 { return None; } let unquote = |s: &str| s.trim_matches('"').to_string(); Some(MarkerResult { name: unquote(fields[0]), chr: unquote(fields[1]), pos: fields[2].parse().ok()?, lod: fields[3].parse().ok()?, extra: fields[4..].iter().map(|s| unquote(s)).collect(), }) }) .collect() } /// R/qtl's permutation thresholds: every permutation's LOD, and the /// suggestive (67th percentile) / significant (95th percentile) /// thresholds GeneNetwork3 computes from them /// (`gn3.computations.rqtl.process_perm_output`, via `numpy.percentile` /// with its default linear interpolation, reproduced in `percentile` /// below). #[derive(Debug, Serialize, utoipa::ToSchema, PartialEq)] pub struct PermutationResult { pub values: Vec, /// The 67th percentile: R/qtl's "suggestive" LOD. pub suggestive: f64, /// The 95th percentile: the genome-wide alpha = 0.05 threshold. pub significant: f64, } impl PermutationResult { /// Thresholds from a permutation null. Shared with `/compute/qtl2`, which /// permutes with qtl2's own `scan1perm`: the null is the genome-wide LOD /// maximum per permutation, so the percentiles mean the same thing /// whichever engine produced them. pub fn from_values(mut values: Vec) -> Self { values.retain(|value| value.is_finite()); let mut sorted = values.clone(); sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); Self { suggestive: percentile(&sorted, 67.0), significant: percentile(&sorted, 95.0), values, } } } /// Write (individual, stratum) pairs as the two-column file `--strata` reads: /// `Samplestratum`. Written the way the scripts expect rather than by /// reusing the covariate writer, which emits one column per covariate. pub fn write_strata_file(path: &std::path::Path, samples: &[(String, String)]) -> std::io::Result<()> { use std::io::Write; let mut out = String::from("Sample\tstratum\n"); for (individual, stratum) in samples { out.push_str(individual); out.push('\t'); out.push_str(stratum); out.push('\n'); } std::fs::write(path, out) } /// Parse R/qtl's permutation output CSV: `"","lod"` header, then /// `"1",value` rows. Confirmed by actually running /// `scripts/rqtl_wrapper.R --nperm 50` against the BXD riset fixture. fn parse_permutation_output(contents: &str) -> Vec { contents .lines() .skip(1) .filter(|line| !line.is_empty()) .filter_map(|line| line.split(',').nth(1)?.parse().ok()) .collect() } /// `numpy.percentile`'s default ("linear") interpolation: for a /// 0-indexed sorted array of length `n`, the percentile-`p` position is /// `(n-1) * p/100`; a fractional position interpolates linearly between /// its floor and ceiling. fn percentile(sorted: &[f64], p: f64) -> f64 { match sorted.len() { 0 => f64::NAN, 1 => sorted[0], n => { let idx = (n - 1) as f64 * p / 100.0; let lo = idx.floor() as usize; let hi = idx.ceil() as usize; if lo == hi { sorted[lo] } else { sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo as f64) } } } } pub struct RqtlOutput { pub markers: Vec, pub permutation: Option, pub notes: Vec, } /// Run `scripts/rqtl_wrapper.R` (`scanone`, plus permutations if /// `nperm > 0`) and return the parsed results. #[allow(clippy::too_many_arguments)] pub async fn compute( r_wrapper_path: &str, cache_dir: &str, genotype_file: &str, trait_name: &str, samples: &[Sample], covariates: &[Covariate], model: &str, method: &str, nperm: u32, scale: &str, control: Option<&str>, run_id: Option<&str>, ) -> Result { // The R script hardcodes its own output directory as // "{outdir}/gn3/..." (both the cross file it builds and every // result file it writes), so that subdirectory has to exist first, // same as GeneNetwork3's own api/rqtl.py (`os.mkdir(outdir)`) does. let outdir = Path::new(cache_dir).join("gn3"); std::fs::create_dir_all(&outdir).map_err(RqtlError::Io)?; let unique = format!( "{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_nanos()) .unwrap_or(0) ); let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt")); let pheno_contents = write_pheno_file(&pheno_path, trait_name, samples, covariates).map_err(RqtlError::Io)?; let pheno_file = pheno_path.to_string_lossy().into_owned(); // The covariate kinds decide the model (categorical is dummy-expanded // by the wrapper, numerical is not), so they belong in the cache key // alongside the options; their values are already in the phenotype // contents. let covarstruct_path = Path::new(cache_dir).join(format!("{unique}-covarstruct.txt")); let covarstruct_file = if covariates.is_empty() { None } else { write_covarstruct_file(&covarstruct_path, covariates).map_err(RqtlError::Io)?; Some(covarstruct_path.to_string_lossy().into_owned()) }; let geno_contents = std::fs::read(genotype_file).map_err(RqtlError::Io)?; let covariates_key = covariates .iter() .map(|covariate| format!("{}:{}", covariate.name, covariate.kind.as_str())) .collect::>() .join("+"); let options_key = format!("{model},{method},{nperm},{scale},{control:?},{covariates_key}"); let filename = output_filename( &unique, &String::from_utf8_lossy(&geno_contents), &pheno_contents, &options_key, ); let log_path = streaming::create_run_log(cache_dir, run_id)?; // Everything after the log exists reports its outcome into the log: // success ends with `COMPLETION_MARKER`, any error with // `FAILURE_MARKER: ` -- so a `status` poller can never // mistake a failed run for a still-running one (a healthy R run is // also quiet for long stretches). let outcome = async { let nperm_str = nperm.to_string(); let args = rqtl_args( r_wrapper_path, genotype_file, &pheno_file, model, method, &nperm_str, scale, control, covarstruct_file.as_deref(), cache_dir, &filename, ); let note = format!("Rscript {}", args.join(" ")); let output = streaming::run_streaming("Rscript", &args, log_path.as_deref()) .await .map_err(RqtlError::Io)?; if !output.status.success() { return Err(RqtlError::CommandFailed( String::from_utf8_lossy(&output.stderr).into_owned(), )); } let main_contents = std::fs::read_to_string(outdir.join(&filename)).map_err(RqtlError::OutputFile)?; let markers = parse_mapping_output(&main_contents); let permutation = if nperm > 0 { let perm_contents = std::fs::read_to_string(outdir.join(format!("PERM_{filename}"))) .map_err(RqtlError::OutputFile)?; let values = parse_permutation_output(&perm_contents); let mut sorted = values.clone(); sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); Some(PermutationResult { suggestive: percentile(&sorted, 67.0), significant: percentile(&sorted, 95.0), values, }) } else { None }; Ok(RqtlOutput { markers, permutation, notes: vec![note], }) } .await; match outcome { Ok(output) => { streaming::mark_complete(&log_path); Ok(output) } Err(e) => { streaming::mark_failed(&log_path, &e); Err(e) } } } /// `MarkerResult`'s fixed fields, in the order `to_json` lays out each /// marker row -- the names a caller matches `markers`' positional rows /// up against (a 4-way cross's `extra` effect columns, if present, /// trail these and aren't named here; see `MarkerResult`'s own doc /// comment). pub const MARKER_COLUMNS: [&str; 4] = ["name", "chr", "pos", "lod"]; /// `RqtlOutput` as the single JSON value the route returns -- see /// `crate::gemma`'s own note on why `serde_json::Value`, not a `Vec`, /// is what utoipa needs here. `markers` rows are positional (see /// `MARKER_COLUMNS`), the same space-saving shape `crate::gemma::compute` /// uses for its own (much larger) marker rows. pub fn to_json(output: RqtlOutput) -> Value { let markers: Vec = output .markers .into_iter() .map(|m| { let mut row = vec![ Value::String(m.name), Value::String(m.chr), serde_json::json!(m.pos), serde_json::json!(m.lod), ]; row.extend(m.extra.into_iter().map(Value::String)); Value::Array(row) }) .collect(); serde_json::json!({ "markers": markers, "permutation": output.permutation, }) } /// The marker with the strongest LOD in a `to_json` result, if there is /// one: the QTL's peak, which is what a PheWAS wants to be centred on. pub fn peak_marker(data: &Value) -> Option { data.get("markers")? .as_array()? .iter() .filter_map(|row| { let row = row.as_array()?; let lod = row.get(3)?.as_f64()?; let name = row.first()?.as_str()?.to_string(); Some((lod, name)) }) .max_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)) .map(|(_, name)| name) } #[cfg(test)] mod tests { use super::*; #[test] fn output_filename_is_unique_per_call_even_for_identical_inputs() { let a = output_filename("1-100", "geno", "pheno", "normal,hk,0,mb,None"); let b = output_filename("2-200", "geno", "pheno", "normal,hk,0,mb,None"); assert_ne!( a, b, "two concurrent identical requests must not share an output file" ); assert!(a.starts_with("1-100-") && a.ends_with("-output.csv")); } #[test] fn write_pheno_file_uses_x_for_missing_and_a_named_trait_column() { let path = std::env::temp_dir().join(format!("gn4-rqtl-test-{}", std::process::id())); let samples = vec![ Sample { name: "BXD1".to_string(), value: Some(9.5), }, Sample { name: "BXD2".to_string(), value: None, }, ]; write_pheno_file(&path, "mytrait", &samples, &[]).unwrap(); let contents = std::fs::read_to_string(&path).unwrap(); std::fs::remove_file(&path).unwrap(); assert_eq!(contents, "Sample\tmytrait\nBXD1\t9.5\nBXD2\tx\n"); } #[test] fn covariates_are_written_after_the_trait_and_missing_ones_are_x() { let path = std::env::temp_dir().join(format!("gn4-rqtl-cov-{}", std::process::id())); let samples = vec![ Sample { name: "F1".to_string(), value: Some(9.5) }, Sample { name: "F2".to_string(), value: Some(8.0) }, ]; let covariates = vec![ Covariate { name: "Sex".to_string(), kind: CovariateKind::Categorical, values: vec![Some(0.0), Some(1.0)], levels: vec![ crate::covariates::Level { name: "F".to_string(), code: 0.0, }, crate::covariates::Level { name: "M".to_string(), code: 1.0, }, ], }, Covariate { name: "Age".to_string(), kind: CovariateKind::Numerical, values: vec![Some(183.0), None], levels: Vec::new(), }, ]; let contents = write_pheno_file(&path, "trait", &samples, &covariates).unwrap(); let struct_path = std::env::temp_dir().join(format!("gn4-rqtl-cs-{}", std::process::id())); let covarstruct = write_covarstruct_file(&struct_path, &covariates).unwrap(); let _ = std::fs::remove_file(&path); let _ = std::fs::remove_file(&struct_path); assert_eq!( contents, "Sample\ttrait\tSex\tAge\nF1\t9.5\t0\t183\nF2\t8\t1\tx\n" ); assert_eq!(covarstruct, "Sex\tcategorical\nAge\tnumerical\n"); } #[test] fn parse_mapping_output_strips_quotes_from_name_and_chr_only() { let contents = "\"\",\"chr\",\"pos\",\"lod\"\n\"rs1\",\"1\",3.01,0.092\n\"rs2\",\"2\",5.01,0.013\n"; let rows = parse_mapping_output(contents); assert_eq!( rows, vec![ MarkerResult { name: "rs1".into(), chr: "1".into(), pos: 3.01, lod: 0.092, extra: vec![] }, MarkerResult { name: "rs2".into(), chr: "2".into(), pos: 5.01, lod: 0.013, extra: vec![] }, ] ); } #[test] fn parse_mapping_output_keeps_extra_effect_columns_for_4way_crosses() { let contents = "\"\",\"chr\",\"pos\",\"lod\",\"AC\",\"AD\"\n\"rs1\",\"1\",3.01,0.092,\"5 ± 1\",\"6 ± 2\"\n"; let rows = parse_mapping_output(contents); assert_eq!(rows[0].extra, vec!["5 \u{b1} 1", "6 \u{b1} 2"]); } #[test] fn parse_permutation_output_skips_header_and_reads_the_second_column() { let contents = "\"\",\"lod\"\n\"1\",0.54\n\"2\",0.10\n"; assert_eq!(parse_permutation_output(contents), vec![0.54, 0.10]); } #[test] fn percentile_matches_numpy_linear_interpolation() { let sorted = vec![1.0, 2.0, 3.0, 4.0, 5.0]; assert_eq!(percentile(&sorted, 0.0), 1.0); assert_eq!(percentile(&sorted, 100.0), 5.0); assert_eq!(percentile(&sorted, 50.0), 3.0); // index = (5-1)*25/100 = 1.0 exactly -> sorted[1] = 2.0 assert_eq!(percentile(&sorted, 25.0), 2.0); // index = (5-1)*67/100 = 2.68 -> interpolate sorted[2]..sorted[3] assert!((percentile(&sorted, 67.0) - 3.68).abs() < 1e-9); } #[test] fn the_peak_marker_is_the_strongest_lod() { let data = serde_json::json!({ "markers": [ ["rs1", "1", 3.0, 0.5], ["rs2", "3", 9.0, 4.2], ["rs3", "5", 12.0, 1.1] ], "permutation": null }); assert_eq!(peak_marker(&data).as_deref(), Some("rs2")); assert_eq!(peak_marker(&serde_json::json!({"markers": []})), None); assert_eq!(peak_marker(&serde_json::json!({})), None); } #[test] fn rqtl_args_includes_control_only_when_given() { let with_control = rqtl_args( "wrapper.R", "g.geno", "p.txt", "normal", "hk", "0", "cM", Some("rs1"), None, "/out", "f.csv", ); assert!(with_control.contains(&"--control")); let without = rqtl_args( "wrapper.R", "g.geno", "p.txt", "normal", "hk", "0", "cM", None, None, "/out", "f.csv", ); assert!(!without.contains(&"--control")); } }