//! R/qtl2: genome scans with a linear mixed model, including panels whose //! relatedness makes `scanone` misleading. //! //! `/compute/rqtl` runs R/qtl's `scanone` (Haley-Knott, no kinship) and //! `/compute/gemma` fits an LMM but only for biallelic BIMBAM input with //! mean-imputed genotypes. R/qtl2 sits between them: genotype //! *probabilities* from a hidden Markov model, and an LMM with //! leave-one-chromosome-out kinship, which is how multiparent populations //! (DO, CC, 4-way) are analysed -- and a better-conditioned model than //! either of ours for a plain riset panel. //! //! Two pieces: //! //! - [`prepare_cross`] converts this deployment's own `.geno` file into the //! `read_cross2` control file and CSVs, in a cache directory, because the //! shipped `genotype/rqtl2/` copies are not usable: they name paths //! relative to a directory the data is not in, declare no genotype-code //! map (so every genotype is read as missing) and no orientation (so the //! individuals are read as markers). Four separate traps, all of which //! produce a plausible-looking flat scan rather than an error; //! - [`compute`] runs `scripts/qtl2_scan.R` over that cross and parses its //! `marker,chr,pos,lod` rows, the same shape `/compute/rqtl` returns and //! `/plot/rqtl` draws. //! //! Claimed: autosomes only. The X needs a sex covariate to be fitted //! properly (and `read_cross2` refuses a map it cannot square with one), //! and the Y and mitochondrion degenerate the per-chromosome kinship -- //! the same reasons GEMMA's LOCO run is given autosomal markers only. use std::collections::BTreeMap; use std::io; use std::path::{Path, PathBuf}; use serde_json::Value; use crate::genotypes::Resolved; use crate::rqtl::Sample; use crate::streaming; /// The columns of every row, in order -- `/compute/rqtl`'s shape, so the /// plot endpoint needs no changes. pub const COLUMNS: [&str; 4] = ["marker", "chr", "pos", "lod"]; #[derive(Debug)] pub enum Qtl2Error { Io(io::Error), /// The panel is a cross type this endpoint does not fit. Unsupported(String), /// R/qtl2 (or its script) failed; the message is its own complaint. CommandFailed(String), } impl std::fmt::Display for Qtl2Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Qtl2Error::Io(error) => write!(f, "{error}"), Qtl2Error::Unsupported(detail) => write!(f, "{detail}"), Qtl2Error::CommandFailed(detail) => write!(f, "{detail}"), } } } /// A converted cross, ready for `read_cross2`. #[derive(Debug, Clone)] pub struct PreparedCross { /// The control file, in the cross's own directory (qtl2 joins the /// control file's directory onto every path it finds, so absolute paths /// do not work). pub control: PathBuf, /// Individual names in the cross, in order. pub individuals: Vec, /// Markers fitted (autosomal). pub markers: usize, /// Markers left out as non-autosomal. pub non_autosomal: usize, /// qtl2's name for the cross: `riself`, `f2`, `do`, `4way`, … pub crosstype: String, /// Founder alleles the cross fits (8 for the DO). Zero when the cross /// carries no founder genotypes, which is every `.geno` panel here. pub founders: usize, /// Heterozygous calls the conversion wrote as missing. R/qtl's /// `convert2riself` drops the same ones; the count is what turns "the /// engines disagree" into "this panel has 14 % H at that marker". pub het_dropped: usize, /// Cells in the converted matrix (individuals x autosomal markers), for /// turning `het_dropped` into a percentage. pub cells: usize, /// True when qtl2 fits one coefficient per founder allele, so a LOD /// asks whether *any* founder differs rather than which allele at a /// biallelic marker does. pub multiparent: bool, } /// What the conversion does with a heterozygous call on a `riself` panel. /// /// R/qtl's `convert2riself` drops them, and so does the default here: an /// inbred line has no heterozygotes, so an `H` is a genotyping error or a /// residual one. But "drop" also means qtl2 never *sees* that call -- the HMM /// re-imputes it from flanking markers -- and on a panel where a marker is /// 14 % H (BXD.8's `rs33299937`) that is a different predictor from the one /// R/qtl fits. [`Het::Keep`] keeps the call as a third genotype, which makes /// the cross an `f2` for qtl2: a diagnostic, not a recommendation, because the /// het state is then modeled as a real genotype. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Het { /// Heterozygous calls become missing (the default; R/qtl's behaviour). Drop, /// Heterozygous calls become a third genotype (an `f2` cross). Keep, } impl Het { pub fn parse(value: Option<&str>) -> Result { match value.unwrap_or("drop").trim().to_ascii_lowercase().as_str() { "drop" | "missing" => Ok(Het::Drop), "keep" | "f2" => Ok(Het::Keep), other => Err(format!( "het '{other}' not recognized; use 'drop' (default) or 'keep'" )), } } pub fn as_str(self) -> &'static str { match self { Het::Drop => "drop", Het::Keep => "keep", } } } /// Cross types qtl2 fits one coefficient per founder allele for. Such a /// cross needs founder genotypes; without them qtl2 cannot resolve alleles /// and the scan is biallelic whatever the file is called. const MULTIPARENT_CROSSTYPES: [&str; 8] = ["do", "cc", "hs", "magic", "mp", "ail", "4way", "ri4way"]; /// Where a named cross's control file may live, under the genotype files /// root: the rqtl2 directory GeneNetwork shares, then the panel's own /// directory (`do/do.json`), then the root itself. const CONTROL_LOCATIONS: [&str; 3] = [ "genotype/rqtl2/{name}.json", "{name}/{name}.json", "{name}.json", ]; /// One finished scan. pub struct Qtl2Output { pub rows: Value, pub markers: usize, pub command: String, pub summary: String, /// The permutation null, when `nperm > 0`: one genome-wide LOD maximum per /// permutation, and the thresholds derived from them. `None` otherwise. pub permutation: Option, /// How the permutations were run, for the reply's notes. pub permutation_note: Option, } /// How many genotype cells carry one code -- the count a cached conversion /// needs to say what it dropped, without re-converting. fn count_calls(text: &str, code: &str) -> usize { let mut seen_header = false; let mut start = 3usize; let mut count = 0usize; for line in text.lines() { let line = line.trim_end_matches('\r'); if line.trim().is_empty() || line.starts_with('#') || line.starts_with('@') { continue; } let columns: Vec<&str> = line.split('\t').collect(); if !seen_header { seen_header = true; start = if columns.len() > 3 && columns[3] == "Mb" { 4 } else { 3 }; continue; } count += columns.iter().skip(start).filter(|cell| cell.trim() == code).count(); } count } /// The header's `@het` code, if it declares one. fn metadata_het(text: &str) -> Option { text.lines() .find_map(|line| line.strip_prefix("@het:")) .map(|value| value.trim().to_string()) } fn io_error(path: &Path, error: io::Error) -> Qtl2Error { Qtl2Error::Io(io::Error::new( error.kind(), format!("{}: {error}", path.display()), )) } /// Convert `resolved`'s `.geno` into a cached qtl2 cross. pub fn prepare_cross( resolved: &Resolved, cache_dir: &Path, het: Het, ) -> Result { let geno = &resolved.geno; let metadata = std::fs::metadata(geno).map_err(|e| io_error(geno, e))?; let stamp = format!( "{}-{}", metadata.len(), metadata .modified() .ok() .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) .map(|since| since.as_secs()) .unwrap_or(0) ); // The version tag is part of the key: a change to this conversion (the // heterozygous calls dropped for a riself panel, say) must not silently // reuse the previous cross's probabilities and kinship. // v3 keyed the heterozygous policy in as well: a cross converted with the // calls kept is a different cross from one with them dropped. const CONVERSION_VERSION: &str = "v3"; let directory = cache_dir.join("crosses").join(format!( "{CONVERSION_VERSION}-{}-{}-{stamp}", het.as_str(), resolved.version.replace(['/', '.'], "_") )); let control = directory.join("control.json"); let text = std::fs::read_to_string(geno).map_err(|e| io_error(geno, e))?; // Metadata: the genotype codes, and the cross type. let mut codes: BTreeMap = BTreeMap::new(); let mut cross_type = String::new(); for line in text.lines() { if let Some(rest) = line.strip_prefix('@') { let Some((key, value)) = rest.split_once(':') else { continue }; let value = value.trim().to_string(); match key.trim() { "mat" => { codes.insert(value, "A".to_string()); } "het" => { codes.insert(value, "H".to_string()); } "pat" => { codes.insert(value, "B".to_string()); } "type" => cross_type = value, _ => {} } } } let crosstype = match cross_type.as_str() { // A riset panel read as `f2` is a riset panel whose H calls are kept: // qtl2 needs a cross type that has a heterozygous state for that. "riset" if het == Het::Keep => "f2", "riset" => "riself", "f2" => "f2", other => { return Err(Qtl2Error::Unsupported(format!( "R/qtl2 here fits riset and f2 crosses; this panel is '{other}'. A multiparent \ cross needs founder genotypes (see genotype-files/do/), which a .geno file does \ not carry." ))) } }; // A riself panel is inbred lines: a heterozygous call is an error in the // genotyping, not a third allele. R/qtl's `convert2riself` omits those // genotypes, and qtl2 needs the same treatment -- left in, it reads the // cross as multiparent and its scan1 takes a path this panel cannot // satisfy ("scan1_pg ..."). let mut het_kept: Option = None; if crosstype == "riself" { let het_code = metadata_het(&text); if let Some(het_code) = het_code { codes.remove(&het_code); het_kept = Some(het_code); } } if codes.len() < 2 { return Err(Qtl2Error::Unsupported( "the .geno header declares no @mat/@pat codes, so its genotypes cannot be coded" .to_string(), )); } // Header row: Chr, Locus, cM, Mb?, then the individuals. let mut lines = text.lines(); let Some(header) = lines.find(|line| line.to_lowercase().starts_with("chr\t")) else { return Err(Qtl2Error::Unsupported( "the .geno file has no 'Chr' header row".to_string(), )); }; let fields: Vec<&str> = header.split('\t').collect(); let first_individual = if fields.get(3).copied() == Some("Mb") { 4 } else { 3 }; let individuals: Vec = fields[first_individual..] .iter() .map(|name| name.trim().to_string()) .collect(); if individuals.is_empty() { return Err(Qtl2Error::Unsupported( "the .geno header names no individuals".to_string(), )); } if !control.is_file() { std::fs::create_dir_all(&directory).map_err(|e| io_error(&directory, e))?; let mut geno_csv = String::new(); let mut gmap_csv = String::from("marker,chr,pos\n"); let mut pmap_csv = String::from("marker,chr,pos\n"); geno_csv.push_str("marker,"); geno_csv.push_str(&individuals.join(",")); geno_csv.push('\n'); let mut markers = 0usize; let mut non_autosomal = 0usize; let mut het_dropped = 0usize; for line in lines { let line = line.trim_end_matches('\r'); if line.trim().is_empty() || line.starts_with('#') { continue; } let row: Vec<&str> = line.split('\t').collect(); if row.len() < first_individual { continue; } let (chromosome, marker, centimorgans, megabases) = (row[0].trim(), row[1].trim(), row[2].trim(), row.get(3).copied().unwrap_or("")); // Autosomes only: see the module comment. if !chromosome.chars().all(|c| c.is_ascii_digit()) { non_autosomal += 1; continue; } markers += 1; geno_csv.push_str(marker); for value in &row[first_individual..] { let call = codes.get(value.trim()).map(String::as_str).unwrap_or(""); if call.is_empty() && het_kept.as_deref() == Some(value.trim()) { het_dropped += 1; } geno_csv.push(','); geno_csv.push_str(call); } geno_csv.push('\n'); gmap_csv.push_str(&format!("{marker},{chromosome},{centimorgans}\n")); let base_pairs = megabases.parse::().map(|mb| (mb * 1e6).round() as i64).unwrap_or(0); pmap_csv.push_str(&format!("{marker},{chromosome},{base_pairs}\n")); } if markers == 0 { return Err(Qtl2Error::Unsupported( "the panel has no autosomal markers to scan".to_string(), )); } // The cross needs a phenotype file of its own to be readable; the // trait comes from the request, through the script's --pheno. let mut pheno_csv = String::from("id,trait\n"); for name in &individuals { pheno_csv.push_str(&format!("{name},1\n")); } for (path, contents) in [ (directory.join("geno.csv"), &geno_csv), (directory.join("gmap.csv"), &gmap_csv), (directory.join("pmap.csv"), &pmap_csv), (directory.join("pheno.csv"), &pheno_csv), ] { std::fs::write(&path, contents).map_err(|e| io_error(&path, e))?; } let control_json = serde_json::json!({ // Paths are relative to the control file: qtl2 joins its own // directory onto whatever it finds here. "crosstype": crosstype, "geno": "geno.csv", "gmap": "gmap.csv", "pmap": "pmap.csv", "pheno": "pheno.csv", "sep": ",", "na.strings": [""], "genotypes": {"A": 1, "H": 2, "B": 3}, // Our own CSVs are markers-in-rows, individuals-in-columns. "geno_transposed": true, }); std::fs::write(&control, control_json.to_string()).map_err(|e| io_error(&control, e))?; // What the conversion discarded, kept beside the cross: a cached cross // must be able to say how many calls it dropped. let report = serde_json::json!({ "het_dropped": het_dropped, "cells": individuals.len() * markers, }); let report_path = directory.join("conversion.json"); let _ = std::fs::write(&report_path, report.to_string()); return Ok(PreparedCross { control, individuals: individuals.clone(), markers, non_autosomal, crosstype: crosstype.to_string(), het_dropped, cells: individuals.len() * markers, founders: 0, multiparent: false, }); } // Cached: count what the conversion produced, without re-reading the // panel. The marker counts come from the maps it wrote. let markers = std::fs::read_to_string(directory.join("pmap.csv")) .map_err(|e| io_error(&directory.join("pmap.csv"), e))? .lines() .filter(|line| !line.trim().is_empty()) .count() .saturating_sub(1); let report_path = directory.join("conversion.json"); let mut report: Value = std::fs::read_to_string(&report_path) .ok() .and_then(|text| serde_json::from_str(&text).ok()) .unwrap_or_else(|| serde_json::json!({})); if report.get("het_dropped").is_none() { // A cross converted before the count was kept: measure it once from the // panel and record it beside the cross, so later replies are free. let het_code = if crosstype == "riself" { metadata_het(&text) } else { None }; let dropped = het_code .as_deref() .map(|code| count_calls(&text, code)) .unwrap_or(0); report = serde_json::json!({ "het_dropped": dropped, "cells": individuals.len() * markers, }); let _ = std::fs::write(&report_path, report.to_string()); } let het_dropped = report .get("het_dropped") .and_then(Value::as_u64) .unwrap_or(0) as usize; let cells = report.get("cells").and_then(Value::as_u64).unwrap_or(0) as usize; Ok(PreparedCross { control, individuals, markers, non_autosomal: 0, crosstype: crosstype.to_string(), het_dropped, cells, founders: 0, multiparent: false, }) } /// Find the control file of a cross named in a request. pub fn resolve_control_file(root: &Path, requested: &str) -> Result { let name = requested.trim(); if name.is_empty() { return Err(Qtl2Error::Unsupported( "name a cross, e.g. \"do\" -- see /groups/{group}/genotypes".to_string(), )); } // A path the caller spelled out is taken as given: `genotype_file` // already accepts absolute paths, so this grants nothing new. if name.contains('/') || name.ends_with(".json") { let stated = Path::new(name); let path = if stated.is_absolute() { stated.to_path_buf() } else { root.join(stated) }; if path.is_file() { return Ok(path); } } let stem = name.trim_end_matches(".json"); for template in CONTROL_LOCATIONS { let path = root.join(template.replace("{name}", stem)); if path.is_file() { return Ok(path); } } Err(Qtl2Error::Unsupported(format!( "no R/qtl2 control file for '{name}' under {}; looked for {}. Only a cross that ships \ founder genotypes can be analysed as multiparent -- do/ is the one this deployment has. \ A .geno panel can be scanned, but only as biallelic: give `group` instead.", root.display(), CONTROL_LOCATIONS .iter() .map(|template| template.replace("{name}", stem)) .collect::>() .join(", ") ))) } /// A path a control file names, joined onto the control file's own directory /// (qtl2 reads every path relative to the file that names it). fn control_path(control: &Path, json: &Value, key: &str) -> Option { let directory = control.parent()?; json.get(key) .and_then(Value::as_str) .map(|name| directory.join(name)) } /// The data lines of a CSV: comments and blank lines skipped, header dropped. fn data_lines(path: &Path, comment: &str) -> Result, Qtl2Error> { let text = std::fs::read_to_string(path).map_err(|e| io_error(path, e))?; Ok(text .lines() .map(|line| line.trim_end_matches('\r').to_string()) .filter(|line| !line.trim().is_empty() && !line.starts_with(comment)) .skip(1) .collect()) } /// How many founders a `founder_geno` file has: their names are its header /// when it is transposed (the DO's is), its rows when it is not. fn count_founders( path: &Path, json: &Value, sep: &str, comment: &str, ) -> Result { let transposed = json .get("founder_geno_transposed") .and_then(Value::as_bool) .unwrap_or(false); let text = std::fs::read_to_string(path).map_err(|e| io_error(path, e))?; let mut lines = text.lines().map(str::trim_end).filter(|line| { !line.trim().is_empty() && !line.starts_with(comment) }); let Some(first) = lines.next() else { return Ok(0) }; Ok(if transposed { first.split(sep).count().saturating_sub(1) } else { lines.count() }) } /// Read a cross that ships as a qtl2 control file, founder genotypes and /// all. Nothing is converted, which is exactly what makes a multiparent scan /// possible: a `.geno` panel carries two codes per marker, not a founder /// allele. pub fn prepare_control_cross(root: &Path, requested: &str) -> Result { let control = resolve_control_file(root, requested)?; let text = std::fs::read_to_string(&control).map_err(|e| io_error(&control, e))?; let json: Value = serde_json::from_str(&text).map_err(|error| { Qtl2Error::Unsupported(format!( "{} is not a JSON control file: {error}", control.display() )) })?; let comment = json .get("comment.char") .and_then(Value::as_str) .unwrap_or("#") .to_string(); let sep = json .get("sep") .and_then(Value::as_str) .unwrap_or(",") .to_string(); let crosstype = json .get("crosstype") .and_then(Value::as_str) .unwrap_or("unknown") .to_string(); let pheno = control_path(&control, &json, "pheno").ok_or_else(|| { Qtl2Error::Unsupported(format!( "{}: the cross declares no `pheno` file, so it names no individuals", control.display() )) })?; let individuals: Vec = data_lines(&pheno, &comment)? .iter() .filter_map(|line| line.split(&sep).next()) .map(|name| name.trim().to_string()) .filter(|name| !name.is_empty()) .collect(); if individuals.is_empty() { return Err(Qtl2Error::Unsupported(format!( "{}: the phenotype file names no individuals", pheno.display() ))); } // Markers and chromosomes, from whichever map the cross declares. let map = control_path(&control, &json, "pmap").or_else(|| control_path(&control, &json, "gmap")); let mut markers = 0usize; let mut non_autosomal = 0usize; if let Some(map) = map { for line in data_lines(&map, &comment)? { let fields: Vec<&str> = line.split(&sep).collect(); let Some(chromosome) = fields.get(1) else { continue }; if chromosome.trim().chars().all(|c| c.is_ascii_digit()) { markers += 1; } else { non_autosomal += 1; } } } let founders = match control_path(&control, &json, "founder_geno") { Some(path) => count_founders(&path, &json, &sep, &comment)?, None => 0, }; let multiparent = MULTIPARENT_CROSSTYPES.contains(&crosstype.as_str()); if multiparent && founders < 3 { return Err(Qtl2Error::Unsupported(format!( "{}: crosstype '{crosstype}' is multiparent, but the cross declares no founder \ genotypes, so qtl2 cannot say which allele a genotype is. Point `founder_geno` at a \ founder genotype file, as do/do.json does.", control.display() ))); } Ok(PreparedCross { control, individuals, markers, non_autosomal, crosstype, het_dropped: 0, cells: 0, founders, multiparent, }) } /// Run the scan over a prepared cross. #[allow(clippy::too_many_arguments)] pub async fn compute( script: &str, cache_dir: &str, cross: &PreparedCross, trait_name: &str, samples: &[Sample], covariates: &[crate::covariates::Covariate], kinship: &str, model: &str, cores: u32, permutations: Permutations<'_>, ) -> Result { let out_dir = Path::new(cache_dir); std::fs::create_dir_all(out_dir).map_err(|e| io_error(out_dir, e))?; let unique = format!( "{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_nanos()) .unwrap_or(0) ); let pheno_path = out_dir.join(format!("{unique}-pheno.txt")); crate::rqtl::write_pheno_file(&pheno_path, trait_name, samples, covariates) .map_err(Qtl2Error::Io)?; let covarstruct_path = out_dir.join(format!("{unique}-covarstruct.txt")); let covarstruct_arg = if covariates.is_empty() { None } else { crate::covariates::write_covarstruct_file(&covarstruct_path, covariates) .map_err(Qtl2Error::Io)?; Some(covarstruct_path.to_string_lossy().into_owned()) }; let result_path = out_dir.join(format!("{unique}-qtl2.csv")); let cores_str = cores.to_string(); let result_arg = result_path.to_string_lossy().into_owned(); let pheno_arg = pheno_path.to_string_lossy().into_owned(); let control_arg = cross.control.to_string_lossy().into_owned(); let mut args = vec![ script.to_string(), "--control".to_string(), control_arg, "--pheno".to_string(), pheno_arg, "--kinship".to_string(), kinship.to_string(), "--model".to_string(), model.to_string(), "--cores".to_string(), cores_str, "--cache-dir".to_string(), cache_dir.to_string(), "--out".to_string(), result_arg, "--quiet".to_string(), ]; if let Some(covarstruct) = covarstruct_arg.as_deref() { args.push("--covariates".to_string()); args.push(covarstruct_path.to_string_lossy().into_owned()); args.push("--covarstruct".to_string()); args.push(covarstruct.to_string()); } // The permutations write their own file: one LOD per permutation, in the // same two-column shape R/qtl's scanone permutations produce, so the // thresholds come from the same percentile code either way. let perm_path = out_dir.join(format!("{unique}-perm.csv")); let mut permutation_note = None; let mut strata_path_arg: Option = None; if let Some(permutations) = permutations.enabled() { if !permutations.samples.is_empty() { let strata_path = out_dir.join(format!("{unique}-strata.txt")); crate::rqtl::write_strata_file(&strata_path, permutations.samples) .map_err(Qtl2Error::Io)?; strata_path_arg = Some(strata_path.to_string_lossy().into_owned()); } args.push("--nperm".to_string()); args.push(permutations.count.to_string()); args.push("--seed".to_string()); args.push(permutations.seed.to_string()); args.push("--perm-out".to_string()); args.push(perm_path.to_string_lossy().into_owned()); permutation_note = Some(format!( "{} permutations (seed {}), strata: {}", permutations.count, permutations.seed, if permutations.samples.is_empty() { "none -- permuted across all individuals".to_string() } else { format!("{} (permuted within)", permutations.strata_name) } )); if let Some(strata) = strata_path_arg.as_deref() { args.push("--strata".to_string()); args.push(strata.to_string()); } } let borrowed: Vec<&str> = args.iter().map(String::as_str).collect(); let command = format!("Rscript {}", borrowed.join(" ")); let output = streaming::run_streaming("Rscript", &borrowed, None) .await .map_err(Qtl2Error::Io)?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); let stdout = String::from_utf8_lossy(&output.stdout); // R ends every failure with "Execution halted"; the sentence before // it is the error. Prefer an `Error` line, then the wrapper's own // output, so a caller sees the cause rather than the epilogue. let interesting = |line: &&str| { let line = line.trim(); !line.is_empty() && !line.starts_with("Warning") && !line.starts_with("In ") && line != "Execution halted" }; let lines: Vec<&str> = stderr .lines() .chain(stdout.lines()) .filter(interesting) .collect(); // R wraps a long call across lines, so the sentence that matters is // often the line *after* "Error in foo(...) :". Take the Error line // and its continuation rather than half a complaint. let complaint = lines .iter() .position(|line| line.trim_start().starts_with("Error") || line.contains("Error in")) .map(|start| { lines[start..] .iter() .take(3) .map(|line| line.trim()) .filter(|line| !line.starts_with("Calls:")) .collect::>() .join(" ") }) .or_else(|| lines.last().map(|line| line.trim().to_string())) .unwrap_or_else(|| "R/qtl2 failed without saying why".to_string()); return Err(Qtl2Error::CommandFailed(complaint)); } let contents = std::fs::read_to_string(&result_path).map_err(|e| io_error(&result_path, e))?; let rows = parse_rows(&contents); let permutation = if permutations.enabled().is_some() { let perm_contents = std::fs::read_to_string(&perm_path).map_err(|e| io_error(&perm_path, e))?; let values = parse_permutations(&perm_contents); if values.is_empty() { return Err(Qtl2Error::CommandFailed( "R/qtl2 ran the permutations but wrote no values".to_string(), )); } Some(crate::rqtl::PermutationResult::from_values(values)) } else { None }; Ok(Qtl2Output { markers: rows.len(), rows: Value::Array(rows), command, permutation, permutation_note, summary: String::from_utf8_lossy(&output.stdout) .lines() .filter(|line| !line.trim().is_empty()) .next_back() .unwrap_or_default() .trim() .to_string(), }) } /// What a caller asked for in the way of permutations. /// /// `count == 0` means "none", and then nothing else is read -- so a request /// that leaves the whole thing out behaves exactly as before this existed. pub struct Permutations<'a> { pub count: u32, pub seed: u64, /// (individual, stratum) pairs, empty when permutation is not stratified. pub samples: &'a [(String, String)], /// The case attribute the strata came from, for the note. pub strata_name: &'a str, } impl Permutations<'_> { fn enabled(&self) -> Option<&Self> { if self.count == 0 { None } else { Some(self) } } } /// Parse the script's permutation CSV: a `lod` header and one value per line. /// /// Deliberately tolerant of a blank second column: R/qtl's own permutation /// file is `"","lod"`, and writing the same shape from both engines is the /// goal, so a two-column file reads the same way here. fn parse_permutations(contents: &str) -> Vec { contents .lines() .skip(1) .filter(|line| !line.trim().is_empty()) .filter_map(|line| { let fields: Vec<&str> = line.split(',').map(str::trim).collect(); fields .iter() .rev() .find_map(|field| field.trim_matches('"').trim().parse::().ok()) }) .collect() } /// Parse the script's `marker,chr,pos,lod` CSV into positional rows. fn parse_rows(contents: &str) -> Vec { contents .lines() .skip(1) .filter(|line| !line.trim().is_empty()) .filter_map(|line| { let fields: Vec<&str> = line.split(',').map(str::trim).collect(); if fields.len() < 4 { return None; } Some(Value::Array(vec![ Value::String(fields[0].to_string()), Value::String(fields[1].to_string()), serde_json::json!(fields[2].parse::().ok()?), serde_json::json!(fields[3].parse::().ok()?), ])) }) .collect() } #[cfg(test)] mod tests { use super::*; #[test] fn permutation_csv_is_one_lod_per_line() { // What scripts/qtl2_scan.R writes: a `lod` header, then the genome-wide // maximum of each permutation. Quoted or not, and with or without a // leading empty column (R/qtl's shape), the numbers are what matter. let one_column = "lod\n1.3574\n1.3550\n2.1\n"; assert_eq!(parse_permutations(one_column), vec![1.3574, 1.3550, 2.1]); let two_column = "\"\",lod\n\"1\",1.5\n\"2\",2.5\n"; assert_eq!(parse_permutations(two_column), vec![1.5, 2.5]); assert!(parse_permutations("lod\n").is_empty()); } #[test] fn permutation_thresholds_are_percentiles_of_the_maxima() { // 100 maxima from 0.01 to 1.00. The percentiles are linearly // interpolated at position (n-1)*p/100 -- numpy's default and R's // quantile type 7 -- so the 67th is between 0.67 and 0.68 (0.6733) and // the 95th between 0.95 and 0.96 (0.9505), not the round values. let values: Vec = (1..=100).map(|n| n as f64 / 100.0).collect(); let result = crate::rqtl::PermutationResult::from_values(values); assert!((result.suggestive - 0.6733).abs() < 0.0001, "{}", result.suggestive); assert!((result.significant - 0.9505).abs() < 0.0001, "{}", result.significant); assert_eq!(result.values.len(), 100); } #[test] fn rows_are_positional_marker_chr_pos_lod() { let rows = parse_rows("marker,chr,pos,lod\nrs1,9,74.99,4.85\nrs2,X,3.0,0.5\n"); assert_eq!(rows.len(), 2); let row = rows[0].as_array().unwrap(); assert_eq!(row[0], Value::String("rs1".to_string())); assert_eq!(row[1], Value::String("9".to_string())); assert_eq!(row[2], serde_json::json!(74.99)); assert_eq!(row[3], serde_json::json!(4.85)); } /// A cross directory shaped like `do/`: a control file that names its /// own files, founder genotypes and all. fn founder_cross(root: &Path, crosstype: &str, with_founders: bool) -> PathBuf { let directory = root.join("do"); std::fs::create_dir_all(&directory).unwrap(); let mut control = serde_json::json!({ "crosstype": crosstype, "sep": ",", "comment.char": "#", "geno": "geno.csv", "gmap": "gmap.csv", "pmap": "pmap.csv", "pheno": "pheno.csv", }); if with_founders { control["founder_geno"] = serde_json::json!("founder.csv"); control["founder_geno_transposed"] = serde_json::json!(true); std::fs::write( directory.join("founder.csv"), "# founder genotype data\nmarker,A,B,C,D,E,F,G,H\nm1,0,1,3,0,1,3,0,1\n", ) .unwrap(); } std::fs::write( directory.join("pheno.csv"), "# phenotype data\nid,WBC\nS1,4.3\nS2,5.1\n", ) .unwrap(); std::fs::write( directory.join("pmap.csv"), "# physical map\nmarker,chr,pos\nm1,1,3000000\nm2,1,9000000\nm3,X,4000000\n", ) .unwrap(); let path = directory.join("do.json"); std::fs::write(&path, control.to_string()).unwrap(); path } fn scratch_dir(name: &str) -> PathBuf { let directory = std::env::temp_dir().join(format!( "gn4-qtl2-test-{name}-{}-{:?}", std::process::id(), std::thread::current().id() )); let _ = std::fs::remove_dir_all(&directory); std::fs::create_dir_all(&directory).unwrap(); directory } #[test] fn a_named_cross_is_found_where_qtl2_keeps_it() { let root = scratch_dir("resolve"); founder_cross(&root, "do", true); let found = resolve_control_file(&root, "do").unwrap(); assert!(found.ends_with("do/do.json"), "{}", found.display()); // The rqtl2 directory GeneNetwork shares comes first. std::fs::create_dir_all(root.join("genotype/rqtl2")).unwrap(); std::fs::write(root.join("genotype/rqtl2/cc.json"), "{}").unwrap(); assert!(resolve_control_file(&root, "cc") .unwrap() .ends_with("genotype/rqtl2/cc.json")); // A name nobody has: the refusal says where it looked. let error = resolve_control_file(&root, "nonesuch").unwrap_err().to_string(); assert!(error.contains("no R/qtl2 control file"), "{error}"); assert!(error.contains("genotype/rqtl2/nonesuch.json"), "{error}"); std::fs::remove_dir_all(&root).ok(); } #[test] fn a_multiparent_cross_reports_its_founders() { let root = scratch_dir("multiparent"); founder_cross(&root, "do", true); let cross = prepare_control_cross(&root, "do").unwrap(); assert_eq!(cross.crosstype, "do"); assert!(cross.multiparent, "the DO is multiparent"); assert_eq!(cross.founders, 8, "eight founder alleles, from the founder header"); assert_eq!(cross.individuals, vec!["S1".to_string(), "S2".to_string()]); assert_eq!(cross.markers, 2, "autosomal markers only"); assert_eq!(cross.non_autosomal, 1, "the X is counted, not scanned"); std::fs::remove_dir_all(&root).ok(); } #[test] fn a_multiparent_crosstype_without_founder_genotypes_is_refused() { let root = scratch_dir("no-founders"); founder_cross(&root, "do", false); let error = prepare_control_cross(&root, "do").unwrap_err().to_string(); assert!(error.contains("no founder genotypes"), "{error}"); std::fs::remove_dir_all(&root).ok(); } #[test] fn a_biallelic_cross_is_not_mistaken_for_multiparent() { let root = scratch_dir("biallelic"); founder_cross(&root, "riself", false); // The file lives in do/ either way; the *crosstype inside* is what // decides whether founder alleles are fitted. let cross = prepare_control_cross(&root, "do").unwrap(); assert!(!cross.multiparent); assert_eq!(cross.founders, 0); std::fs::remove_dir_all(&root).ok(); } #[test] fn a_broken_row_is_skipped_rather_than_reported_as_zero() { let rows = parse_rows("marker,chr,pos,lod\nrs1,1,not-a-number,2.0\nrs2,1,3.0,4.0\n"); assert_eq!(rows.len(), 1); } }