//! Genotype files: which versions a group has, which one is current, and //! the panel each one describes. //! //! The layout is GeneNetwork2's `genotype_files` tree (see //! docs/genotypes.md): `genotype/.json` lists a group's genotype //! files newest-first (`genofile: [{title, location, sample_list?}]`), //! `genotype/..geno` are the files themselves, and GEMMA's //! input is the `bimbam/_geno.txt` + `_snps.txt` pair derived //! from the same stem. //! //! Two rules come straight from the older services and are reproduced //! here rather than invented: //! //! - **current = first**: GN2 offers the JSON's list to the mapping UI //! and defaults to its first entry (`datasetgroup.py:get_genofiles`). //! For groups with no JSON (HET3-ITP has none) there is no recorded //! order, so the highest version number wins -- a guess, but the same //! one a human makes, and the listing says where it came from. //! - **the panel is the `.geno` header**: the first line that is not a //! `#` comment or `@` metadata line, sample names from column 5 when //! column 4 says `Mb`, otherwise from column 4 (`gn3/db/genotypes.py`). //! //! Nothing here serves a file's *contents* or its absolute path to a //! caller: the listing endpoint reports versions and sizes, and the //! mapping endpoints take a group and a version, resolving the path //! themselves. Paths only appear in responses relativised to the //! genotype-files directory, which is what makes a `notes` line //! reproducible without publishing the deployment's layout. use std::collections::BTreeSet; use std::path::{Path, PathBuf}; /// Which engine a prepared file set is for. #[derive(Clone, Copy, PartialEq, Eq)] pub enum Engine { /// `genotype/.geno` (GN format), as R/qtl's wrapper reads it. Rqtl, /// `genotype/bimbam/_geno.txt` + `_snps.txt`. Gemma, } /// A genotyping error a caller can act on. #[derive(Debug)] pub enum GenotypeError { /// No such group, version, or required file. NotFound(String), /// Reading the directory, the JSON, or a `.geno` header failed. Io(String), /// The request names an individual (or selection) that cannot be honoured: /// the caller's fault, not a missing file. Invalid(String), } impl std::fmt::Display for GenotypeError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { GenotypeError::NotFound(what) | GenotypeError::Invalid(what) => write!(f, "{what}"), GenotypeError::Io(what) => write!(f, "{what}"), } } } /// One genotype file a group offers. pub struct GenotypeFile { /// The stem, e.g. "BXD.8" -- what a mapping request names. pub version: String, /// The genotype file's name, e.g. "BXD.8.geno". A name, not a path: /// enough to say what ran, not where the deployment keeps it. pub file: String, /// The JSON's `title`, when it has one, e.g. "WGS-based (Mar2022)". pub title: Option, /// Individuals in the file's header. pub individuals: usize, /// Marker rows in the file. pub markers: usize, /// Whether the `bimbam/` pair exists, i.e. GEMMA can run on it. pub gemma: bool, /// Whether the `.geno` is readable, i.e. R/qtl can run on it. pub rqtl: bool, /// First in the list, so what a mapping request gets by default. pub current: bool, /// Where the ordering came from: the group JSON, or the file names. pub order_source: OrderSource, } /// Where a group's version order came from. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum OrderSource { /// `genotype/.json`, which records the order. Json, /// Highest `..geno` first: inferred, not recorded. FileNames, } /// A resolved genotype file set for one group at one version. #[derive(Debug, Clone)] pub struct Resolved { pub version: String, pub title: Option, /// Absolute path to the GN `.geno` (R/qtl's input). pub geno: PathBuf, /// Absolute path to GEMMA's genotype, when the bimbam pair exists. pub bimbam_geno: Option, /// Absolute path to GEMMA's SNP map, when it exists. pub bimbam_snps: Option, /// Panel sample names, in the file's own column order. pub panel: Vec, } impl Resolved { /// Path relative to the genotype-files directory, for `notes` -- /// reproducible without publishing where the deployment keeps things. pub fn relative(&self, dir: &Path, path: &Path) -> String { path.strip_prefix(dir) .unwrap_or(path) .to_string_lossy() .into_owned() } } /// The largest `.geno` a whole-file read will touch: the markers listing, a /// panel-wide tally. Above this the answer is another call (one marker, a /// smaller version), not a wait -- `HSNIH-Palmer_true.geno` is 3.7 GB. pub const MAX_SCAN_BYTES: u64 = 256 * 1024 * 1024; /// The `.geno` a request named when no group lists it: a bare version /// (`BXD.4`), a `.geno` name, or a path relative to the genotype files /// directory. pub fn resolve_named(root: &Path, requested: &str) -> Result { let name = requested.trim(); if name.is_empty() || name.contains("..") { return Err(GenotypeError::NotFound(format!( "'{requested}' is not a genotype file name" ))); } let genotype_dir = root.join("genotype"); let mut candidates = Vec::new(); if name.ends_with(".geno") { candidates.push(genotype_dir.join(name)); candidates.push(root.join(name)); } else { candidates.push(genotype_dir.join(format!("{name}.geno"))); candidates.push(genotype_dir.join(name)); candidates.push(root.join(name)); } for path in &candidates { if path.is_file() { return Ok(path.clone()); } } Err(GenotypeError::NotFound(format!( "no genotype file '{name}' under {}; looked for {}", root.display(), candidates .iter() .map(|path| path.display().to_string()) .collect::>() .join(", ") ))) } fn io_err(path: &Path, error: std::io::Error) -> GenotypeError { GenotypeError::Io(format!("{}: {error}", path.display())) } /// The `.geno` header's sample names, in column order (GN3's rule). pub fn panel_of(geno: &Path) -> Result, GenotypeError> { let text = std::fs::read_to_string(geno).map_err(|e| io_err(geno, e))?; for line in text.lines() { let line = line.trim_end_matches(['\r', '\n']); if line.trim().is_empty() || line.starts_with('#') || line.starts_with('@') { continue; } let columns: Vec<&str> = line.split('\t').collect(); let start = if columns.len() > 3 && columns[3] == "Mb" { 4 } else { 3 }; return Ok(columns[start.min(columns.len())..] .iter() .map(|name| name.trim().to_string()) .collect()); } Err(GenotypeError::NotFound(format!( "{}: no header line", geno.display() ))) } /// Number of marker rows in a `.geno` (every non-comment line but the /// header). pub fn marker_count(geno: &Path) -> Result { let text = std::fs::read_to_string(geno).map_err(|e| io_err(geno, e))?; let mut seen_header = false; let mut markers = 0; for line in text.lines() { if line.trim().is_empty() || line.starts_with('#') || line.starts_with('@') { continue; } if seen_header { markers += 1; } else { seen_header = true; } } Ok(markers) } /// The versions a group offers, newest first. pub fn list(dir: &Path, group: &str) -> Result, GenotypeError> { let genotype_dir = dir.join("genotype"); if !genotype_dir.is_dir() { return Err(GenotypeError::NotFound(format!( "no genotype-files directory at {}", dir.display() ))); } let (stems, source) = match json_versions(&genotype_dir, group)? { Some(entries) => (entries, OrderSource::Json), None => (scanned_versions(&genotype_dir, group), OrderSource::FileNames), }; if stems.is_empty() { return Err(GenotypeError::NotFound(format!( "no genotype files for group '{group}'" ))); } let mut files = Vec::with_capacity(stems.len()); for (index, (stem, title)) in stems.into_iter().enumerate() { let geno = genotype_dir.join(format!("{stem}.geno")); let bimbam_geno = genotype_dir.join(format!("bimbam/{stem}_geno.txt")); let bimbam_snps = genotype_dir.join(format!("bimbam/{stem}_snps.txt")); let rqtl = geno.is_file(); let (individuals, markers) = if rqtl { ( panel_of(&geno).map(|panel| panel.len()).unwrap_or(0), marker_count(&geno).unwrap_or(0), ) } else { (0, 0) }; files.push(GenotypeFile { file: format!("{stem}.geno"), version: stem, title, individuals, markers, gemma: bimbam_geno.is_file() && bimbam_snps.is_file(), rqtl, current: index == 0, order_source: source, }); } Ok(files) } /// The version list from `genotype/.json`, when it has one. The /// file is also used for older marker dumps (a bare JSON list), which are /// not a version list and are ignored. fn json_versions( genotype_dir: &Path, group: &str, ) -> Result)>>, GenotypeError> { let path = genotype_dir.join(format!("{group}.json")); if !path.is_file() { return Ok(None); } let text = std::fs::read_to_string(&path).map_err(|e| io_err(&path, e))?; let document: serde_json::Value = serde_json::from_str(&text) .map_err(|e| GenotypeError::Io(format!("{}: {e}", path.display())))?; let Some(entries) = document.get("genofile").and_then(|v| v.as_array()) else { return Ok(None); }; let mut versions = Vec::with_capacity(entries.len()); for entry in entries { let location = entry.get("location").and_then(|v| v.as_str()); let Some(location) = location else { continue }; let stem = location.strip_suffix(".geno").unwrap_or(location); let title = entry .get("title") .and_then(|v| v.as_str()) .map(str::to_string); versions.push((stem.to_string(), title)); } if versions.is_empty() { return Ok(None); } Ok(Some(versions)) } /// `.geno` plus `..geno`, highest N first, worst case. fn scanned_versions(genotype_dir: &Path, group: &str) -> Vec<(String, Option)> { let Ok(entries) = std::fs::read_dir(genotype_dir) else { return Vec::new(); }; let mut numbered: Vec<(u32, String)> = Vec::new(); let mut plain = None; for entry in entries.flatten() { let name = entry.file_name().to_string_lossy().into_owned(); let Some(stem) = name.strip_suffix(".geno") else { continue; }; if stem == group { plain = Some(stem.to_string()); continue; } let Some(rest) = stem.strip_prefix(&format!("{group}.")) else { continue; }; if let Ok(number) = rest.parse::() { numbered.push((number, stem.to_string())); } } numbered.sort_by(|a, b| b.0.cmp(&a.0)); numbered .into_iter() .map(|(_, stem)| (stem, None)) .chain(plain.map(|stem| (stem, None))) .collect() } /// Resolve one version (`None` = the current one) to the files an engine /// needs. pub fn resolve( dir: &Path, group: &str, version: Option<&str>, ) -> Result { let files = list(dir, group)?; let chosen = match version { Some(wanted) => files .iter() .find(|file| file.version == wanted || file.version == format!("{group}.{wanted}")) .ok_or_else(|| { GenotypeError::NotFound(format!( "no genotype file '{wanted}' for group '{group}'; available: {}", files .iter() .map(|file| file.version.as_str()) .collect::>() .join(", ") )) })?, None => &files[0], }; let genotype_dir = dir.join("genotype"); let geno = genotype_dir.join(format!("{}.geno", chosen.version)); let panel = panel_of(&geno)?; let bimbam = |suffix: &str| { let path = genotype_dir.join(format!("bimbam/{}_{suffix}", chosen.version)); path.is_file().then_some(path) }; Ok(Resolved { version: chosen.version.clone(), title: chosen.title.clone(), geno, bimbam_geno: bimbam("geno.txt"), bimbam_snps: bimbam("snps.txt"), panel, }) } /// The caller's own choice of individuals, on top of the parent/F1 rule. /// /// Why this exists: `parents_and_f1s.json` is *data* about seven groups, and a /// request cannot reach it -- a group it does not list, a renamed parent, or a /// substrain the caller wants gone (`BXD48a` beside `BXD48`) all needed a code /// or data change. Naming individuals is also safer than pattern matching: the /// F1 names differ per group (`B6D2F1` for BXD, `BHF1`/`HBF1` for BXH, /// `C57BL/6JxBXD037F1` for the intercross F1s). /// /// `exclude` wins over `include`, so the two lists cannot contradict silently. /// An unknown name is an error rather than a shrug: a dropped animal changes a /// kinship matrix without anyone noticing. #[derive(Debug, Clone, Default, serde::Deserialize, utoipa::ToSchema)] pub struct Individuals { /// Names to keep even where the parent/F1 rule would drop them. #[serde(default)] pub include: Vec, /// Names to drop in addition to (or instead of) the parent/F1 rule. #[serde(default)] pub exclude: Vec, } impl Individuals { pub fn is_empty(&self) -> bool { self.include.is_empty() && self.exclude.is_empty() } } /// What a selection did, so a reply can say it rather than leave a reader to /// infer why a panel shrank. #[derive(Debug, Clone, Default)] pub struct Selection { /// Panel indices that enter the genotype file, in panel order. pub keep: Vec, /// Individuals dropped by the parent/F1 rule (and not re-included). pub dropped_parents_f1: usize, /// Individuals the caller named to keep against that rule. pub included_by_name: usize, /// Individuals the caller named to drop. pub excluded_by_name: usize, } /// Choose the individuals of a panel: the parent/F1 rule first, then the /// caller's `individuals` lists. /// /// `include_parents_f1` short-circuits the first step, exactly as before; the /// caller's lists still apply afterwards, so "everyone except X" and "only /// these two F1s" are both expressible. pub fn select_panel( panel: &[String], parents_f1s: &BTreeSet, include_parents_f1: bool, individuals: &Individuals, ) -> Result { let position = |name: &str| panel.iter().position(|member| member == name); let unknown = |name: &str, list: &str| { let sample: Vec<&str> = panel.iter().take(6).map(String::as_str).collect(); GenotypeError::Invalid(format!( "individual '{name}' (in `individuals.{list}`) is not in this panel, which has {} \ member(s), starting: {}{}", panel.len(), sample.join(", "), if panel.len() > sample.len() { ", …" } else { "" } )) }; // Resolve the names first, so a typo is reported even when the panel would // otherwise have been filtered to nothing. let mut include: Vec = Vec::new(); for name in &individuals.include { include.push(position(name).ok_or_else(|| unknown(name, "include"))?); } let mut exclude: Vec = Vec::new(); for name in &individuals.exclude { exclude.push(position(name).ok_or_else(|| unknown(name, "exclude"))?); } let mut keep = Vec::with_capacity(panel.len()); let mut dropped_parents_f1 = 0; let mut included_by_name = 0; for (index, name) in panel.iter().enumerate() { let is_parent_or_f1 = !include_parents_f1 && parents_f1s.contains(name); if exclude.contains(&index) { continue; } if include.contains(&index) { if is_parent_or_f1 { included_by_name += 1; } keep.push(index); continue; } if is_parent_or_f1 { dropped_parents_f1 += 1; continue; } keep.push(index); } if keep.is_empty() { return Err(GenotypeError::Invalid( "no individuals are left after the parent/F1 rule and `individuals`; \n the request would map nothing".to_string(), )); } let excluded_by_name = exclude.len(); Ok(Selection { keep, dropped_parents_f1, included_by_name, excluded_by_name, }) } /// The individuals a mapping should exclude by default: parents and F1s, /// as in GN2 (`incparentsf1` off) and GN3 (`inc_par=False, inc_f1=False`). /// /// GN2 keeps the lists in `parents_and_f1s.json`; the F1 columns in the /// BXD file are additionally named `xF1`, which is what /// catches them when the JSON has no entry for the group. /// One role in a parental design: a strain, and the individuals of it that a /// group's metadata names (BXD-Longevity lists them; most groups do not). #[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] pub struct DesignRole { pub strain: String, /// Individuals the metadata attributes to this role, when it says. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub individuals: Vec, } /// A family's parental design: who the parents are, which F1s were made, and /// what a mapping does with them. /// /// This is metadata (`parents_and_f1s.json`, ported from GN2's `ParInfo`), not /// policy: a caller can *see* it instead of inferring why an individual was /// dropped, and can name one back with `individuals.include`. #[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] pub struct ParentalDesign { pub group: String, pub f1s: Vec, pub maternal: Option, pub paternal: Option, /// Where the roles came from: "metadata" when the group is listed, "name /// heuristic" when only the `F1` substring rule applies (an unlisted group /// has parents nobody declared, and its mapping may include strains that /// are not RI lines). pub source: &'static str, /// The `@type` of the group's genotype file, e.g. `riset` or `f2`. pub design: Option, /// Individuals of the current panel the default rule would drop. pub excluded_from_panel: Vec, /// Panel members the `F1` name heuristic catches -- reported separately, /// because it is a heuristic and an unlisted group's parents are invisible /// to it. pub excluded_by_name: Vec, } /// The roles a group's metadata declares, or `None` when the group is unlisted. fn design_roles(dir: &Path, group: &str) -> Option<(Vec, Option, Option)> { let path = dir.join("parents_and_f1s.json"); let text = std::fs::read_to_string(&path).ok()?; let document: serde_json::Value = serde_json::from_str(&text).ok()?; let entry = document.get(group)?; let role = |key: &str| { entry.get(key).and_then(|value| { let strain = value.get("strain").and_then(|s| s.as_str())?.trim(); if strain.is_empty() { return None; } Some(DesignRole { strain: strain.to_string(), individuals: value .get("individuals") .and_then(|list| list.as_array()) .map(|list| { list.iter() .filter_map(|name| name.as_str()) .map(str::to_string) .collect() }) .unwrap_or_default(), }) }) }; let f1s: Vec = entry .get("f1s") .and_then(|list| list.as_array()) .map(|list| { list.iter() .filter_map(|value| { let strain = value.get("strain").and_then(|s| s.as_str())?.trim(); if strain.is_empty() { return None; } Some(DesignRole { strain: strain.to_string(), individuals: value .get("individuals") .and_then(|list| list.as_array()) .map(|list| { list.iter() .filter_map(|name| name.as_str()) .map(str::to_string) .collect() }) .unwrap_or_default(), }) }) .collect() }) .unwrap_or_default(); Some((f1s, role("maternal"), role("paternal"))) } /// The parental design of one group, as `/groups/{group}/parents-and-f1s` /// reports it. /// /// `panel` is the group's individual list; it is what turns the metadata into /// "these members would be dropped", which is the fact a caller needs. An /// unlisted group still gets an answer: `source` says the roles are unknown and /// only the name heuristic applied. pub fn parental_design(dir: &Path, group: &str, panel: Option<&[String]>, design: Option) -> ParentalDesign { let roles = design_roles(dir, group); let (f1s, maternal, paternal) = roles.clone().unwrap_or_default(); let empty: &[String] = &[]; let panel = panel.unwrap_or(empty); let declared: Vec<&str> = f1s .iter() .map(|role| role.strain.as_str()) .chain(maternal.iter().map(|role| role.strain.as_str())) .chain(paternal.iter().map(|role| role.strain.as_str())) .collect(); let excluded_from_panel: Vec = panel .iter() .filter(|name| declared.contains(&name.as_str())) .cloned() .collect(); let excluded_by_name: Vec = panel .iter() .filter(|name| name.contains("F1")) .cloned() .collect(); ParentalDesign { group: group.to_string(), f1s, maternal, paternal, source: if roles.is_some() { "metadata" } else { "name heuristic" }, design, excluded_from_panel, excluded_by_name, } } /// The individuals a mapping excludes by default: parents and F1s. /// /// The roles come from `parents_and_f1s.json` (GN2's `ParInfo`, roles named /// rather than positional -- GN2 unpacks it as `f1, f12, maternal, paternal`, /// and the file once had those two swapped). The `F1` substring rule is a /// fallback for groups the metadata does not list, and an unlisted group's /// parents are invisible to it: that is why `parental_design` reports which /// source answered. pub fn non_strains(dir: &Path, group: &str, panel: &[String]) -> BTreeSet { let mut excluded = BTreeSet::new(); if let Some((f1s, maternal, paternal)) = design_roles(dir, group) { for role in f1s .iter() .chain(maternal.iter()) .chain(paternal.iter()) { excluded.insert(role.strain.clone()); } } for name in panel { if name.contains("F1") { excluded.insert(name.clone()); } } excluded } /// One marker's position, as the `.geno` records it -- the metadata a /// caller needs to interpret a mapping, without the genotypes themselves. #[derive(Debug, Clone)] pub struct Marker { pub name: String, pub chr: String, pub cm: Option, pub mb: Option, } /// Everything a `.geno` header and marker table say, and nothing it does /// not: the `@key:value` labels, the panel's individual names in file /// order, and one entry per marker. #[derive(Debug)] pub struct FileMeta { /// The `@`-labels (`name`, `type`, `mat`, `pat`, `het`, `unk`, /// `scale`), in file order. pub labels: Vec<(String, String)>, /// Individuals, in the file's column order. pub individuals: Vec, /// Markers, in file order. pub markers: Vec, } /// Read a `.geno`'s metadata: labels, individuals, marker positions. The /// genotype calls themselves are read and discarded, never returned. pub fn file_meta(geno: &Path) -> Result { let text = std::fs::read_to_string(geno).map_err(|e| io_err(geno, e))?; let mut labels = Vec::new(); let mut individuals = Vec::new(); let mut markers = Vec::new(); let mut header_seen = false; let mut start = 3; for line in text.lines() { let line = line.trim_end_matches('\r'); if line.trim().is_empty() || line.starts_with('#') { continue; } if let Some(rest) = line.strip_prefix('@') { if let Some((key, value)) = rest.split_once(':') { labels.push((key.trim().to_string(), value.trim().to_string())); } continue; } let columns: Vec<&str> = line.split('\t').collect(); if !header_seen { header_seen = true; start = if columns.len() > 3 && columns[3] == "Mb" { 4 } else { 3 }; individuals = columns[start.min(columns.len())..] .iter() .map(|name| name.trim().to_string()) .collect(); continue; } let number = |index: usize| -> Option { columns.get(index).and_then(|value| value.trim().parse().ok()) }; markers.push(Marker { chr: columns.first().copied().unwrap_or("").trim().to_string(), name: columns.get(1).copied().unwrap_or("").trim().to_string(), cm: number(2), mb: number(3).or_else(|| number(2)), }); } if !header_seen { return Err(GenotypeError::NotFound(format!( "{}: no header line", geno.display() ))); } Ok(FileMeta { labels, individuals, markers, }) } /// A genotype file set as one engine needs it, plus a note on what it is. pub struct Prepared { pub genotype_file: PathBuf, pub snps_file: Option, /// Individuals in the prepared file (after any exclusion). pub individuals: usize, /// How many panel members were dropped as parents/F1s. pub excluded: usize, /// Path relative to the genotype-files directory, for `notes`. pub note: String, /// Markers left out for the engine's sake (GEMMA: the Y and /// mitochondrial chromosomes, where its leave-one-chromosome-out LMM /// degenerates -- see `drop_degenerate_chromosomes`). pub markers_dropped: usize, } /// Prepare the genotype input for `engine`, restricted to `keep` (panel /// indices, in panel order). When everything is kept the deployment's own /// file is used untouched; otherwise a column-subset copy is written /// under `cache_dir/prepared`, keyed by source, panel and mtime so a /// repeat request reuses it. pub fn prepare( dir: &Path, resolved: &Resolved, engine: Engine, keep: &[usize], cache_dir: &Path, ) -> Result { let excluded = resolved.panel.len().saturating_sub(keep.len()); let all_kept = keep.len() == resolved.panel.len(); match engine { Engine::Rqtl => { if all_kept { return Ok(Prepared { note: resolved.relative(dir, &resolved.geno), genotype_file: resolved.geno.clone(), snps_file: None, individuals: resolved.panel.len(), excluded, markers_dropped: 0, }); } let path = subset_path(cache_dir, &resolved.version, keep, "geno")?; if !path.is_file() { write_subset_geno(&resolved.geno, keep, &path)?; } Ok(Prepared { note: format!( "prepared panel subset {}", path.file_name().unwrap_or_default().to_string_lossy() ), genotype_file: path, snps_file: None, individuals: keep.len(), excluded, markers_dropped: 0, }) } Engine::Gemma => { let (Some(geno), Some(snps)) = (resolved.bimbam_geno.clone(), resolved.bimbam_snps.clone()) else { return Err(GenotypeError::NotFound(format!( "genotype file '{}' has no bimabm pair under bimbam/, so GEMMA cannot run on it", resolved.version ))); }; // Individuals first (a column subset), then the markers GEMMA's // LOCO cannot fit -- the Y and the mitochondrion. let (panel_file, individuals) = if all_kept { (geno.clone(), resolved.panel.len()) } else { let path = subset_path(cache_dir, &resolved.version, keep, "bimbam_geno.txt")?; if !path.is_file() { write_subset_bimbam(&geno, keep, &path)?; } (path, keep.len()) }; let filtered_geno = subset_path(cache_dir, &resolved.version, keep, "bimbam_geno-noY.txt")?; let filtered_snps = subset_path(cache_dir, &resolved.version, keep, "bimbam_snps-noY.txt")?; if !filtered_geno.is_file() || !filtered_snps.is_file() { let written = drop_degenerate_chromosomes(&panel_file, &snps, &filtered_geno, &filtered_snps)?; if written == 0 { let _ = std::fs::remove_file(&filtered_geno); let _ = std::fs::remove_file(&filtered_snps); } } // The count is the difference in marker counts, which is cheap // and works whether the pair was just written or was cached. let markers_dropped = match ( count_lines(&snps), count_lines(&filtered_snps), ) { (Ok(total), Ok(kept)) => total.saturating_sub(kept), _ => 0, }; let (genotype_file, snps_file, extra) = if markers_dropped > 0 { ( filtered_geno, filtered_snps, format!("; {markers_dropped} Y/M markers left out (GEMMA's LOCO cannot fit them)"), ) } else { (panel_file, snps, String::new()) }; Ok(Prepared { note: format!( "{}{extra} (SNP map: {})", resolved.relative(dir, &genotype_file), resolved.relative(dir, &snps_file) ), genotype_file, snps_file: Some(snps_file), individuals, excluded, markers_dropped, }) } } } /// Chromosomes GEMMA's LOCO run cannot fit: the Y is (near) monomorphic in /// a riset panel and the mitochondrion is maternally inherited, so the /// per-chromosome LMM degenerates. Measured on DOL (200 individuals, /// 112 913 markers): with these two left in, GEMMA's Y run dies in /// `fastblas.cpp ... fast_cblas_dgemm` ("Enforce failed for N>0") and the /// whole request is a 502 with nothing about the cause. pub const DEGENERATE_CHROMOSOMES: [&str; 2] = ["Y", "M"]; /// Copy a bimbam geno/snps pair without the degenerate chromosomes. The two /// files are line-aligned (one marker per line, in the same order), so the /// same keep mask applies to both. Returns how many markers were dropped. fn drop_degenerate_chromosomes( geno: &Path, snps: &Path, out_geno: &Path, out_snps: &Path, ) -> Result { let snps_text = std::fs::read_to_string(snps).map_err(|e| io_err(snps, e))?; let mut keep_row: Vec = Vec::new(); let mut kept_snps = String::with_capacity(snps_text.len()); for line in snps_text.lines() { if line.trim().is_empty() { continue; } // `, , ` (GN writes ", " separators) let chromosome = line .split(',') .nth(2) .map(str::trim) .unwrap_or_default() .to_string(); let keep = !DEGENERATE_CHROMOSOMES.contains(&chromosome.as_str()); keep_row.push(keep); if keep { kept_snps.push_str(line); kept_snps.push('\n'); } } let dropped = keep_row.iter().filter(|keep| !**keep).count(); if dropped == 0 { return Ok(0); } let geno_text = std::fs::read_to_string(geno).map_err(|e| io_err(geno, e))?; let mut kept_geno = String::with_capacity(geno_text.len()); for (index, line) in geno_text.lines().enumerate() { if line.trim().is_empty() { continue; } if keep_row.get(index).copied().unwrap_or(true) { kept_geno.push_str(line.trim_end_matches('\r')); kept_geno.push('\n'); } } std::fs::write(out_geno, kept_geno).map_err(|e| io_err(out_geno, e))?; std::fs::write(out_snps, kept_snps).map_err(|e| io_err(out_snps, e))?; Ok(dropped) } /// Non-empty lines in a file (the marker count of a snps annotation). fn count_lines(path: &Path) -> Result { Ok(std::fs::read_to_string(path) .map_err(|e| io_err(path, e))? .lines() .filter(|line| !line.trim().is_empty()) .count()) } fn subset_path( cache_dir: &Path, version: &str, keep: &[usize], suffix: &str, ) -> Result { let dir = cache_dir.join("prepared"); std::fs::create_dir_all(&dir).map_err(|e| io_err(&dir, e))?; let mut hash = 1469598103934665603u64; for index in keep { hash = (hash ^ *index as u64).wrapping_mul(1099511628211); } Ok(dir.join(format!("{version}-{hash:016x}-{suffix}"))) } fn write_subset_geno(geno: &Path, keep: &[usize], out: &Path) -> Result<(), GenotypeError> { let text = std::fs::read_to_string(geno).map_err(|e| io_err(geno, e))?; let mut result = String::with_capacity(text.len()); let mut seen_header = false; for line in text.lines() { let trimmed = line.trim_end_matches('\r'); if trimmed.trim().is_empty() || trimmed.starts_with('#') || trimmed.starts_with('@') { result.push_str(trimmed); result.push('\n'); continue; } let columns: Vec<&str> = trimmed.split('\t').collect(); let start = if !seen_header && columns.len() > 3 && columns[3] == "Mb" { 4 } else { 3 }; if !seen_header { seen_header = true; result.push_str(&columns[..start].join("\t")); for index in keep { result.push('\t'); result.push_str(columns.get(start + index).copied().unwrap_or("")); } result.push('\n'); continue; } result.push_str(&columns[..start].join("\t")); for index in keep { result.push('\t'); result.push_str(columns.get(start + index).copied().unwrap_or("")); } result.push('\n'); } std::fs::write(out, result).map_err(|e| io_err(out, e)) } fn write_subset_bimbam(geno: &Path, keep: &[usize], out: &Path) -> Result<(), GenotypeError> { let text = std::fs::read_to_string(geno).map_err(|e| io_err(geno, e))?; let mut result = String::with_capacity(text.len()); for line in text.lines() { let trimmed = line.trim_end_matches('\r'); if trimmed.is_empty() { continue; } let fields: Vec<&str> = trimmed.split(',').collect(); if fields.len() < 4 { result.push_str(trimmed); result.push('\n'); continue; } result.push_str( &fields[..3] .iter() .map(|field| field.trim()) .collect::>() .join(","), ); for index in keep { result.push(','); result.push_str(fields.get(3 + index).map(|v| v.trim()).unwrap_or("NA")); } result.push('\n'); } std::fs::write(out, result).map_err(|e| io_err(out, e)) } /// Align named values onto a panel subset: one entry per `keep` index, /// matched **by name**, `None` where the trait has no value. /// /// Returns the aligned vector and how many of the caller's values found a /// place. Values whose name is not in the panel are dropped -- the caller /// may hold a superset (a trait spanning several cohorts of a panel that /// only covers one), and silently mapping the wrong individuals is the /// failure mode this whole module exists to avoid. pub fn align( panel: &[String], keep: &[usize], values: &[(String, Option)], ) -> (Vec>, usize) { let mut by_name = std::collections::HashMap::with_capacity(values.len()); for (name, value) in values { by_name.insert(name.as_str(), *value); } let mut aligned = Vec::with_capacity(keep.len()); let mut matched = 0; for index in keep { let value = panel .get(*index) .and_then(|name| by_name.get(name.as_str())) .copied() .flatten(); if value.is_some() { matched += 1; } aligned.push(value); } (aligned, matched) } #[cfg(test)] mod tests { fn panel(names: &[&str]) -> Vec { names.iter().map(|name| (*name).to_string()).collect() } fn parents(names: &[&str]) -> BTreeSet { names.iter().map(|name| (*name).to_string()).collect() } #[test] fn the_design_names_the_roles_the_way_gn2_does() { let dir = scratch("design"); std::fs::write( dir.join("parents_and_f1s.json"), // GN2 unpacks ParInfo as f1, f12, maternal, paternal. r#"{"BXD": {"f1s": [{"strain": "B6D2F1"}, {"strain": "D2B6F1"}], "maternal": {"strain": "C57BL/6J"}, "paternal": {"strain": "DBA/2J"}}, "EMPTY": {"f1s": [{"strain": ""}, {"strain": ""}], "maternal": {"strain": "A"}, "paternal": {"strain": "B"}}}"#, ) .unwrap(); let design = parental_design(&dir, "BXD", Some(&["BXD1".to_string(), "B6D2F1".to_string()]), Some("riset".into())); assert_eq!(design.source, "metadata"); assert_eq!(design.design.as_deref(), Some("riset")); assert_eq!(design.f1s.len(), 2); assert_eq!(design.maternal.as_ref().unwrap().strain, "C57BL/6J"); assert_eq!(design.paternal.as_ref().unwrap().strain, "DBA/2J"); assert_eq!(design.excluded_from_panel, vec!["B6D2F1"]); assert_eq!(design.excluded_by_name, vec!["B6D2F1"]); // An empty F1 name is not an exclusion entry. let design = parental_design(&dir, "EMPTY", None, None); assert!(design.f1s.is_empty(), "empty names are dropped: {:?}", design.f1s); // An unlisted group says so. let design = parental_design(&dir, "NOT-LISTED", None, None); assert_eq!(design.source, "name heuristic"); assert!(design.maternal.is_none() && design.paternal.is_none()); } #[test] fn the_parent_f1_rule_still_drops_and_counts() { let p = panel(&["BXD1", "B6D2F1", "C57BL/6J"]); let selection = select_panel(&p, &parents(&["B6D2F1", "C57BL/6J"]), false, &Individuals::default()) .unwrap(); assert_eq!(selection.keep, vec![0]); assert_eq!(selection.dropped_parents_f1, 2); assert_eq!(selection.included_by_name, 0); } #[test] fn a_named_individual_is_kept_against_the_rule() { let p = panel(&["BXD1", "B6D2F1", "C57BL/6J"]); let individuals = Individuals { include: vec!["B6D2F1".to_string()], exclude: vec![] }; let selection = select_panel(&p, &parents(&["B6D2F1", "C57BL/6J"]), false, &individuals).unwrap(); assert_eq!(selection.keep, vec![0, 1], "the F1 stays, the parent does not"); assert_eq!(selection.dropped_parents_f1, 1); assert_eq!(selection.included_by_name, 1); } #[test] fn exclude_wins_over_include() { let p = panel(&["BXD1", "BXD48", "BXD48a"]); let individuals = Individuals { include: vec!["BXD48a".to_string()], exclude: vec!["BXD48a".to_string()], }; let selection = select_panel(&p, &parents(&[]), false, &individuals).unwrap(); assert_eq!(selection.keep, vec![0, 1]); assert_eq!(selection.excluded_by_name, 1); } #[test] fn an_unknown_name_is_an_error_that_names_the_panel() { let p = panel(&["BXD1", "BXD2"]); let individuals = Individuals { include: vec![], exclude: vec!["BXD9x".to_string()] }; let error = select_panel(&p, &parents(&[]), false, &individuals).unwrap_err(); let message = error.to_string(); assert!(message.contains("BXD9x"), "{message}"); assert!(message.contains("BXD1, BXD2"), "{message}"); } #[test] fn selecting_everything_away_is_refused() { let p = panel(&["BXD1", "BXD2"]); let individuals = Individuals { include: vec![], exclude: vec!["BXD1".to_string(), "BXD2".to_string()], }; assert!(select_panel(&p, &parents(&[]), false, &individuals).is_err()); } use super::*; use std::fs; fn scratch(name: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!("gn4-genotypes-{name}-{}", std::process::id())); let _ = fs::remove_dir_all(&dir); fs::create_dir_all(dir.join("genotype/bimbam")).unwrap(); dir } fn write_geno(dir: &Path, stem: &str, samples: &[&str], markers: usize) { let mut text = String::from("# comment\n@name:BXD\n@type:riset\n"); text.push_str(&format!( "Chr\tLocus\tcM\tMb\t{}\n", samples.join("\t") )); for marker in 0..markers { text.push_str(&format!("1\trs{marker}\t0\t1.0")); for _ in samples { text.push_str("\tB"); } text.push('\n'); } fs::write(dir.join(format!("genotype/{stem}.geno")), text).unwrap(); } #[test] fn the_panel_is_the_header_sample_names() { let dir = scratch("panel"); write_geno(&dir, "T.1", &["A", "B", "C"], 2); assert_eq!( panel_of(&dir.join("genotype/T.1.geno")).unwrap(), vec!["A", "B", "C"] ); assert_eq!(marker_count(&dir.join("genotype/T.1.geno")).unwrap(), 2); } #[test] fn file_meta_reads_labels_individuals_and_marker_positions() { let dir = scratch("meta"); fs::write( dir.join("genotype/R.1.geno"), "# a comment\n@name:R\n@type:riset\n@mat:B\n@pat:D\n\ Chr\tLocus\tcM\tMb\tS1\tS2\n\ 1\trs1\t1.5\t3.0\tB\tD\n\ 2\trs2\t2.5\t5.0\tD\tB\n", ) .unwrap(); let meta = file_meta(&dir.join("genotype/R.1.geno")).unwrap(); assert_eq!( meta.labels, vec![ ("name".to_string(), "R".to_string()), ("type".to_string(), "riset".to_string()), ("mat".to_string(), "B".to_string()), ("pat".to_string(), "D".to_string()), ] ); assert_eq!(meta.individuals, vec!["S1", "S2"]); assert_eq!(meta.markers.len(), 2); assert_eq!(meta.markers[1].name, "rs2"); assert_eq!(meta.markers[1].chr, "2"); assert_eq!(meta.markers[1].cm, Some(2.5)); assert_eq!(meta.markers[1].mb, Some(5.0)); } #[test] fn the_json_order_decides_which_version_is_current() { let dir = scratch("json"); write_geno(&dir, "G.9", &["S1"], 1); write_geno(&dir, "G.2", &["S1"], 1); fs::write( dir.join("genotype/G.json"), r#"{"genofile": [{"title": "New", "location": "G.2.geno"}, {"title": "Old", "location": "G.9.geno"}]}"#, ) .unwrap(); let files = list(&dir, "G").unwrap(); assert_eq!(files[0].version, "G.2"); assert!(files[0].current); assert_eq!(files[0].title.as_deref(), Some("New")); assert_eq!(files[1].version, "G.9"); assert!(!files[1].current); assert_eq!(files[0].order_source, OrderSource::Json); // ... even though .9 is numerically higher: the JSON wins. assert_eq!(resolve(&dir, "G", None).unwrap().version, "G.2"); assert_eq!(resolve(&dir, "G", Some("G.9")).unwrap().version, "G.9"); assert_eq!(resolve(&dir, "G", Some("9")).unwrap().version, "G.9"); } #[test] fn with_no_json_the_highest_version_number_is_current() { let dir = scratch("scan"); write_geno(&dir, "H.2", &["S1"], 1); write_geno(&dir, "H.11", &["S1"], 1); write_geno(&dir, "H.3", &["S1"], 1); let files = list(&dir, "H").unwrap(); assert_eq!( files.iter().map(|f| f.version.as_str()).collect::>(), vec!["H.11", "H.3", "H.2"] ); assert_eq!(files[0].order_source, OrderSource::FileNames); assert_eq!(resolve(&dir, "H", None).unwrap().version, "H.11"); } #[test] fn an_unknown_version_says_what_is_available() { let dir = scratch("unknown"); write_geno(&dir, "K.1", &["S1"], 1); let error = resolve(&dir, "K", Some("K.7")).unwrap_err(); let message = error.to_string(); assert!(message.contains("K.7"), "{message}"); assert!(message.contains("K.1"), "{message}"); } #[test] fn gemma_needs_the_bimbam_pair_and_rqtl_the_geno() { let dir = scratch("engines"); write_geno(&dir, "M.1", &["S1", "S2"], 3); let files = list(&dir, "M").unwrap(); assert!(files[0].rqtl); assert!(!files[0].gemma, "no bimbam pair yet"); fs::write(dir.join("genotype/bimbam/M.1_geno.txt"), "rs1,A,B,0,1\n").unwrap(); fs::write(dir.join("genotype/bimbam/M.1_snps.txt"), "rs1,1,1\n").unwrap(); assert!(list(&dir, "M").unwrap()[0].gemma); } #[test] fn parents_and_f1s_are_excluded_from_the_panel_by_default() { let dir = scratch("panel2"); write_geno(&dir, "N.1", &["S1", "S2", "C57BL/6J", "S1xS2F1"], 2); fs::write( dir.join("parents_and_f1s.json"), r#"{"N": {"paternal": {"strain": "C57BL/6J"}, "maternal": {"strain": "DBA/2J"}, "f1s": [{"strain": "S1xS2F1"}]}}"#, ) .unwrap(); let resolved = resolve(&dir, "N", None).unwrap(); let excluded = non_strains(&dir, "N", &resolved.panel); assert!(excluded.contains("C57BL/6J")); assert!(excluded.contains("S1xS2F1")); let keep: Vec = resolved .panel .iter() .enumerate() .filter(|(_, name)| !excluded.contains(*name)) .map(|(index, _)| index) .collect(); assert_eq!(keep, vec![0, 1]); // Subsetting keeps the header and drops the columns. let cache = dir.join("cache"); let prepared = prepare(&dir, &resolved, Engine::Rqtl, &keep, &cache).unwrap(); assert_eq!(prepared.individuals, 2); assert_eq!(prepared.excluded, 2); let subset = fs::read_to_string(&prepared.genotype_file).unwrap(); let header = subset .lines() .find(|line| line.starts_with("Chr")) .unwrap(); assert_eq!(header, "Chr\tLocus\tcM\tMb\tS1\tS2"); assert!(subset.contains("@type:riset"), "metadata lines survive"); // Cached: a second call reuses the file. let again = prepare(&dir, &resolved, Engine::Rqtl, &keep, &cache).unwrap(); assert_eq!(again.genotype_file, prepared.genotype_file); } #[test] fn keeping_everyone_uses_the_deployment_file_untouched() { let dir = scratch("all"); write_geno(&dir, "P.1", &["S1", "S2"], 2); let resolved = resolve(&dir, "P", None).unwrap(); let keep: Vec = (0..resolved.panel.len()).collect(); let prepared = prepare(&dir, &resolved, Engine::Rqtl, &keep, &dir.join("cache")).unwrap(); assert_eq!(prepared.genotype_file, resolved.geno); assert_eq!(prepared.excluded, 0); assert_eq!(prepared.note, "genotype/P.1.geno"); } #[test] fn values_are_aligned_by_name_onto_the_subset_and_unknown_names_dropped() { let panel: Vec = ["S1", "S2", "F1", "S3"].iter().map(|s| s.to_string()).collect(); let keep = vec![0usize, 1, 3]; let values = vec![ ("S3".to_string(), Some(3.0)), ("S1".to_string(), Some(1.0)), ("NOT-IN-PANEL".to_string(), Some(9.0)), ("F1".to_string(), Some(2.0)), ]; let (aligned, matched) = align(&panel, &keep, &values); assert_eq!(aligned, vec![Some(1.0), None, Some(3.0)]); assert_eq!(matched, 2, "the F1 value and the stranger are not used"); } #[test] fn a_bimbam_subset_keeps_the_first_three_fields_and_the_kept_columns() { let dir = scratch("bimbam-subset"); write_geno(&dir, "Q.1", &["S1", "S2", "S3"], 2); fs::write( dir.join("genotype/bimbam/Q.1_geno.txt"), "rs1, A, B, 0, 1, 0.5\nrs2, A, B, 1, 0, NA\n", ) .unwrap(); fs::write(dir.join("genotype/bimbam/Q.1_snps.txt"), "rs1,1,1\n").unwrap(); let resolved = resolve(&dir, "Q", None).unwrap(); let prepared = prepare(&dir, &resolved, Engine::Gemma, &[0, 2], &dir.join("cache")).unwrap(); let text = fs::read_to_string(&prepared.genotype_file).unwrap(); assert_eq!(text, "rs1,A,B,0,0.5\nrs2,A,B,1,NA\n"); assert!(prepared.snps_file.unwrap().ends_with("Q.1_snps.txt")); assert_eq!(prepared.individuals, 2); assert_eq!(prepared.excluded, 1); } } /// Which individuals in a BIMBAM genotype file have any genotype at all. /// /// GEMMA imputes a *missing* genotype with the SNP mean, but an individual /// with no genotypes anywhere contributes an empty row to the kinship /// matrix, and the resulting K is singular: the run dies inside LAPACK /// ("eigenvalues close to zero" then `LUDecomp ... input domain error`), a /// message that says nothing about the cause. Measured on this deployment: /// `BXD-AE.4` has 136 of 344 individuals like that, `MDP` 93 of 178, /// `BXD-Heart-Metals.4` 200 of 481 -- and dropping them makes those panels /// map in seconds. /// /// Returns one flag per individual, in the file's column order; `None` when /// the file cannot be read (the caller then proceeds as before rather than /// refusing the run). pub fn individuals_with_genotypes(geno: &std::path::Path) -> Option> { let text = std::fs::read_to_string(geno).ok()?; let mut mask: Option> = None; for line in text.lines() { let line = line.trim(); if line.is_empty() || line.starts_with('#') || line.starts_with('@') { continue; } let fields: Vec<&str> = line .split(|c: char| c == ',' || c.is_whitespace()) .filter(|field| !field.is_empty()) .collect(); if fields.len() < 4 { continue; } let genotypes = &fields[3..]; let mask = mask.get_or_insert_with(|| vec![false; genotypes.len()]); if genotypes.len() != mask.len() { continue; } for (index, value) in genotypes.iter().enumerate() { if !matches!(*value, "NA" | "na" | "." | "") { mask[index] = true; } } } mask } /// One value per individual of a `.geno`, in the file's own order, from values /// given by name. Used when a request names a file rather than a group -- an /// uploaded population, typically -- because the caller has a trait's values /// for its animals, not the file's column order. pub fn align_to_file( geno: &Path, values: &[(String, Option)], ) -> Result>, GenotypeError> { let individuals = panel_of(geno)?; let by_name: std::collections::HashMap<&str, Option> = values .iter() .map(|(name, value)| (name.as_str(), *value)) .collect(); Ok(individuals .iter() .map(|name| by_name.get(name.as_str()).copied().flatten()) .collect()) }