//! Genotype frequencies, counted from a `.geno` and reported as a map. //! //! A `.geno` cell is a genotype *call* (`B`, `D`, `H`, `A`–`H` for the DO's //! founder alleles), or -- in a file that has been through a dosage or //! probability pipeline -- a number. Calls are counted as they are; a column //! of probabilities is binned, because 0.4187 and 0.4191 are the same //! statement about a marker and listing them separately hides the shape of //! the distribution. The default bin is 0.05 wide (twenty bins over [0, 1]), //! and a file with only a handful of distinct numbers (`0`, `0.5`, `1` for a //! biallelic dosage) is reported exactly: binning three values would lose the //! very thing the caller is looking at. //! //! The genotypes themselves are counted and discarded, never returned -- the //! same rule `/groups/{group}/genotypes` follows. What a caller gets is the //! distribution, which is also what makes a panel's quality legible: a marker //! that is 99 % missing, or calls that are not the codes its header declares, //! shows up here and nowhere else in the API. //! //! Two costs are bounded on purpose. Cells are counted through an interner, so //! a 100-million-cell panel costs one hash lookup per cell and one allocation //! per *distinct* call, not per cell. And a file larger than [`MAX_SCAN_BYTES`] //! is refused rather than read: `HSNIH-Palmer_true.geno` is 3.7 GB (134 927 //! markers × 6 147 individuals), which is not a diagnosis anyone waits for. use std::collections::{BTreeMap, HashMap}; use std::io::{BufRead, BufReader}; use std::path::Path; use crate::genotypes::MAX_SCAN_BYTES; /// Cells that are not calls. `U` is GeneNetwork's "unknown", the rest are /// what the files in the wild use for the same thing. const MISSING: [&str; 9] = ["", "U", "N", "NA", "x", "X", "?", "-", "."]; /// How many distinct numeric values a column may have before it is treated /// as probabilities and binned rather than listed exactly. const EXACT_LIMIT: usize = 20; /// Why a tally could not be produced. #[derive(Debug)] pub enum FreqError { /// No such file, no such marker in it, or no calls at all. NotFound(String), /// The file is bigger than [`MAX_SCAN_BYTES`]. TooLarge(String), /// The file could not be read. Io(String), } impl std::fmt::Display for FreqError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { FreqError::NotFound(what) | FreqError::TooLarge(what) | FreqError::Io(what) => { write!(f, "{what}") } } } } /// The distribution of calls over the cells that carry one. #[derive(Debug, Clone, PartialEq)] pub struct Tally { /// Call (or bin label) -> frequency, summing to 1 over `called`. pub frequencies: BTreeMap, /// Cells that carry a call. pub called: u64, /// Cells whose call is missing. pub missing: u64, /// Marker rows read (one, when a single marker was asked for). pub markers: usize, /// Bin width, when the calls were numeric probabilities and got binned. pub bin_width: Option, } /// Where the counted marker sits, when one was named. #[derive(Debug, Clone, PartialEq)] pub struct Counted { pub tally: Tally, pub chr: Option, pub cm: Option, pub mb: Option, } fn is_missing(cell: &str) -> bool { MISSING.contains(&cell) } /// Counts cells through an interner: one hash lookup per cell, one allocation /// per distinct call. A probability-valued file switches to binning the moment /// its distinct values pass [`EXACT_LIMIT`], so a dosages file cannot fill /// memory with distinct floats. struct Counter { width: f64, calls: Vec, counts: Vec, index: HashMap, bins: BTreeMap, binned: bool, } impl Counter { fn new(width: f64) -> Self { Self { width, calls: Vec::new(), counts: Vec::new(), index: HashMap::new(), bins: BTreeMap::new(), binned: false, } } fn add(&mut self, cell: &str) { if self.binned { match self.bin_of(cell) { Some(index) => *self.bins.entry(index).or_insert(0) += 1, // A mixed file: a code among probabilities keeps its own key. None => self.add_exact(cell), } return; } self.add_exact(cell); if self.calls.len() > EXACT_LIMIT && self .calls .iter() .all(|call| call.parse::().map(|v| (0.0..=1.0).contains(&v)).unwrap_or(false)) { self.binned = true; let calls = std::mem::take(&mut self.calls); let counts = std::mem::take(&mut self.counts); self.index.clear(); for (call, count) in calls.into_iter().zip(counts) { if let Some(index) = self.bin_of(&call) { *self.bins.entry(index).or_insert(0) += count; } } } } fn add_exact(&mut self, cell: &str) { match self.index.get(cell) { Some(slot) => self.counts[*slot] += 1, None => { self.index.insert(cell.to_string(), self.calls.len()); self.calls.push(cell.to_string()); self.counts.push(1); } } } /// The bin a value falls in. fn bin_of(&self, call: &str) -> Option { let value = call .parse::() .ok() .filter(|value| (0.0..=1.0).contains(value))?; let bins = ((1.0 / self.width).round() as usize).max(1); Some(((value / self.width).floor() as usize).min(bins - 1)) } fn total(&self) -> u64 { self.counts.iter().sum::() + self.bins.values().sum::() } fn finish(self, called: u64) -> (BTreeMap, Option) { let mut frequencies: BTreeMap = BTreeMap::new(); if self.binned { for (index, count) in &self.bins { let low = *index as f64 * self.width; let high = (low + self.width).min(1.0); frequencies.insert(format!("{low:.2}-{high:.2}"), proportion(*count, called)); } for (call, count) in self.calls.iter().zip(&self.counts) { frequencies.insert(call.clone(), proportion(*count, called)); } return (frequencies, Some(self.width)); } let mut ordered: Vec<(&String, &u64)> = self.calls.iter().zip(&self.counts).collect(); // Numeric calls sort by value; anything else alphabetically. ordered.sort_by(|left, right| match (left.0.parse::(), right.0.parse::()) { (Ok(a), Ok(b)) => a.partial_cmp(&b).unwrap_or(std::cmp::Ordering::Equal), _ => left.0.cmp(right.0), }); for (call, count) in ordered { frequencies.insert(call.clone(), proportion(*count, called)); } (frequencies, None) } } /// A frequency, rounded so that a client's sum does not wobble in the last /// digit. fn proportion(count: u64, total: u64) -> f64 { ((count as f64 / total as f64) * 1e6).round() / 1e6 } /// Count the calls in `geno`, over the whole panel or one marker. pub fn tally(geno: &Path, only: Option<&str>, bin_width: Option) -> Result { let metadata = std::fs::metadata(geno) .map_err(|error| FreqError::NotFound(format!("{}: {error}", geno.display())))?; if only.is_none() && metadata.len() > MAX_SCAN_BYTES { return Err(FreqError::TooLarge(format!( "{} is {} MB, past the {} MB a whole-panel tally reads; ask for one marker instead, \ or pick a smaller version of the panel", geno.display(), metadata.len() / (1024 * 1024), MAX_SCAN_BYTES / (1024 * 1024) ))); } let file = std::fs::File::open(geno) .map_err(|error| FreqError::Io(format!("{}: {error}", geno.display())))?; let width = bin_width .filter(|width| *width > 0.0 && *width <= 1.0) .unwrap_or(0.05); let mut counter = Counter::new(width); let mut missing = 0u64; let mut markers = 0usize; let mut seen_header = false; let mut start = 3usize; let mut position: Option<(String, Option, Option)> = None; let mut found = only.is_none(); for line in BufReader::with_capacity(1 << 20, file).lines() { let line = line.map_err(|error| FreqError::Io(format!("{}: {error}", geno.display())))?; let line = line.trim_end_matches('\r'); if line.trim().is_empty() || line.starts_with('#') || line.starts_with('@') { continue; } let columns: Vec<&str> = line.split('\t').collect(); if !seen_header { seen_header = true; start = if columns.len() > 3 && columns[3] == "Mb" { 4 } else { 3 }; continue; } let name = columns.get(1).copied().unwrap_or("").trim(); if let Some(wanted) = only { if name != wanted { continue; } found = true; let number = |index: usize| -> Option { columns .get(index) .and_then(|value| value.trim().parse().ok()) }; position = Some(( columns.first().copied().unwrap_or("").trim().to_string(), number(2), number(3).or_else(|| number(2)), )); } markers += 1; for cell in columns.iter().skip(start) { let cell = cell.trim(); if is_missing(cell) { missing += 1; } else { counter.add(cell); } } if only.is_some() { // One row was the whole request. break; } } if !seen_header { return Err(FreqError::NotFound(format!( "{}: no header line", geno.display() ))); } if !found { return Err(FreqError::NotFound(format!( "{}: no marker '{}'", geno.display(), only.unwrap_or_default() ))); } let called = counter.total(); if called == 0 { return Err(FreqError::NotFound(format!( "{}: no genotype calls{}", geno.display(), only.map(|name| format!(" for '{name}'")).unwrap_or_default() ))); } let (frequencies, bin_width) = counter.finish(called); let (chr, cm, mb) = match position { Some((chr, cm, mb)) => (Some(chr), cm, mb), None => (None, None, None), }; Ok(Counted { tally: Tally { frequencies, called, missing, markers, bin_width, }, chr, cm, mb, }) } #[cfg(test)] mod tests { use super::*; fn counter_of(calls: &[&str]) -> Counter { let mut counter = Counter::new(0.05); for call in calls { counter.add(call); } counter } #[test] fn calls_are_reported_exactly_and_sum_to_one() { let mut counter = Counter::new(0.05); for (call, times) in [("B", 60), ("D", 38), ("H", 2)] { for _ in 0..times { counter.add(call); } } let called = counter.total(); assert_eq!(called, 100); let (frequencies, width) = counter.finish(called); assert_eq!(width, None, "three distinct calls are not binned"); assert_eq!(frequencies["B"], 0.6); assert_eq!(frequencies["D"], 0.38); assert_eq!(frequencies["H"], 0.02); let sum: f64 = frequencies.values().sum(); assert!((sum - 1.0).abs() < 1e-6, "{sum}"); } #[test] fn numbers_are_ordered_by_value_not_by_string() { let counter = counter_of(&[ "0.5", "0.5", "0.5", "0.5", "0.5", "0.5", "0.5", "0.5", "0.5", "0.5", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "1", "1", "1", "1", "1", "1", "1", "1", "1", "1", ]); let called = counter.total(); let (frequencies, width) = counter.finish(called); assert_eq!(width, None, "a dosage with three values stays exact"); let order: Vec<&String> = frequencies.keys().collect(); assert_eq!(order, vec!["0", "0.5", "1"]); } #[test] fn probabilities_are_binned_at_a_twentieth() { let values: Vec = (0..25) .map(|step| format!("{:.4}", step as f64 / 25.0)) .collect(); let mut counter = Counter::new(0.05); for value in &values { for _ in 0..4 { counter.add(value); } } counter.add("1"); let called = counter.total(); let (frequencies, width) = counter.finish(called); assert_eq!(width, Some(0.05)); assert_eq!(frequencies.len(), 20, "twenty bins over [0, 1]"); assert!(frequencies.contains_key("0.00-0.05")); assert!(frequencies.contains_key("0.95-1.00")); let sum: f64 = frequencies.values().sum(); // Each frequency is rounded to 6 decimal places, so twenty of them can // drift by a few parts per million. assert!((sum - 1.0).abs() < 1e-4, "{sum}"); } #[test] fn a_requested_bin_width_is_honoured() { let values: Vec = (0..40).map(|step| format!("{}", step as f64 / 40.0)).collect(); let mut counter = Counter::new(0.1); for value in &values { counter.add(value); } let called = counter.total(); let (frequencies, width) = counter.finish(called); assert_eq!(width, Some(0.1)); assert_eq!(frequencies.len(), 10, "ten bins, as asked"); } #[test] fn a_code_among_probabilities_keeps_its_own_key() { let values: Vec = (0..5000) .map(|step| format!("{:.4}", step as f64 / 5000.0)) .collect(); let mut counter = Counter::new(0.05); for value in &values { counter.add(value); } assert!(counter.binned, "5000 distinct probabilities are binned"); assert!(counter.calls.is_empty(), "and dropped from the exact side"); counter.add("A"); let called = counter.total(); let (frequencies, width) = counter.finish(called); assert_eq!(width, Some(0.05)); assert!(frequencies.contains_key("0.00-0.05")); assert!((frequencies["A"] - 1.0 / 5001.0).abs() < 1e-6, "{}", frequencies["A"]); let sum: f64 = frequencies.values().sum(); assert!((sum - 1.0).abs() < 1e-4, "{sum}"); } #[test] fn a_geno_panel_reports_its_calls() { let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/test_data/rqtl_sample1.geno"); let counted = tally(&path, None, None).unwrap(); assert!(counted.tally.called > 0); assert_eq!(counted.tally.markers, 6, "the fixture's six markers"); let sum: f64 = counted.tally.frequencies.values().sum(); assert!((sum - 1.0).abs() < 1e-6, "{sum}"); assert!( counted.tally.frequencies.keys().all(|call| call.len() <= 2), "calls, not positions: {:?}", counted.tally.frequencies.keys() ); } #[test] fn one_marker_reads_only_its_own_row() { let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/test_data/rqtl_sample1.geno"); let counted = tally(&path, Some("rs6269442"), None).unwrap(); assert_eq!(counted.tally.markers, 1); assert_eq!(counted.chr.as_deref(), Some("1")); assert!(counted.tally.called > 0); assert!( tally(&path, Some("nonesuch"), None).is_err(), "an absent marker is an error, not an empty tally" ); } } /// One marker's genotype calls, in the file's individual order: `(name, /// Some(call))` where a call is present, `(name, None)` where it is missing. /// /// This is the per-animal row the tally deliberately does not return -- it /// stays inside the server, joined to a trait's values and reduced to a /// per-class summary (see `GET .../markers/{marker}/classes`). It is never /// serialised: the endpoint above it answers with statistics, not calls. pub struct MarkerCalls { pub chr: Option, pub cm: Option, pub mb: Option, pub calls: Vec<(String, Option)>, } /// Read one marker's calls. The header defines the individuals; a row may be /// shorter than the header (a truncated file), and those individuals are /// missing rather than absent. pub fn marker_calls(geno: &Path, marker: &str) -> Result { let metadata = std::fs::metadata(geno) .map_err(|error| FreqError::NotFound(format!("{}: {error}", geno.display())))?; if metadata.len() > MAX_SCAN_BYTES { return Err(FreqError::TooLarge(format!( "{} is {} MB, past the {} MB a per-marker read scans; pick a smaller version", geno.display(), metadata.len() / (1024 * 1024), MAX_SCAN_BYTES / (1024 * 1024) ))); } let file = std::fs::File::open(geno) .map_err(|error| FreqError::Io(format!("{}: {error}", geno.display())))?; let mut individuals: Vec = Vec::new(); let mut start = 3usize; let mut found: Option = None; for line in BufReader::with_capacity(1 << 20, file).lines() { let line = line.map_err(|error| FreqError::Io(format!("{}: {error}", geno.display())))?; let line = line.trim_end_matches('\r'); if line.trim().is_empty() || line.starts_with('#') || line.starts_with('@') { continue; } let columns: Vec<&str> = line.split('\t').collect(); if individuals.is_empty() { start = if columns.len() > 3 && columns[3] == "Mb" { 4 } else { 3 }; individuals = columns[start.min(columns.len())..] .iter() .map(|name| name.trim().to_string()) .collect(); if individuals.is_empty() { return Err(FreqError::NotFound(format!( "{}: the header names no individuals", geno.display() ))); } continue; } if columns.get(1).map(|name| name.trim()) != Some(marker) { continue; } let number = |index: usize| -> Option { columns .get(index) .and_then(|value| value.trim().parse().ok()) }; let calls = individuals .iter() .enumerate() .map(|(index, name)| { let call = columns.get(start + index).map(|cell| cell.trim()); let call = call.filter(|cell| !is_missing(cell)); (name.clone(), call.map(str::to_string)) }) .collect(); found = Some(MarkerCalls { chr: columns.first().map(|chr| chr.trim().to_string()), cm: number(2), mb: number(3).or_else(|| number(2)), calls, }); break; } found.ok_or_else(|| { FreqError::NotFound(format!( "{}: no marker '{marker}'", geno.display() )) }) }