//! Uploading a genotype matrix: updating an existing population. //! //! The format is the one a genotype matrix already has, minus GN's own //! metadata lines: //! //! ```text //! marker chr pos BXD1 BXD2 ... //! rs1234567 1 3.49 B D //! rs7654321 1 12.1 D H //! ``` //! //! Tab-delimited; the header names the individuals from column 4 on; column 1 //! is the marker, 2 the chromosome, 3 the position in Mb. Values must be the //! same *kind* as the base population's -- letters (`B`, `D`, `H`), whole //! numbers (`0`, `1`, `2`) or probabilities (`0.02`, `0.98`) -- because a //! mapping engine will read the result the way it reads the population it //! updates. //! //! What is enforced, and why: //! //! * The base version must exist: an upload *updates* a population, it does not //! invent one. //! * Individuals: at most [`MAX_NEW_INDIVIDUALS`] names not in the base, and at //! least one name that is. No overlap means it is a different population. //! * Markers: at most [`MAX_MARKERS`]. Beyond that the sysadmin has to organize //! the file -- this is a request path, not an ingest. //! * Values: the same kind as the base's. A letters panel with `0/1` in it, or a //! probabilities panel with `A/B`, would be read as the wrong thing silently. //! //! The upload is stored as text in the owner's SQLite file (one row, no schema //! explosion at 200 000 markers) and materialised as a GN `.geno` file on //! demand, so the R tooling reads exactly what the user uploaded. use std::collections::BTreeSet; use std::path::Path; use crate::baskets::BasketError; /// The most markers an upload may carry (the user's limit). pub const MAX_MARKERS: usize = 200_000; /// The most individuals an upload may add to a base population. pub const MAX_NEW_INDIVIDUALS: usize = 50; /// What kind of values a matrix holds. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ValueKind { /// Genotype calls: single letters, e.g. `A`, `B`, `H`. Letters, /// Whole-number dosage: `0`, `1`, `2`. Integers, /// Probabilities or dosages in [0, 1] with decimals. Probabilities, } impl ValueKind { pub fn as_str(self) -> &'static str { match self { ValueKind::Letters => "letters", ValueKind::Integers => "integers", ValueKind::Probabilities => "probabilities", } } } /// A parsed upload: the individuals, the markers, and what kind of values it /// holds. #[derive(Debug)] pub struct Parsed { pub individuals: Vec, pub markers: Vec<(String, String, f64)>, pub kind: ValueKind, /// Cells that are missing, counted once for the report. pub missing: u64, } fn is_missing(cell: &str) -> bool { matches!(cell, "" | "U" | "u" | "N" | "NA" | "na" | "-" | "." | "?") } /// What kind of value a non-missing cell is, or None if it is none of them. fn kind_of(cell: &str) -> Option { if cell.chars().all(|character| character.is_ascii_alphabetic()) { return Some(ValueKind::Letters); } if let Ok(number) = cell.parse::() { if !number.is_finite() { return None; } if (0.0..=1.0).contains(&number) && cell.contains('.') { return Some(ValueKind::Probabilities); } if number.fract() == 0.0 && number.abs() <= 2.0 { return Some(ValueKind::Integers); } return Some(ValueKind::Probabilities); } None } /// Parse and validate the table itself (not yet the base population). pub fn parse(text: &str) -> Result { let mut lines = text .lines() .map(|line| line.trim_end_matches('\r')) .filter(|line| !line.trim().is_empty() && !line.starts_with('#')); let Some(header) = lines.next() else { return Err(BasketError::Invalid("the upload is empty".to_string())); }; if header.contains(',') && !header.contains('\t') { return Err(BasketError::Invalid( "the upload is tab-delimited: the header's fields must be separated by tabs".to_string(), )); } let header: Vec<&str> = header.split('\t').collect(); if header.len() < 4 { return Err(BasketError::Invalid(format!( "the header has {} field(s); it needs a marker column, chr, pos and at least one \ individual", header.len() ))); } let individuals: Vec = header[3..] .iter() .map(|name| name.trim().to_string()) .collect(); let mut seen = BTreeSet::new(); for name in &individuals { if name.is_empty() { return Err(BasketError::Invalid( "the header must name every individual from column 4 on".to_string(), )); } if !seen.insert(name.clone()) { return Err(BasketError::Invalid(format!( "individual '{name}' appears twice in the header" ))); } } let mut markers = Vec::new(); let mut kind: Option = None; let mut missing = 0u64; let mut mismatched: Vec = Vec::new(); for (index, line) in lines.enumerate() { let fields: Vec<&str> = line.split('\t').collect(); if fields.len() != header.len() { return Err(BasketError::Invalid(format!( "row {} has {} field(s) but the header has {}: {}", index + 2, fields.len(), header.len(), line.chars().take(60).collect::() ))); } let marker = fields[0].trim(); if marker.is_empty() { return Err(BasketError::Invalid(format!( "row {} has no marker name", index + 2 ))); } let chr = fields[1].trim().to_string(); let pos = fields[2] .trim() .parse::() .map_err(|_| { BasketError::Invalid(format!( "{marker}: '{}' is not a position (column 3 is in Mb)", fields[2] )) })?; markers.push((marker.to_string(), chr, pos)); if markers.len() > MAX_MARKERS { return Err(BasketError::Invalid(format!( "more than {MAX_MARKERS} markers: that is an ingest, not an upload -- the \ sysadmin has to organize a file that size" ))); } for cell in &fields[3..] { let cell = cell.trim(); if is_missing(cell) { missing += 1; continue; } match kind_of(cell) { Some(found) => match kind { None => kind = Some(found), Some(expected) if expected == found => {} Some(expected) => { if mismatched.len() < 5 { mismatched.push(format!( "{marker}: '{cell}' is {}, the file starts as {}", found.as_str(), expected.as_str() )); } } _ => {} }, None => { if mismatched.len() < 5 { mismatched.push(format!("{marker}: '{cell}' is not a genotype value")); } } } } } if markers.is_empty() { return Err(BasketError::Invalid( "the upload has no marker rows".to_string(), )); } if !mismatched.is_empty() { return Err(BasketError::Invalid(format!( "mixed or unusable values: {}", mismatched.join("; ") ))); } Ok(Parsed { individuals, markers, kind: kind.unwrap_or(ValueKind::Letters), missing, }) } /// A base population's value kind, and its individuals. pub fn base_shape(geno: &Path) -> Result<(ValueKind, Vec), BasketError> { let text = std::fs::read_to_string(geno) .map_err(|error| BasketError::Store(format!("{}: {error}", geno.display())))?; let mut individuals = Vec::new(); let mut kind: Option = None; let mut start = 3usize; let mut header_seen = false; for line in text.lines() { if line.starts_with('#') || line.starts_with('@') || line.trim().is_empty() { continue; } let fields: Vec<&str> = line.split('\t').collect(); if !header_seen { header_seen = true; start = if fields.len() > 3 && fields[3] == "Mb" { 4 } else { 3 }; individuals = fields[start.min(fields.len())..] .iter() .map(|name| name.trim().to_string()) .collect(); continue; } if kind.is_some() { continue; } for cell in fields.iter().skip(start) { let cell = cell.trim(); if is_missing(cell) { continue; } kind = kind_of(cell); if kind.is_some() { break; } } } Ok((kind.unwrap_or(ValueKind::Letters), individuals)) } /// Check an upload against the population it updates. pub fn check_base( parsed: &Parsed, base_individuals: &[String], base_kind: ValueKind, ) -> Result<(usize, usize), BasketError> { let base: BTreeSet<&str> = base_individuals.iter().map(String::as_str).collect(); let overlap = parsed .individuals .iter() .filter(|name| base.contains(name.as_str())) .count(); let new = parsed.individuals.len() - overlap; if overlap == 0 { return Err(BasketError::Invalid(format!( "none of the {} individual(s) are in the base population: an upload updates a \ population, so at least one name has to match", parsed.individuals.len() ))); } if new > MAX_NEW_INDIVIDUALS { return Err(BasketError::Invalid(format!( "{new} individual(s) are not in the base population; at most \ {MAX_NEW_INDIVIDUALS} new ones can be added by an upload" ))); } if parsed.kind != base_kind { return Err(BasketError::Invalid(format!( "the values are {} but this population's are {}; a mapping engine would read them \ as the wrong thing", parsed.kind.as_str(), base_kind.as_str() ))); } Ok((overlap, new)) } /// Convert the upload to a GN `.geno` file: `Chr`, `Locus`, `cM`, `Mb`, then /// the individuals -- the layout every R script here already reads -- carrying /// the base population's `@`-metadata so R/qtl treats it as the same kind of /// cross. pub fn to_geno(contents: &str, labels: &[(String, String)]) -> String { let mut text = String::new(); for (key, value) in labels { text.push_str(&format!("@{key}:{value}\n")); } if labels.is_empty() { text.push_str("@type:riset\n"); } text.push_str("Chr\tLocus\tcM\tMb"); let mut header_seen = false; let mut header_individuals: Vec<&str> = Vec::new(); for line in contents.lines() { let line = line.trim_end_matches('\r'); if line.trim().is_empty() || line.starts_with('#') { continue; } let fields: Vec<&str> = line.split('\t').collect(); if !header_seen { header_seen = true; header_individuals = fields.iter().skip(3).map(|name| name.trim()).collect(); for name in &header_individuals { text.push('\t'); text.push_str(name); } text.push('\n'); continue; } if fields.len() < 4 { continue; } let marker = fields[0].trim(); let chr = fields[1].trim(); let pos = fields[2].trim(); // cM is left as the position too: GN's own files do the same for a // physical-only panel, and R/qtl2 reads the Mb column. text.push_str(&format!("{chr}\t{marker}\t{pos}\t{pos}")); for index in 0..header_individuals.len() { text.push('\t'); text.push_str(fields.get(3 + index).copied().unwrap_or("").trim()); } text.push('\n'); } text } use crate::indexer::sha256::hex_digest; fn now() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|elapsed| elapsed.as_secs() as i64) .unwrap_or(0) } /// An uploaded genotype, as the listing reports it. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] pub struct GenotypeInfo { /// `-`, as uploaded. pub name: String, /// The population it updates. pub base_version: String, #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, pub individuals: usize, /// Individuals in the base that this upload keeps. pub overlap: usize, /// Individuals the base does not have (at most [`MAX_NEW_INDIVIDUALS`]). pub new_individuals: usize, pub markers: usize, /// `letters`, `integers` or `probabilities`. pub value_kind: String, pub digest: String, pub created_at: i64, pub updated_at: i64, } /// Store a validated upload. #[allow(clippy::too_many_arguments)] pub fn create( dir: &Path, owner: &str, name: &str, base_version: &str, description: Option<&str>, parsed: &Parsed, overlap: usize, new_individuals: usize, contents: &str, ) -> Result { crate::baskets::validate_name(name)?; let mut connection = crate::baskets::open(dir, owner)?; let exists: Option = { use rusqlite::OptionalExtension; connection .query_row( "SELECT id FROM genotype WHERE name = ?1", rusqlite::params![name.trim()], |row| row.get(0), ) .optional() .map_err(|error| BasketError::Store(error.to_string()))? }; let at = now(); let digest = hex_digest(contents.as_bytes()); // An upload replaces the upload of the same name. That is the user's call // to make: a genotype is a file they keep, not a record the deployment // curates, and a mapping they ran against it quotes the digest. if exists.is_some() { connection .execute( "DELETE FROM genotype WHERE name = ?1", rusqlite::params![name.trim()], ) .map_err(|error| BasketError::Store(error.to_string()))?; } connection .execute( "INSERT INTO genotype (name, base_version, description, individuals, markers, \ value_kind, new_individuals, overlap, digest, contents, \ created_at, updated_at) \ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?11)", rusqlite::params![ name.trim(), base_version, description, parsed.individuals.len() as i64, parsed.markers.len() as i64, parsed.kind.as_str(), new_individuals as i64, overlap as i64, digest, contents, at ], ) .map_err(|error| BasketError::Store(error.to_string()))?; get(dir, owner, name) } fn info_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { Ok(GenotypeInfo { name: row.get(0)?, base_version: row.get(1)?, description: row.get(2)?, individuals: row.get::<_, i64>(3)? as usize, markers: row.get::<_, i64>(4)? as usize, value_kind: row.get(5)?, new_individuals: row.get::<_, i64>(6)? as usize, overlap: row.get::<_, i64>(7)? as usize, digest: row.get(8)?, created_at: row.get(9)?, updated_at: row.get(10)?, }) } const COLUMNS: &str = "name, base_version, description, individuals, markers, value_kind, \ new_individuals, overlap, digest, created_at, updated_at"; /// One uploaded genotype. pub fn get(dir: &Path, owner: &str, name: &str) -> Result { use rusqlite::OptionalExtension; let connection = crate::baskets::open(dir, owner)?; connection .query_row( &format!("SELECT {COLUMNS} FROM genotype WHERE name = ?1"), rusqlite::params![name.trim()], info_from_row, ) .optional() .map_err(|error| BasketError::Store(error.to_string()))? .ok_or_else(|| BasketError::NotFound(format!("no uploaded genotype named '{name}'"))) } /// Every upload this user has, newest first. pub fn list(dir: &Path, owner: &str) -> Result, BasketError> { let connection = crate::baskets::open(dir, owner)?; let mut statement = connection .prepare(&format!( "SELECT {COLUMNS} FROM genotype ORDER BY updated_at DESC, name" )) .map_err(|error| BasketError::Store(error.to_string()))?; let rows = statement .query_map([], info_from_row) .map_err(|error| BasketError::Store(error.to_string()))?; rows.collect::, _>>() .map_err(|error| BasketError::Store(error.to_string())) } /// The uploaded table, exactly as it was sent. pub fn contents(dir: &Path, owner: &str, name: &str) -> Result { use rusqlite::OptionalExtension; let connection = crate::baskets::open(dir, owner)?; connection .query_row( "SELECT contents FROM genotype WHERE name = ?1", rusqlite::params![name.trim()], |row| row.get::<_, String>(0), ) .optional() .map_err(|error| BasketError::Store(error.to_string()))? .ok_or_else(|| BasketError::NotFound(format!("no uploaded genotype named '{name}'"))) } /// Delete an upload. pub fn delete(dir: &Path, owner: &str, name: &str) -> Result<(), BasketError> { // The .geno written on demand goes too: a file nobody can name is a file // that will confuse the next operator. let written = dir .join("genotypes") .join(crate::baskets::file_stem(owner)) .join(format!("{}.geno", crate::baskets::file_stem(name))); let _ = std::fs::remove_file(written); let connection = crate::baskets::open(dir, owner)?; let changed = connection .execute( "DELETE FROM genotype WHERE name = ?1", rusqlite::params![name.trim()], ) .map_err(|error| BasketError::Store(error.to_string()))?; if changed == 0 { return Err(BasketError::NotFound(format!( "no uploaded genotype named '{name}'" ))); } Ok(()) } /// Write the upload out as a GN `.geno` file the R tooling can read, and /// return its path. Idempotent: the file is rewritten every time, so it always /// matches what is stored. pub fn materialise( dir: &Path, owner: &str, name: &str, base_labels: &[(String, String)], ) -> Result { let contents = contents(dir, owner, name)?; let root = dir.join("genotypes").join(crate::baskets::file_stem(owner)); std::fs::create_dir_all(&root) .map_err(|error| BasketError::Store(format!("{}: {error}", root.display())))?; let path = root.join(format!("{}.geno", crate::baskets::file_stem(name))); std::fs::write(&path, to_geno(&contents, base_labels)) .map_err(|error| BasketError::Store(format!("{}: {error}", path.display())))?; Ok(path) } #[cfg(test)] mod tests { use super::*; fn base(count: usize) -> Vec { (1..=count).map(|index| format!("BXD{index}")).collect() } fn table(individuals: &[&str], cells: &[&str]) -> String { let mut text = String::from("marker\tchr\tpos"); for name in individuals { text.push('\t'); text.push_str(name); } text.push('\n'); for (index, cell) in cells.iter().enumerate() { text.push_str(&format!("rs{}\t1\t{}.0", index + 1, index + 3)); for _ in individuals { text.push('\t'); text.push_str(cell); } text.push('\n'); } text } #[test] fn a_letters_matrix_parses_into_individuals_markers_and_kind() { let parsed = parse(&table(&["BXD1", "BXD2"], &["B", "D", "H"])).unwrap(); assert_eq!(parsed.individuals, vec!["BXD1", "BXD2"]); assert_eq!(parsed.markers.len(), 3); assert_eq!(parsed.markers[0], ("rs1".to_string(), "1".to_string(), 3.0)); assert_eq!(parsed.kind, ValueKind::Letters); assert_eq!(parsed.missing, 0); } #[test] fn the_three_value_kinds_are_told_apart() { assert_eq!(parse(&table(&["BXD1"], &["0", "1", "2"])).unwrap().kind, ValueKind::Integers); assert_eq!( parse(&table(&["BXD1"], &["0.02", "0.98"])).unwrap().kind, ValueKind::Probabilities ); // Missing cells are not a kind. let with_missing = parse(&table(&["BXD1"], &["B", "", "U"])).unwrap(); assert_eq!(with_missing.kind, ValueKind::Letters); assert_eq!(with_missing.missing, 2); } #[test] fn a_malformed_or_mixed_upload_is_refused() { // Commas are not this format. let commas = "marker,chr,pos,BXD1\nrs1,1,3.4,B\n"; assert!(parse(commas).unwrap_err().to_string().contains("tab-delimited")); // A row that is not as wide as the header. let ragged = "marker\tchr\tpos\tBXD1\tBXD2\nrs1\t1\t3.4\tB\n"; assert!(parse(ragged).unwrap_err().to_string().contains("field")); // Mixed kinds, named. let mixed = "marker\tchr\tpos\tBXD1\nrs1\t1\t3.4\tB\nrs2\t1\t4.0\t1\n"; let error = parse(mixed).unwrap_err().to_string(); assert!(error.contains("integers") && error.contains("letters"), "{error}"); // A position that is not a number. let bad_position = "marker\tchr\tpos\tBXD1\nrs1\t1\tnear\tB\n"; assert!(parse(bad_position).unwrap_err().to_string().contains("position")); } #[test] fn the_update_limits_are_enforced() { let individuals = base(60); let parsed = parse(&table(&["BXD1", "BXD2"], &["B"])).unwrap(); // Two of sixty is an update. let (overlap, new) = check_base(&parsed, &individuals, ValueKind::Letters).unwrap(); assert_eq!((overlap, new), (2, 0)); // Fifty-one new names is an ingest. let many: Vec = (0..51).map(|index| format!("NEW{index}")).collect(); let names: Vec<&str> = std::iter::once("BXD1") .chain(many.iter().map(String::as_str)) .collect(); let too_many = parse(&table(&names, &["B"])).unwrap(); let error = check_base(&too_many, &individuals, ValueKind::Letters) .unwrap_err() .to_string(); assert!(error.contains("at most 50"), "{error}"); // No overlap is not an update. let strangers = parse(&table(&["NEW1", "NEW2"], &["B"])).unwrap(); let error = check_base(&strangers, &individuals, ValueKind::Letters) .unwrap_err() .to_string(); assert!(error.contains("updates a population"), "{error}"); // And the value kind has to match the population's. let numbers = parse(&table(&["BXD1"], &["1"])).unwrap(); let error = check_base(&numbers, &individuals, ValueKind::Letters) .unwrap_err() .to_string(); assert!(error.contains("letters") && error.contains("integers"), "{error}"); } #[test] fn the_geno_layout_is_what_the_r_scripts_read() { let labels = vec![ ("type".to_string(), "riset".to_string()), ("mat".to_string(), "B".to_string()), ("pat".to_string(), "D".to_string()), ]; let text = to_geno(&table(&["BXD1", "BXD2"], &["B", "D"]), &labels); assert!(text.starts_with("@type:riset\n@mat:B\n@pat:D\n"), "{text}"); assert!(text.contains("Chr\tLocus\tcM\tMb\tBXD1\tBXD2\n"), "{text}"); // Chr first, then the marker: GN's layout, not the upload's. // The helper gives every individual the same call per marker. assert!(text.contains("\n1\trs1\t3.0\t3.0\tB\tB\n"), "{text}"); assert!(text.contains("\n1\trs2\t4.0\t4.0\tD\tD\n"), "{text}"); } } /// Convert the upload to GEMMA's BIMBAM pair: `(geno, snps)`. /// /// GN's own BIMBAM files encode a call as a dosage in {0, 0.5, 1} with the /// alleles left as the placeholders `X, Y`, and the snps file as /// `marker, position in bp, chromosome`. The coding follows the base /// population's `@mat`/`@het`/`@pat` when it declares one (mat -> 1, het -> /// 0.5, pat -> 0), and falls back to the file's own first/last letters, so the /// direction matches the population the upload updates. pub fn to_bimbam(contents: &str, labels: &[(String, String)]) -> (String, String) { let label = |key: &str| { labels .iter() .find(|(name, _)| name == key) .map(|(_, value)| value.trim().to_string()) }; let het = label("het").unwrap_or_else(|| "H".to_string()); let mut coding: std::collections::BTreeMap = std::collections::BTreeMap::new(); if let (Some(mat), Some(pat)) = (label("mat"), label("pat")) { coding.insert(mat, "1"); coding.insert(het.clone(), "0.5"); coding.insert(pat, "0"); } let mut geno = String::new(); let mut snps = String::new(); let mut header_seen = false; let mut individuals = 0usize; for line in contents.lines() { let line = line.trim_end_matches('\r'); if line.trim().is_empty() || line.starts_with('#') { continue; } let fields: Vec<&str> = line.split('\t').collect(); if !header_seen { header_seen = true; individuals = fields.len().saturating_sub(3); continue; } if fields.len() < 4 { continue; } let marker = fields[0].trim(); let chr = fields[1].trim(); let positions: Vec<&str> = fields[3..].iter().take(individuals).map(|c| c.trim()).collect(); // Without declared codes, use the calls themselves: the "het" is 0.5, // the rest are the two homozygotes in the order they first appear. if coding.is_empty() { let mut letters: Vec = positions .iter() .filter(|cell| !is_missing(cell)) .map(|cell| cell.to_string()) .collect(); letters.sort(); letters.dedup(); for letter in letters { let code = if letter == het { "0.5" } else if coding.len() < 2 { "1" } else { "0" }; coding.insert(letter, code); } } geno.push_str(marker); geno.push_str(", X, Y"); for cell in &positions { geno.push_str(", "); let value = if is_missing(cell) { "NA".to_string() } else { coding.get(*cell).copied().unwrap_or("0.5").to_string() }; geno.push_str(&value); } geno.push('\n'); let base_pairs = fields[2] .trim() .parse::() .map(|megabases| (megabases * 1e6).round() as i64) .unwrap_or(0); snps.push_str(&format!("{marker}, {base_pairs}, {chr}\n")); } (geno, snps) } /// Write the upload out as a BIMBAM pair, returning `(geno, snps)` paths. pub fn materialise_bimbam( dir: &Path, owner: &str, name: &str, base_labels: &[(String, String)], ) -> Result<(std::path::PathBuf, std::path::PathBuf), BasketError> { let contents = contents(dir, owner, name)?; let (geno_text, snps_text) = to_bimbam(&contents, base_labels); let root = dir.join("genotypes").join(crate::baskets::file_stem(owner)); std::fs::create_dir_all(&root) .map_err(|error| BasketError::Store(format!("{}: {error}", root.display())))?; let stem = crate::baskets::file_stem(name); let geno = root.join(format!("{stem}_geno.txt")); let snps = root.join(format!("{stem}_snps.txt")); for (path, text) in [(&geno, geno_text), (&snps, snps_text)] { std::fs::write(path, text) .map_err(|error| BasketError::Store(format!("{}: {error}", path.display())))?; } Ok((geno, snps)) } /// The individuals of an upload, in its own order (the geno column order). pub fn individuals(dir: &Path, owner: &str, name: &str) -> Result, BasketError> { let parsed = parse(&contents(dir, owner, name)?)?; Ok(parsed.individuals) }