//! Covariates, shared by the two engines that can fit them. //! //! A covariate is a per-individual number fitted alongside the trait. The //! two runners take them differently, which is why this module exists: //! //! - R/qtl's wrapper wants extra **phenotype-file columns** plus a //! `--covarstruct` file saying which are categorical, and expands more //! than two levels itself (`write_covarstruct_file`); //! - GEMMA wants a separate **covariate file**, positional over the //! genotype panel, numeric only, and knows nothing about factors -- so a //! categorical covariate has to be dummy-expanded here //! (`write_gemma_covariates_file`). //! //! Both get the same inputs: the caller names a case attribute of the //! group (`{"case_attribute": "Sex"}`), sends named values, or maps text //! explicitly, and [`resolve`] aligns whatever arrives onto the samples //! being mapped and codes the text (see [`code_case_attribute`]). A //! missing value is `None`, which each runner turns into its own missing //! marker -- and both then drop that individual, per the measured //! behaviour of the engines themselves (R/qtl: `na.strings="x"` in the //! wrapper; GEMMA: a literal `NA` in the covariate file, after which its //! log says "number of analyzed individuals" one lower). //! //! Covariates are never inferred from database text except through //! [`code_case_attribute`]'s explicit rules: 2-20 distinct values are //! coded in sorted order and the coding is reported in the response's //! `notes`, so two runs of "the same" covariate can be compared. use std::collections::{BTreeMap, HashMap}; use std::io; use std::path::Path; use axum::http::StatusCode; use crate::db; use crate::error::ApiError; use crate::response::RequestInfo; /// Whether a covariate is a factor or a number. R/qtl reads this from the /// generated `--covarstruct` file; GEMMA has no factors, so it decides how /// many columns [`write_gemma_covariates_file`] writes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CovariateKind { /// Two or more levels: R/qtl's wrapper dummy-expands more than two /// itself; GEMMA always gets k-1 dummies. Categorical, /// A number, used as-is. Numerical, } impl CovariateKind { pub fn as_str(self) -> &'static str { match self { CovariateKind::Categorical => "categorical", CovariateKind::Numerical => "numerical", } } } /// One covariate, already on the samples' scale: `values` is positional, /// one entry per sample, `None` where that sample has no covariate value /// (which both engines turn into a dropped individual, not a zero). #[derive(Debug, Clone)] pub struct Covariate { pub name: String, pub kind: CovariateKind, pub values: Vec>, /// The distinct levels, in the order they were coded, for a /// categorical covariate: GEMMA gets k-1 dummy columns named after /// them (`sex.M`), where R/qtl expands the single column itself. /// Empty for a numerical covariate. pub levels: Vec, } /// The most distinct values an un-mapped text attribute may have and /// still be coded automatically. Beyond this a covariate is refused /// rather than silently turned into an arbitrary ordering. pub const MAX_AUTO_LEVELS: usize = 20; /// Turn one case attribute's text values into a covariate: numeric /// columns pass through, anything else is coded 0..k-1 and reported as /// categorical. /// /// An explicit `map` is honoured exactly (and must cover every value /// present); without one, text is only coded when it has between 2 and /// [`MAX_AUTO_LEVELS`] distinct values. pub fn code_case_attribute( values: &[Option], map: Option<&BTreeMap>, ) -> Result { if let Some(map) = map { let mut coded = Vec::with_capacity(values.len()); let mut missing_mapping: Vec<&str> = Vec::new(); for value in values { let coded_value = match value.as_deref() { None => None, Some(text) => match map.get(text) { Some(number) => Some(*number), None => { missing_mapping.push(text); None } }, }; coded.push(coded_value); } if !missing_mapping.is_empty() { missing_mapping.sort_unstable(); missing_mapping.dedup(); return Err(format!( "the map does not cover {}; add mappings for those or leave the \ attribute out", missing_mapping .iter() .map(|value| format!("'{value}'")) .collect::>() .join(", ") )); } let levels = map.len(); if levels < 2 { return Err("the map has fewer than two levels, so the covariate cannot vary".to_string()); } return Ok(CodedAttribute { values: coded, kind: CovariateKind::Categorical, coding: Some( map.iter() .map(|(value, number)| format!("{value}={number}")) .collect::>() .join(", "), ), levels: map .iter() .map(|(name, code)| Level::new(name.clone(), *code)) .collect(), }); } let mut levels: Vec<&str> = values .iter() .filter_map(|value| value.as_deref()) .filter(|value| !value.is_empty()) .collect(); levels.sort_unstable(); levels.dedup(); if levels.len() < 2 { return Err(format!( "the attribute has {} usable level(s) among these samples ({}), so it cannot \ be a covariate", levels.len(), if levels.is_empty() { "no values at all".to_string() } else { format!("all '{}'", levels[0]) } )); } // All numeric? Use them as they are. let numbers: Option> = levels .iter() .map(|level| level.trim().parse::().ok()) .collect(); if numbers.is_some() { let coded = values .iter() .map(|value| { value .as_deref() .map(str::trim) .filter(|text| !text.is_empty()) .and_then(|text| text.parse::().ok()) }) .collect(); return Ok(CodedAttribute { values: coded, kind: CovariateKind::Numerical, coding: None, levels: Vec::new(), }); } // Mostly numbers with a few number-*like* entries -- `SeqCvge`'s // "36+39L" beside "40" -- would otherwise be coded as categories, and // GEMMA then fails on the resulting dummies with a message about its own // LU decomposition rather than about the covariate. Refuse it here and // say what to send instead. Entries with no digits at all ("unknown") // stay categorical: that is a real level, not a dirty number. let numeric = levels.iter().filter(|level| level.parse::().is_ok()).count(); if numeric > 0 && numeric < levels.len() { let dirty: Vec<&str> = levels .iter() .copied() .filter(|level| level.parse::().is_err() && level.chars().any(|c| c.is_ascii_digit())) .collect(); if !dirty.is_empty() { let example = dirty.first().copied().unwrap_or_default(); return Err(format!( "the attribute mixes numbers with number-like text ({} of {} values look \ numeric, but e.g. '{example}' does not), so coding it as categories would fit \ a quantity as a factor -- and GEMMA fails on the result. Pass an explicit \ `map` giving each of those a number (there are {} such value(s): {}), or send \ the numbers inline", numeric, levels.len(), dirty.len(), dirty.join(", ") )); } } if levels.len() > MAX_AUTO_LEVELS { return Err(format!( "the attribute has {} distinct values, too many to code automatically (limit \ {}); pass an explicit `map`", levels.len(), MAX_AUTO_LEVELS )); } let mut mapping = std::collections::BTreeMap::new(); for (index, level) in levels.iter().enumerate() { mapping.insert((*level).to_string(), index as f64); } let coded = values .iter() .map(|value| { value .as_deref() .filter(|text| !text.is_empty()) .and_then(|text| mapping.get(text).copied()) }) .collect(); Ok(CodedAttribute { values: coded, kind: CovariateKind::Categorical, coding: Some( mapping .iter() .map(|(value, number)| format!("{value}={number}")) .collect::>() .join(", "), ), levels: mapping .iter() .map(|(name, code)| Level::new(name.clone(), *code)) .collect(), }) } /// One level of a categorical covariate: what it is called, and the code /// the values carry for it. Both are needed -- an explicit `map` may code /// `HFD=1` where `LFD=0`, so the level's position in the sorted list is /// not its code. #[derive(Debug, Clone, PartialEq)] pub struct Level { pub name: String, pub code: f64, } impl Level { fn new(name: impl Into, code: f64) -> Self { Level { name: name.into(), code, } } } /// What [`code_case_attribute`] produced. #[derive(Debug)] pub struct CodedAttribute { pub values: Vec>, pub kind: CovariateKind, pub coding: Option, /// The levels, in coded order (empty for a numerical result). pub levels: Vec, } /// Write the wrapper's `--covarstruct` file: one `namekind` row per /// covariate, the format `read.table(opt$covarstruct, sep="\t", /// header=FALSE)` expects. pub fn write_covarstruct_file(path: &Path, covariates: &[Covariate]) -> io::Result { let mut contents = String::new(); for covariate in covariates { contents.push_str(&format!( "{}\t{}\n", covariate.name, covariate.kind.as_str() )); } std::fs::write(path, &contents)?; Ok(contents) } /// How many covariates one request may fit. pub const MAX_COVARIATES: usize = 8; /// One covariate in a request: a case attribute of the group, by name, or /// explicit per-sample numbers. Both runners take this same shape. #[derive(Debug, Clone, serde::Deserialize, utoipa::ToSchema)] #[schema(as = CovariateSpec)] pub struct Spec { /// Case attribute to use, e.g. "Sex". The server reads it for the /// group, aligns it onto the panel and codes its text. pub case_attribute: Option, /// Optional explicit coding for a case attribute, e.g. {"F": 0, "M": 1}. pub map: Option>, /// For an inline covariate: its name. pub name: Option, /// For an inline covariate: "categorical" (default) or "numerical". pub kind: Option, /// For an inline covariate: one value per sample, matched by name. pub values: Option>, } /// A group's case-attribute columns for this request: the values aligned /// onto the samples being mapped, and the descriptions that explain the /// names. #[derive(Default)] struct AttributeColumns { columns: HashMap>>, descriptions: HashMap, } /// One inline covariate value, by sample name. #[derive(Debug, Clone, serde::Deserialize, utoipa::ToSchema)] #[schema(as = CovariateValue)] pub struct NamedValue { pub name: String, pub value: Option, } /// Resolve request specs onto the samples being mapped. /// /// Returns the covariates in request order and one `notes` line each /// (what was coded, how many samples have a value, and -- when /// `has_trait` says which samples carry a trait value -- how many have /// both, i.e. the complete cases the fit will use). `group` is where a /// `case_attribute` is read from; without one, only inline covariates are /// possible. pub async fn resolve( specs: &[Spec], sample_names: &[String], group: Option<&str>, has_trait: Option<&[bool]>, rid: RequestInfo, ) -> Result<(Vec, Vec), ApiError> { // `RequestInfo` is not `Copy`; every error path here needs its own. let rid_at = |_: ()| rid.clone(); let mut covariates: Vec = Vec::new(); let mut notes: Vec = Vec::new(); if specs.is_empty() { return Ok((covariates, notes)); } if specs.len() > MAX_COVARIATES { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "too_many_covariates", "Too many covariates", format!( "{} covariates given; at most {MAX_COVARIATES} are accepted.", specs.len() ), "covariates", ) .with_request_id(rid_at(()))); } // Case attributes come from the group, once for the whole request. let wanted: Vec<&str> = specs .iter() .filter_map(|spec| spec.case_attribute.as_deref()) .map(str::trim) .collect(); let attributes = if wanted.is_empty() { AttributeColumns::default() } else { let Some(group) = group else { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "covariate_unavailable", "Covariate unavailable", "a case attribute needs a `group` on the request; with a genotype_file, pass the covariate's values inline" .to_string(), "covariates", ) .with_request_id(rid_at(()))); }; let group_owned = group.to_string(); let names = sample_names.to_vec(); let attributes = tokio::task::spawn_blocking(move || { let Some(row) = db::get_group_by_name(&group_owned)? else { return Ok::<_, db::DbError>(None); }; db::get_group_case_attributes(row.id).map(Some) }) .await .map_err(|e| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", e.to_string(), "covariates", ) .with_request_id(rid_at(())) })? .map_err(|e| ApiError::from_db(e, "covariates").with_request_id(rid_at(())))?; let Some(attributes) = attributes else { return Err(ApiError::new( StatusCode::NOT_FOUND, "group_not_found", "Group not found", "the request's group does not exist, so its case attributes cannot be read" .to_string(), "covariates", ) .with_request_id(rid_at(()))); }; for attribute in &wanted { if !attributes.names.iter().any(|name| name == attribute) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "covariate_unavailable", "Covariate unavailable", format!( "case attribute '{attribute}' is not defined for this group; it has: {}", if attributes.names.is_empty() { "none".to_string() } else { attributes.names.join(", ") } ), "covariates", ) .with_request_id(rid_at(()))); } } let by_strain: HashMap<&str, &HashMap> = attributes .values .iter() .map(|row| (row.strain.as_str(), &row.case_attributes)) .collect(); let mut found = AttributeColumns::default(); for attribute in wanted { let column = names .iter() .map(|name| { by_strain .get(name.as_str()) .and_then(|attributes| attributes.get(attribute)) .map(|value| value.trim().to_string()) .filter(|value| !value.is_empty()) }) .collect(); if let Some(description) = attributes.descriptions.get(attribute) { found .descriptions .insert(attribute.to_string(), description.clone()); } found.columns.insert(attribute.to_string(), column); } found }; for spec in specs { let (name, kind, values, coding, levels) = match (&spec.case_attribute, &spec.values) { (Some(attribute), None) => { let attribute = attribute.trim(); let Some(column) = attributes.columns.get(attribute) else { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "covariate_unavailable", "Covariate unavailable", format!("case attribute '{attribute}' could not be read"), "covariates", ) .with_request_id(rid_at(()))); }; let coded = code_case_attribute(column, spec.map.as_ref()).map_err(|detail| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_covariate", "Invalid covariate", format!("case attribute '{attribute}': {detail}"), "covariates", ) .with_request_id(rid_at(())) })?; ( attribute.to_string(), coded.kind, coded.values, coded.coding, coded.levels, ) } (None, Some(values)) => { let Some(name) = spec.name.clone() else { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_covariate", "Invalid covariate", "an inline covariate needs a `name`".to_string(), "covariates", ) .with_request_id(rid_at(()))); }; let by_name: HashMap<&str, Option> = values .iter() .map(|value| (value.name.as_str(), value.value)) .collect(); let coded: Vec> = sample_names .iter() .map(|name| by_name.get(name.as_str()).copied().flatten()) .collect(); if coded.iter().all(Option::is_none) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "covariate_unavailable", "Covariate unavailable", format!( "none of the {} values of covariate '{name}' match a sample of the trait", values.len() ), "covariates", ) .with_request_id(rid_at(()))); } let (kind, levels) = match spec .kind .as_deref() .unwrap_or("categorical") .to_ascii_lowercase() .as_str() { "categorical" => { // Codes come from the values themselves, sorted, so // the dummy columns (GEMMA) have stable names. let mut distinct: Vec = coded.iter().filter_map(|v| *v).collect(); distinct.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); distinct.dedup(); let levels = distinct .iter() .map(|code| Level::new(format!("{code}"), *code)) .collect(); (CovariateKind::Categorical, levels) } "numerical" => (CovariateKind::Numerical, Vec::new()), other => { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_covariate", "Invalid covariate", format!( "covariate '{name}': kind '{other}' is not 'categorical' or 'numerical'" ), "covariates", ) .with_request_id(rid_at(()))); } }; (name, kind, coded, None, levels) } (Some(_), Some(_)) => { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_covariate", "Invalid covariate", "give either case_attribute or values for a covariate, not both".to_string(), "covariates", ) .with_request_id(rid_at(()))); } (None, None) => { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_covariate", "Invalid covariate", "a covariate needs a case_attribute, or a name and values".to_string(), "covariates", ) .with_request_id(rid_at(()))); } }; if covariates.iter().any(|covariate| covariate.name == name) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "duplicate_covariate", "Duplicate covariate", format!("covariate '{name}' is given more than once."), "covariates", ) .with_request_id(rid_at(()))); } let present = values.iter().filter(|value| value.is_some()).count(); let complete = has_trait .map(|has_trait| { format!( ", {} with both the trait and the covariate", values .iter() .zip(has_trait) .filter(|(value, has_trait)| value.is_some() && **has_trait) .count() ) }) .unwrap_or_default(); // Just the levels: how the runner turns them into columns is the // runner's business (GEMMA's own note names the dummies it was // given; R/qtl's wrapper expands the one column itself). // The attribute's own description, if the database has one: the // names are not self-explanatory (`SeqCvge` is a sequencing depth, // not a sequence coverage *phenotype*), and a covariate whose // meaning is guessed is a covariate fitted for the wrong reason. let meaning = attributes .descriptions .get(&name) .map(|description| format!(" ({description})")) .unwrap_or_default(); notes.push(format!( "Covariate {name}{meaning}: {}{}, {present} of {} samples have a value{complete}{}", kind.as_str(), coding .as_deref() .map(|coding| format!(" coded {coding}")) .unwrap_or_default(), values.len(), if kind == CovariateKind::Categorical { format!("; {} levels", levels.len()) } else { String::new() } )); covariates.push(Covariate { name, kind, values, levels, }); } Ok((covariates, notes)) } /// What GEMMA's covariate file turned out to be. pub struct GemmaCovariates { /// The file's contents: no header, no sample names, tab separated. pub contents: String, /// The columns written, in order (`sex.M`, `Age`, ...). pub columns: Vec, /// Columns left out because they do not vary among the individuals /// that will enter the fit, with why. pub dropped: Vec, /// For each covariate, how many individuals have no value for it -- /// GEMMA drops those, exactly as it drops a missing phenotype. pub missing: Vec<(String, usize)>, } /// Write GEMMA's covariate file: no header, no sample names, one row per /// individual in the panel's order, one column per covariate -- and, for a /// categorical covariate, k-1 dummy columns (`sex.M`) instead of its /// level codes, because GEMMA reads numbers and knows nothing about /// factors (R/qtl's wrapper does that expansion for itself). /// /// A missing value is written as `NA`, which GEMMA treats as a missing /// phenotype does: that individual leaves the fit (measured). /// /// `analysed` says which rows have a phenotype, so a dummy for a level /// nobody analysed carries has to go: an all-zero column is singular and /// GEMMA stops with `LUDecomp ... input domain error` rather than /// dropping it the way R would. Dropping it changes nothing about the /// model -- a level with no observations contributes no term. pub fn write_gemma_covariates_file( path: &Path, covariates: &[Covariate], analysed: &[bool], ) -> std::io::Result { // A row enters the fit only if it has a phenotype and a value for // every covariate (GEMMA drops it otherwise). let mut in_fit: Vec = analysed.to_vec(); in_fit.resize( covariates .first() .map(|covariate| covariate.values.len()) .unwrap_or(0), false, ); for covariate in covariates { for (keep, value) in in_fit.iter_mut().zip(&covariate.values) { if value.is_none() { *keep = false; } } } let mut columns: Vec<(String, Vec>)> = Vec::new(); let mut dropped: Vec = Vec::new(); let mut missing: Vec<(String, usize)> = Vec::new(); for covariate in covariates { let absent = covariate.values.iter().filter(|value| value.is_none()).count(); if absent > 0 { missing.push((covariate.name.clone(), absent)); } let mut candidates: Vec<(String, Vec>)> = Vec::new(); match covariate.kind { CovariateKind::Numerical => { candidates.push((covariate.name.clone(), covariate.values.clone())); } CovariateKind::Categorical => { // The first level is the reference: k-1 dummies. for level in covariate.levels.iter().skip(1) { let indicator = covariate .values .iter() .map(|value| value.map(|code| if code == level.code { 1.0 } else { 0.0 })) .collect(); candidates.push((format!("{}.{}", covariate.name, level.name), indicator)); } } } for (name, values) in candidates { // Constant (or empty) across the rows that matter: no term. let kept: Vec = values .iter() .zip(&in_fit) .filter(|(_, keep)| **keep) .filter_map(|(value, _)| *value) .collect(); let varies = kept .first() .map(|first| kept.iter().any(|value| value != first)) .unwrap_or(false); if varies { columns.push((name, values)); } else { dropped.push(name); } } } let rows = covariates .first() .map(|covariate| covariate.values.len()) .unwrap_or(0); let mut contents = String::new(); for row in 0..rows { for (index, (_, values)) in columns.iter().enumerate() { if index > 0 { contents.push('\t'); } match values.get(row).copied().flatten() { Some(value) => contents.push_str(&value.to_string()), None => contents.push_str("NA"), } } if !columns.is_empty() { contents.push('\n'); } } std::fs::write(path, &contents)?; Ok(GemmaCovariates { contents, columns: columns.into_iter().map(|(name, _)| name).collect(), dropped, missing, }) } #[cfg(test)] mod tests { use super::*; fn levels(levels: &[Level]) -> Vec { levels.iter().map(|level| level.name.clone()).collect() } fn texts(values: &[&str]) -> Vec> { values .iter() .map(|value| Some(value.to_string())) .collect() } #[test] fn text_covariates_are_coded_and_numeric_ones_pass_through() { let sex = texts(&["F", "M", "", "F"]); let coded = code_case_attribute(&sex, None).unwrap(); assert_eq!(coded.kind, CovariateKind::Categorical); assert_eq!(coded.values, vec![Some(0.0), Some(1.0), None, Some(0.0)]); assert_eq!(coded.coding.as_deref(), Some("F=0, M=1")); // The levels, in coded order: what a dummy expansion names itself // after. assert_eq!( levels(&coded.levels), vec!["F".to_string(), "M".to_string()] ); let ages = texts(&["183", "200"]); let coded = code_case_attribute(&ages, None).unwrap(); assert_eq!(coded.kind, CovariateKind::Numerical); assert_eq!(coded.values, vec![Some(183.0), Some(200.0)]); assert!(coded.levels.is_empty()); // One level, or none, cannot be a covariate. let error = code_case_attribute(&texts(&["Live", "Live"]), None).unwrap_err(); assert!(error.contains("cannot be a covariate"), "{error}"); } #[test] fn an_explicit_map_is_honoured_and_must_be_complete() { let values = texts(&["HFD", "LFD"]); let mut map = BTreeMap::new(); map.insert("LFD".to_string(), 0.0); map.insert("HFD".to_string(), 1.0); let coded = code_case_attribute(&values, Some(&map)).unwrap(); assert_eq!(coded.kind, CovariateKind::Categorical); assert_eq!(coded.values, vec![Some(1.0), Some(0.0)]); assert_eq!(coded.coding.as_deref(), Some("HFD=1, LFD=0")); // Ordered by name, coded by the map: HFD=1 comes before LFD=0, so // a dummy has to key on the code, not the position. assert_eq!( levels(&coded.levels), vec!["HFD".to_string(), "LFD".to_string()] ); assert_eq!(coded.levels[0].code, 1.0); map.remove("LFD"); let error = code_case_attribute(&values, Some(&map)).unwrap_err(); assert!(error.contains("'LFD'"), "{error}"); } #[test] fn a_mostly_numeric_attribute_asks_for_a_map_instead_of_fitting_a_factor() { // SeqCvge in BXD: plain depths beside two-library values. let values = texts(&["40", "36+39L", "41", "40+31L"]); let error = code_case_attribute(&values, None).unwrap_err(); assert!(error.contains("mixes numbers with number-like text"), "{error}"); assert!(error.contains("36+39L") && error.contains("40+31L"), "{error}"); assert!(error.contains("`map`"), "{error}"); // An explicit map settles it. let mut map = BTreeMap::new(); map.insert("40".to_string(), 40.0); map.insert("41".to_string(), 41.0); map.insert("36+39L".to_string(), 75.0); map.insert("40+31L".to_string(), 71.0); let coded = code_case_attribute(&values, Some(&map)).unwrap(); assert_eq!(coded.kind, CovariateKind::Categorical); assert_eq!(coded.values[1], Some(75.0)); // A level that is not a dirty number stays a level. let mixed = texts(&["1", "2", "unknown"]); let coded = code_case_attribute(&mixed, None).unwrap(); assert_eq!(coded.kind, CovariateKind::Categorical); assert_eq!(coded.values.len(), 3); } #[test] fn too_many_levels_asks_for_an_explicit_map() { let values: Vec> = (0..MAX_AUTO_LEVELS + 1).map(|i| Some(format!("level-{i}"))).collect(); let error = code_case_attribute(&values, None).unwrap_err(); assert!(error.contains("pass an explicit `map`"), "{error}"); } fn write_analysed(covariates: &[Covariate], analysed: &[bool]) -> GemmaCovariates { let path = std::env::temp_dir().join(format!( "gn4-covariates-{}-{}", std::process::id(), covariates.len() )); let written = write_gemma_covariates_file(&path, covariates, analysed).unwrap(); let _ = std::fs::remove_file(&path); written } #[test] fn gemma_gets_one_column_per_numerical_covariate_and_na_for_a_missing_one() { // No header, no names: GEMMA reads the file positionally. let written = write_analysed( &[Covariate { name: "Age".to_string(), kind: CovariateKind::Numerical, values: vec![Some(183.0), None, Some(200.5)], levels: Vec::new(), }], &[true, true, true], ); assert_eq!(written.contents, "183\nNA\n200.5\n"); assert_eq!(written.columns, vec!["Age".to_string()]); assert_eq!(written.missing, vec![("Age".to_string(), 1)]); } #[test] fn a_two_level_covariate_becomes_one_dummy_named_after_the_second_level() { let written = write_analysed( &[Covariate { name: "Sex".to_string(), kind: CovariateKind::Categorical, values: vec![Some(0.0), Some(1.0), None, Some(0.0)], levels: vec![Level::new("F", 0.0), Level::new("M", 1.0)], }], &[true, true, true, true], ); // F is the reference, so only M's dummy is written -- and the // individual with no value is NA in it, so GEMMA drops them. assert_eq!(written.columns, vec!["Sex.M".to_string()]); assert_eq!(written.contents, "0\n1\nNA\n0\n"); } #[test] fn a_three_level_covariate_becomes_two_dummies_and_a_missing_value_stays_missing() { let written = write_analysed( &[ Covariate { name: "Diet".to_string(), kind: CovariateKind::Categorical, values: vec![Some(0.0), Some(1.0), Some(2.0), None], levels: vec![ Level::new("HFD", 0.0), Level::new("LFD", 1.0), Level::new("CR", 2.0), ], }, Covariate { name: "Age".to_string(), kind: CovariateKind::Numerical, values: vec![Some(1.0), Some(2.0), Some(3.0), Some(4.0)], levels: Vec::new(), }, ], &[true, true, true, true], ); assert_eq!( written.columns, vec!["Diet.LFD".to_string(), "Diet.CR".to_string(), "Age".to_string()] ); assert_eq!(written.contents, "0\t0\t1\n1\t0\t2\n0\t1\t3\nNA\tNA\t4\n"); } #[test] fn a_dummy_no_analysed_individual_carries_is_dropped() { // The level exists in the panel but none of the individuals with // a phenotype is in it: an all-zero column is singular, and GEMMA // stops on it rather than dropping the term as R would. let written = write_analysed( &[Covariate { name: "Status".to_string(), kind: CovariateKind::Categorical, values: vec![Some(0.0), Some(0.0), Some(1.0), Some(2.0)], levels: vec![ Level::new("Cryo", 0.0), Level::new("Live", 1.0), Level::new("Ext", 2.0), ], }], // Only the Cryo and Live individuals have a phenotype. &[true, true, true, false], ); assert_eq!(written.columns, vec!["Status.Live".to_string()]); assert_eq!(written.dropped, vec!["Status.Ext".to_string()]); assert_eq!(written.contents, "0\n0\n1\n0\n"); // Nothing varying at all leaves no covariates to fit. let written = write_analysed( &[Covariate { name: "Status".to_string(), kind: CovariateKind::Categorical, values: vec![Some(0.0), Some(1.0)], levels: vec![Level::new("Cryo", 0.0), Level::new("Live", 1.0)], }], &[true, false], ); assert!(written.columns.is_empty(), "{:?}", written.columns); assert_eq!(written.contents, ""); } #[test] fn an_inline_categorical_is_expanded_from_its_own_codes() { // No names are known for inline values, so the dummies are named // after the codes, which is all a caller can then report. let spec = Spec { case_attribute: None, map: None, name: Some("Batch".to_string()), kind: None, values: Some(vec![ NamedValue { name: "A".to_string(), value: Some(0.0) }, NamedValue { name: "B".to_string(), value: Some(1.0) }, NamedValue { name: "C".to_string(), value: Some(2.0) }, ]), }; let runtime = tokio::runtime::Builder::new_current_thread().build().unwrap(); let names = vec!["A".to_string(), "B".to_string(), "C".to_string()]; let (covariates, notes) = runtime .block_on(resolve( &[spec], &names, None, Some(&[true, true, true]), RequestInfo::new(None), )) .unwrap(); assert_eq!( levels(&covariates[0].levels), vec!["0".to_string(), "1".to_string(), "2".to_string()] ); assert!(notes[0].contains("; 3 levels"), "{notes:?}"); assert!( notes[0].contains("3 with both the trait and the covariate"), "the complete cases are reported: {notes:?}" ); let written = write_analysed(&covariates, &[true, true, true]); assert_eq!(written.columns, vec!["Batch.1".to_string(), "Batch.2".to_string()]); assert_eq!(written.contents, "0\t0\n1\t0\n0\t1\n"); } }