//! Trait-value normalisation: the transforms GeneNetwork's trait page //! offers, plus the outlier handling GN2 only *warns* about. //! //! Two fields, deliberately separate, because they are different kinds of //! operation and compose in a fixed order: //! //! - `normalize` changes the **scale**: `log2`, `log10`, `sqrt`, `negate`, //! `zscore`, `robust_zscore`, `rank`; //! - `winsor_quantile` / `winsor_fences` change the **tails**: clip at a //! quantile, or at Tukey fences (`Q1 - k*IQR`, `Q3 + k*IQR` -- the same //! rule GeneNetwork2 uses to *flag* outliers, `gn2/utility/Plot.py`). //! //! The order is fixed and reported: **transform, then winsorise, then //! standardise.** Clipping acts on the scale being analysed, and the last //! step lands exactly on mean 0 / sd 1 (or the rank scale) rather than on //! the standard deviation of clipped data. //! //! Two GeneNetwork2 quirks are deliberately **not** reproduced: //! //! - its "Quantile" (`qnorm`) is a rank-based inverse-normal transform //! (Blom scores), not array quantile normalisation; here it is called //! `rank` (`qnorm` is accepted as an alias so GN2 muscle memory works); //! - its `sqrt` silently adds 1 when any value is in `[0, 1]`. Here a //! negative value is refused and no offset is ever applied -- a silent //! shift changes every effect size. //! //! Missing values stay missing through every step: they are never //! imputed, so the runs that drop them (GEMMA for a missing covariate or //! phenotype, the R/qtl wrapper for `x`) still do. use std::fmt; /// The three request fields, as query parameters (`/values`) or as plain /// fields on a compute request -- one spelling everywhere. #[derive(Debug, Clone, Default, serde::Deserialize, utoipa::IntoParams)] pub struct Params { /// `none` (default), `log2`, `log10`, `sqrt`, `negate` (GN2: invert), /// `zscore`, `robust_zscore`, or `rank` (GN2: qnorm). pub normalize: Option, /// Clip each tail at this fraction, e.g. 0.05 for the 5th/95th /// percentile. Mutually exclusive with `winsor_fences`. pub winsor_quantile: Option, /// Clip at Tukey fences: `Q1 - k*IQR` / `Q3 + k*IQR` (k defaults to /// 1.5 when this is set to 1.5). Mutually exclusive with /// `winsor_quantile`. pub winsor_fences: Option, } impl Params { /// The pipeline these fields describe. pub fn pipeline(&self) -> Result { Pipeline::parse( self.normalize.as_deref(), self.winsor_quantile, self.winsor_fences, ) } } /// How the trait's scale is changed. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Method { /// As posted. None, Log2, Log10, /// Square root; no offset added (see the module comment). Sqrt, /// `-x`. GeneNetwork2 calls this "Invert +/-"; it is a negation, not a /// reciprocal. Negate, /// `(x - mean) / sd`, sample sd (`n-1`). Zscore, /// `(x - median) / (1.4826 * MAD)` -- resistant to the outliers /// winsorising exists for. Ours; GN2 has no robust standardisation. RobustZscore, /// Blom scores: `Phi^-1((i - 0.5) / n)` of the rank `i`. GN2's /// `qnorm`/“Quantile”. Rank, } impl Method { /// The wire name, also the value a caller sends. pub fn as_str(self) -> &'static str { match self { Method::None => "none", Method::Log2 => "log2", Method::Log10 => "log10", Method::Sqrt => "sqrt", Method::Negate => "negate", Method::Zscore => "zscore", Method::RobustZscore => "robust_zscore", Method::Rank => "rank", } } /// Parse a caller's value, accepting GN2's two names as aliases. pub fn parse(value: &str) -> Result { match value.trim().to_ascii_lowercase().as_str() { "" | "none" | "raw" => Ok(Method::None), "log2" => Ok(Method::Log2), "log10" => Ok(Method::Log10), "sqrt" => Ok(Method::Sqrt), // GN2's "Invert +/-" is a negation. "negate" | "invert" => Ok(Method::Negate), "zscore" | "z_score" | "z-score" => Ok(Method::Zscore), "robust_zscore" | "robust-zscore" | "robust_z" => Ok(Method::RobustZscore), // GN2's `norm_method=qnorm`, labelled "Quantile". "rank" | "qnorm" | "quantile" => Ok(Method::Rank), other => Err(format!( "'{other}' is not a normalisation; use one of none, log2, log10, sqrt, \ negate (GN2: invert), zscore, robust_zscore, rank (GN2: qnorm)" )), } } fn is_identity(self) -> bool { self == Method::None } } impl fmt::Display for Method { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.as_str()) } } /// How the tails are clipped (winsorising). #[derive(Debug, Clone, Copy, PartialEq)] pub enum Winsor { /// Clip each tail at this fraction, e.g. 0.05 = the 5th/95th percentile. Quantile(f64), /// Clip at `Q1 - k*IQR` / `Q3 + k*IQR`, the rule GN2 flags outliers with. Fences(f64), } impl Winsor { fn as_str(self) -> String { match self { Winsor::Quantile(p) => format!("winsorized at the {}% tails", trim_percent(p)), Winsor::Fences(k) => format!("winsorized at Tukey fences ({k} x IQR)"), } } } fn trim_percent(fraction: f64) -> String { let percent = fraction * 100.0; if (percent - percent.round()).abs() < 1e-9 { format!("{}", percent.round()) } else { format!("{percent:.1}") } } /// What a caller asked for. #[derive(Debug, Clone, Copy, PartialEq)] pub struct Pipeline { pub method: Method, pub winsor: Option, } /// What applying it actually did, for the response's `notes`. #[derive(Debug, Clone, PartialEq)] pub struct Applied { /// How many values the pipeline saw (present, not missing). pub values: usize, /// Values clipped, low and high tail, when winsorising. pub clipped: Option<(usize, usize)>, /// Tukey-fence outliers in the values *as given* -- GN2's warning, /// which this API previously said nothing about. pub outliers: usize, /// `mean`/`sd` for `zscore`, `median`/`mad` for `robust_zscore`. pub centre: Option, pub spread: Option, } impl Pipeline { /// The identity pipeline: no transform, no clipping. pub const NONE: Pipeline = Pipeline { method: Method::None, winsor: None, }; /// Read the three request fields, refusing combinations that cannot /// both apply. pub fn parse( normalize: Option<&str>, winsor_quantile: Option, winsor_fences: Option, ) -> Result { let method = match normalize { Some(value) => Method::parse(value)?, None => Method::None, }; let winsor = match (winsor_quantile, winsor_fences) { (Some(_), Some(_)) => { return Err("give either winsor_quantile or winsor_fences, not both: a \\ quantile clip and a fence clip are different rules" .to_string()) } (Some(p), None) => { if !(p > 0.0 && p < 0.5) { return Err(format!( "winsor_quantile must be between 0 and 0.5 (exclusive), not {p}" )); } Some(Winsor::Quantile(p)) } (None, Some(k)) => { if !(k > 0.0) { return Err(format!("winsor_fences must be positive, not {k}")); } Some(Winsor::Fences(k)) } (None, None) => None, }; Ok(Pipeline { method, winsor }) } /// The same fields, for a suggested call's body: omitted entirely when /// the pipeline is the identity, so an untransformed suggestion looks /// exactly as it did before this existed. pub fn to_json(&self) -> serde_json::Map { let mut fields = serde_json::Map::new(); if self.method != Method::None { fields.insert( "normalize".to_string(), serde_json::json!(self.method.as_str()), ); } match self.winsor { Some(Winsor::Quantile(p)) => { fields.insert("winsor_quantile".to_string(), serde_json::json!(p)); } Some(Winsor::Fences(k)) => { fields.insert("winsor_fences".to_string(), serde_json::json!(k)); } None => {} } fields } /// Whether this pipeline would change anything. pub fn is_identity(&self) -> bool { self.method.is_identity() && self.winsor.is_none() } /// One line for `notes`: what ran, **in the order it ran**, over how /// many values, and what it did. /// /// The order matters and is easy to get wrong in prose: the monotone /// transform comes first, winning over the tails second, and the /// standardisation (`zscore`, `robust_zscore`, `rank`) last -- so /// `normalize=rank&winsor_fences=1.5` is "winsorised, then ranked", not /// the other way round. pub fn describe(&self, applied: &Applied) -> String { if self.is_identity() { return format!( "Trait values are as posted (no normalization): {} values{}", applied.values, outlier_clause(applied.outliers, applied.values) ); } let is_standardisation = matches!( self.method, Method::Zscore | Method::RobustZscore | Method::Rank ); let mut steps: Vec = Vec::new(); if !self.method.is_identity() && !is_standardisation { steps.push(self.method.to_string()); } if let Some(winsor) = self.winsor { steps.push(winsor.as_str()); } if is_standardisation { steps.push(self.method.to_string()); } let order = if steps.len() > 1 { " (in that order)" } else { "" }; let mut line = format!( "Normalized: {}{order} over {} values", steps.join(", then "), applied.values ); if let Some((low, high)) = applied.clipped { line.push_str(&format!( "; {} value{} clipped ({} low, {} high)", low + high, if low + high == 1 { "" } else { "s" }, low, high )); } if let (Some(centre), Some(spread)) = (applied.centre, applied.spread) { match self.method { Method::Zscore => line.push_str(&format!( "; standardised with the input's mean {centre:.6} and population sd {spread:.6}" )), Method::RobustZscore => line.push_str(&format!( "; standardised with the input's median {centre:.6} and MAD {spread:.6}" )), _ => line.push_str(&format!( "; centre {centre:.6}, spread {spread:.6}" )), } } line.push_str(&outlier_clause(applied.outliers, applied.values)); line } /// Apply the pipeline: transform, then winsorise, then standardise. pub fn apply(&self, values: &[Option]) -> Result<(Vec>, Applied), String> { let observed: Vec = values.iter().filter_map(|value| *value).collect(); let mut applied = Applied { values: observed.len(), clipped: None, outliers: fence_outliers(&observed), centre: None, spread: None, }; if self.is_identity() || observed.is_empty() { return Ok((values.to_vec(), applied)); } // 1. The monotone transform. let mut working: Vec> = Vec::with_capacity(values.len()); for value in values { working.push(match value { Some(x) => Some(self.transform(*x)?), None => None, }); } // 2. Winsorising, on the scale just produced. if let Some(winsor) = self.winsor { let present: Vec = working.iter().filter_map(|value| *value).collect(); if present.len() < 4 { return Err(format!( "winsorising needs at least 4 values, and only {} are present", present.len() )); } let (low, high) = match winsor { Winsor::Quantile(p) => ( percentile(&present, p), percentile(&present, 1.0 - p), ), Winsor::Fences(k) => { let q1 = percentile(&present, 0.25); let q3 = percentile(&present, 0.75); let step = k * (q3 - q1); (q1 - step, q3 + step) } }; let mut clipped_low = 0; let mut clipped_high = 0; for value in working.iter_mut().flatten() { if *value < low { *value = low; clipped_low += 1; } else if *value > high { *value = high; clipped_high += 1; } } applied.clipped = Some((clipped_low, clipped_high)); } // 3. Standardisation, over the present values. let present: Vec = working.iter().filter_map(|value| *value).collect(); match self.method { Method::Zscore => { let (mean, sd) = mean_sd(&present)?; applied.centre = Some(mean); applied.spread = Some(sd); for value in working.iter_mut().flatten() { *value = (*value - mean) / sd; } } Method::RobustZscore => { let centre = median(&present); let mad = median( &present .iter() .map(|value| (value - centre).abs()) .collect::>(), ); let spread = mad * 1.4826; if spread == 0.0 { return Err(format!( "robust_zscore: the median absolute deviation is 0 over {} values, so \ there is nothing to divide by", present.len() )); } applied.centre = Some(centre); applied.spread = Some(spread); for value in working.iter_mut().flatten() { *value = (*value - centre) / spread; } } Method::Rank => { // Score the values by their rank, mapping back through the // sorted *index* order so ties share the average rank (as // scipy's default rankdata does) and nothing depends on // floating-point equality lookups. let mut order: Vec = (0..working.len()) .filter(|index| working[*index].is_some()) .collect(); order.sort_by(|left, right| { working[*left] .partial_cmp(&working[*right]) .unwrap_or(std::cmp::Ordering::Equal) }); let n = order.len() as f64; let mut index = 0; while index < order.len() { let mut end = index; while end + 1 < order.len() && working[order[end + 1]] == working[order[index]] { end += 1; } let average_rank = ((index + 1) as f64 + (end + 1) as f64) / 2.0; let score = inverse_normal_cdf((average_rank - 0.5) / n); for slot in index..=end { working[order[slot]] = Some(score); } index = end + 1; } } _ => {} } Ok((working, applied)) } fn transform(&self, value: f64) -> Result { match self.method { Method::None | Method::Zscore | Method::RobustZscore | Method::Rank => Ok(value), Method::Log2 => { if value <= 0.0 { return Err(format!( "log2 needs positive values, and the trait has {value}; use sqrt or \ none, or shift the values before posting" )); } Ok(value.log2()) } Method::Log10 => { if value <= 0.0 { return Err(format!( "log10 needs positive values, and the trait has {value}; use sqrt or \ none, or shift the values before posting" )); } Ok(value.log10()) } Method::Sqrt => { if value < 0.0 { return Err(format!( "sqrt needs non-negative values, and the trait has {value}. GeneNetwork2 \ silently adds 1 in that case; this API does not -- a silent offset \ changes every effect size, so shift the values yourself if that is what \ you want" )); } Ok(value.sqrt()) } Method::Negate => Ok(-value), } } } fn outlier_clause(outliers: usize, values: usize) -> String { if outliers == 0 || values == 0 { return String::new(); } format!( "; {outliers} of {values} values lie outside Tukey fences (Q1-1.5*IQR, Q3+1.5*IQR) -- \ GeneNetwork2 warns about these before mapping" ) } /// Tukey fences, the outlier rule GeneNetwork2 flags with /// (`gn2/utility/Plot.py:find_outliers`). fn fence_bounds(values: &[f64]) -> Option<(f64, f64)> { if values.len() < 4 { return None; } let q1 = percentile(values, 0.25); let q3 = percentile(values, 0.75); let step = 1.5 * (q3 - q1); Some((q1 - step, q3 + step)) } fn fence_outliers(values: &[f64]) -> usize { match fence_bounds(values) { Some((low, high)) => values.iter().filter(|v| **v < low || **v > high).count(), None => 0, } } /// Linear-interpolated percentile (numpy's default), on a copy. fn percentile(values: &[f64], fraction: f64) -> f64 { let mut sorted = values.to_vec(); sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); let position = fraction * (sorted.len() as f64 - 1.0); let low = position.floor() as usize; let high = position.ceil() as usize; if low == high { return sorted[low]; } let weight = position - low as f64; sorted[low] * (1.0 - weight) + sorted[high] * weight } fn median(values: &[f64]) -> f64 { percentile(values, 0.5) } /// Mean and **population** standard deviation (`n`, not `n-1`): that is what /// GeneNetwork2 uses (`scipy.stats.mstats.zscore`, whose default is /// `ddof=0`), and matching it is the point of offering this transform. With /// the sample sd instead, every z-score is `sqrt((n-1)/n)` times GN2's -- /// 1.8% low at n = 28, measured against genenetwork.org's own page for /// trait 10010. fn mean_sd(values: &[f64]) -> Result<(f64, f64), String> { if values.len() < 2 { return Err(format!( "zscore needs at least 2 values, and only {} are present", values.len() )); } let mean = values.iter().sum::() / values.len() as f64; let variance = values.iter().map(|v| (v - mean).powi(2)).sum::() / values.len() as f64; let sd = variance.sqrt(); if sd == 0.0 { return Err(format!( "zscore: every one of the {} values is the same, so there is nothing to divide by", values.len() )); } Ok((mean, sd)) } /// `Phi^-1(p)` for `0 < p < 1`: Acklam's rational approximation /// (|error| < 1.15e-9), which is far more than trait values reported to six /// decimals need. /// /// Written out rather than clever: the two tail branches need a different /// `q` (`sqrt(-2 ln p)`) from the central one (`p - 0.5`), and the /// denominators end in `+ 1`, which is exactly where a tidy one-liner /// folding the coefficients goes wrong. fn inverse_normal_cdf(p: f64) -> f64 { // Central branch. const A: [f64; 6] = [ -3.969_683_028_665_376e1, 2.209_460_984_245_205e2, -2.759_285_104_469_687e2, 1.383_577_518_672_69e2, -3.066_479_806_614_716e1, 2.506_628_277_459_239, ]; const B: [f64; 5] = [ -5.447_609_879_822_406e1, 1.615_858_368_580_409e2, -1.556_989_798_598_866e2, 6.680_131_188_771_972e1, -1.328_068_155_288_572e1, ]; // Tail branches. const C: [f64; 6] = [ -7.784_894_002_430_293e-3, -3.223_964_580_411_365e-1, -2.400_758_277_161_838, -2.549_732_539_343_734, 4.374_664_141_464_968, 2.938_163_982_698_783, ]; const D: [f64; 4] = [ 7.784_695_709_041_462e-3, 3.224_671_290_700_398e-1, 2.445_134_137_142_996, 3.754_408_661_907_416, ]; const P_LOW: f64 = 0.024_25; const P_HIGH: f64 = 1.0 - P_LOW; if !(p > 0.0 && p < 1.0) { return if p <= 0.0 { f64::NEG_INFINITY } else { f64::INFINITY }; } if p < P_LOW { let q = (-2.0 * p.ln()).sqrt(); let num = C.iter().fold(0.0, |acc, c| acc * q + c); let den = D.iter().fold(0.0, |acc, d| acc * q + d) * q + 1.0; // The coefficients already carry the sign for the lower tail. return num / den; } if p > P_HIGH { let q = (-2.0 * (1.0 - p).ln()).sqrt(); let num = C.iter().fold(0.0, |acc, c| acc * q + c); let den = D.iter().fold(0.0, |acc, d| acc * q + d) * q + 1.0; return -(num / den); } let q = p - 0.5; let r = q * q; let num = A.iter().fold(0.0, |acc, a| acc * r + a) * q; let den = B.iter().fold(0.0, |acc, b| acc * r + b) * r + 1.0; num / den } #[cfg(test)] mod tests { use super::*; fn values(raw: &[f64]) -> Vec> { raw.iter().map(|value| Some(*value)).collect() } #[test] fn the_inverse_normal_cdf_matches_known_quantiles() { // The values a table would give, to six places. for (p, expected) in [ (0.025, -1.959_964), (0.05, -1.644_854), (0.5, 0.0), (0.95, 1.644_854), (0.975, 1.959_964), (0.982_142_857, 2.100_165), // (n-0.5)/n for n = 28, the largest rank ] { let got = inverse_normal_cdf(p); assert!( (got - expected).abs() < 1e-4, "Phi^-1({p}) = {got}, expected {expected}" ); } } #[test] fn rank_gives_blom_scores_to_the_largest_and_smallest() { let raw: Vec = (1..=28).map(|value| value as f64).collect(); let (out, applied) = Pipeline { method: Method::Rank, winsor: None, } .apply(&values(&raw)) .unwrap(); assert_eq!(applied.values, 28); // max: (28 - 0.5)/28 = 0.982..., min: (1 - 0.5)/28 assert!((out[27].unwrap() - 2.100_165).abs() < 1e-4, "{:?}", out[27]); // min: (1 - 0.5)/28 = 0.017857..., Phi^-1 = -2.100165 assert!((out[0].unwrap() + 2.100_165).abs() < 1e-4, "{:?}", out[0]); // monotone, and the middle of an even count straddles zero assert!(out.windows(2).all(|pair| pair[0].unwrap() < pair[1].unwrap())); } #[test] fn ties_share_their_average_rank() { let out = Pipeline { method: Method::Rank, winsor: None, } .apply(&values(&[5.0, 5.0, 5.0, 9.0])) .unwrap() .0; assert!((out[0].unwrap() - out[1].unwrap()).abs() < 1e-12); assert!((out[0].unwrap() - out[2].unwrap()).abs() < 1e-12); assert!(out[3].unwrap() > out[0].unwrap()); } #[test] fn zscore_is_mean_zero_and_sample_sd_one() { let (out, applied) = Pipeline { method: Method::Zscore, winsor: None, } .apply(&values(&[2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0])) .unwrap(); let mean = out.iter().map(|v| v.unwrap()).sum::() / 8.0; assert!(mean.abs() < 1e-12); assert!((applied.centre.unwrap() - 5.0).abs() < 1e-12); // population sd: sqrt(32/8) = 2.0 (GN2 / scipy mstats default) assert!((applied.spread.unwrap() - 2.0).abs() < 1e-12, "{:?}", applied.spread); } #[test] fn robust_zscore_uses_the_median_and_mad() { let (_, applied) = Pipeline { method: Method::RobustZscore, winsor: None, } .apply(&values(&[1.0, 2.0, 3.0, 4.0, 100.0])) .unwrap(); assert_eq!(applied.centre, Some(3.0)); // MAD = median(|x - 3|) = 1 -> 1.4826 assert!((applied.spread.unwrap() - 1.4826).abs() < 1e-9); } #[test] fn log_and_sqrt_refuse_values_they_cannot_take_and_never_offset() { let error = Pipeline { method: Method::Log2, winsor: None, } .apply(&values(&[1.0, 0.0])) .unwrap_err(); assert!(error.contains("log2 needs positive"), "{error}"); let error = Pipeline { method: Method::Sqrt, winsor: None, } .apply(&values(&[1.0, -1.0])) .unwrap_err(); // GN2's silent +1 is deliberately not reproduced. assert!(error.contains("does not"), "{error}"); // ... while a legitimate sqrt is exactly the sqrt. let (out, _) = Pipeline { method: Method::Sqrt, winsor: None, } .apply(&values(&[4.0, 9.0])) .unwrap(); assert_eq!(out, values(&[2.0, 3.0])); } #[test] fn missing_values_stay_missing_through_every_step() { let raw = vec![Some(1.0), None, Some(2.0), Some(3.0), Some(4.0), Some(100.0)]; let (out, _) = Pipeline { method: Method::Zscore, winsor: Some(Winsor::Quantile(0.2)), } .apply(&raw) .unwrap(); assert!(out[1].is_none(), "the gap is never imputed"); // The 100 is clipped to the 80th percentile (on the raw scale, before // standardising), so it is no longer 100 standard deviations out. assert!(out[5].unwrap() < 4.0, "{:?}", out[5]); } #[test] fn winsorising_clips_both_tails_and_counts_what_it_clipped() { let raw: Vec = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]; let (out, applied) = Pipeline { method: Method::None, winsor: Some(Winsor::Quantile(0.1)), } .apply(&values(&raw)) .unwrap(); // 10th/90th percentiles of 1..10, linearly interpolated: 1.9 and 9.1. assert!((out[0].unwrap() - 1.9).abs() < 1e-9, "{:?}", out[0]); assert!((out[9].unwrap() - 9.1).abs() < 1e-9, "{:?}", out[9]); assert_eq!(applied.clipped, Some((1, 1))); } #[test] fn fences_clip_at_one_and_a_half_iqr_and_report_gn2_outliers() { // The doc example from gn2/utility/Plot.py:find_outliers. let raw = vec![3.504, 5.234, 6.123, 7.234, 3.542, 5.341, 7.852, 4.555, 12.537]; let (_, applied) = Pipeline { method: Method::None, winsor: Some(Winsor::Fences(1.5)), } .apply(&values(&raw)) .unwrap(); // GN2's bounds for this sample are (11.2525, 0.5365): one value above. assert_eq!(applied.outliers, 1); assert_eq!(applied.clipped, Some((0, 1))); // Without winsorising, the identity pipeline still reports them -- // the warning GN2 gives before mapping, which we now give too. let (_, applied) = Pipeline::NONE.apply(&values(&raw)).unwrap(); assert_eq!(applied.outliers, 1); let note = Pipeline::NONE.describe(&applied); assert!(note.contains("outside Tukey fences"), "{note}"); } #[test] fn the_pipeline_fields_are_parsed_and_refused_sensibly() { assert_eq!( Pipeline::parse(None, None, None).unwrap(), Pipeline::NONE ); // GN2's names are accepted. assert_eq!(Pipeline::parse(Some("qnorm"), None, None).unwrap().method, Method::Rank); assert_eq!(Pipeline::parse(Some("invert"), None, None).unwrap().method, Method::Negate); // Mutually exclusive clipping rules. let error = Pipeline::parse(None, Some(0.05), Some(1.5)).unwrap_err(); assert!(error.contains("not both"), "{error}"); let error = Pipeline::parse(None, Some(0.6), None).unwrap_err(); assert!(error.contains("between 0 and 0.5"), "{error}"); let error = Pipeline::parse(Some("median"), None, None).unwrap_err(); assert!(error.contains("is not a normalisation"), "{error}"); // A pipeline that changes nothing carries no fields into a suggestion. assert!(Pipeline::NONE.to_json().is_empty()); let fields = Pipeline::parse(Some("zscore"), Some(0.05), None).unwrap().to_json(); assert_eq!(fields["normalize"], "zscore"); assert_eq!(fields["winsor_quantile"], 0.05); } #[test] fn the_note_says_what_ran_in_which_order() { let (_, applied) = Pipeline { method: Method::Zscore, winsor: Some(Winsor::Fences(1.5)), } .apply(&values(&[1.0, 2.0, 3.0, 4.0, 5.0, 100.0])) .unwrap(); let note = Pipeline { method: Method::Zscore, winsor: Some(Winsor::Fences(1.5)), } .describe(&applied); assert!( note.contains("winsorized at Tukey fences (1.5 x IQR), then zscore (in that order)"), "the note states the order that ran: {note}" ); assert!(note.contains("clipped"), "{note}"); assert!(note.contains("population sd"), "{note}"); } }