about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-11-09 14:01:00 +0000
committerPjotr <wrk@bosa.lan>2026-11-09 14:01:00 +0000
commita698c48ee204cd68efc479b01132ae9c037213bb (patch)
tree9af621bc16457d1ea99ddb4ffa7a28588e5316a7 /src
parent195ee5d285da98bc911813d8e578f4f17e9ba7ba (diff)
downloadgenenetwork4-a698c48ee204cd68efc479b01132ae9c037213bb.tar.gz
covariates: refuse a mostly-numeric attribute, and teach the skill to use them
Fitting `SeqCvge` as a covariate came back as `502 gemma_failed` with a
message about LU decomposition: its values are depths (`40`, `41`) with a
few two-library entries (`36+39L`), so the attribute was coded as **twenty
categories** and GEMMA died on the resulting dummies. The failure named
GEMMA, which is the wrong place to look.

`code_case_attribute` now refuses that shape at the covariate step, naming
the values and what to send instead:

  422 invalid_covariate: case attribute 'SeqCvge': the attribute mixes
  numbers with number-like text (14 of 19 values look numeric, but e.g.
  '36+30L' does not), so coding it as categories would fit a quantity as a
  factor -- and GEMMA fails on the result. Pass an explicit `map` giving
  each of those a number (there are 5 such value(s): ...), or send the
  numbers inline.

The rule is deliberately narrow: an odd level with *no* digits (`1, 2,
unknown`) is still a level and still coded, so nothing that worked stops
working; only dirty numbers are caught.

Also, the skill's covariate coverage -- which the review above found thin:

* when a covariate is worth trying (metadata that could confound: sex, age,
  epoch, batch, sequencing depth), and the covariate step in the session
  outline;
* read `.data.descriptions` before choosing one -- they are the only place
  that says `SeqCvge` is sequencing depth and `Epoch` a family subgroup --
  and the endpoint table now says so;
* read the covariate line in `notes` for the coding and the complete-case
  n, and report that n beside any LOD/p change; plus `Cyro`/`Cryo` being
  two levels in BXD;
* the categorical-vs-numeric rule measured rather than asserted: `RRID`
  refused (82 distinct), `Status`/`Epoch` accepted, and the compound-value
  trap written up with the `map` workaround;
* GEMMA's dummy columns, the column it drops when a level has no fitted
  individual, and that R/qtl's note has no such line; missing values
  dropping individuals; the 8-covariate cap and the `group` requirement.

* src/covariates.rs: the refusal, with a unit test for both sides of it;
* tests/gemma.rs: `SeqCvge` is a 422 naming `map`, not a GEMMA failure;
* docs/www/SKILL.md: the covariate section, the session line, and the
  case-attributes rows in the endpoint table.

300 tests pass; the same four as before do not.
Diffstat (limited to 'src')
-rw-r--r--src/covariates.rs54
1 files changed, 54 insertions, 0 deletions
diff --git a/src/covariates.rs b/src/covariates.rs
index ed30af0..ed6cb63 100644
--- a/src/covariates.rs
+++ b/src/covariates.rs
@@ -179,6 +179,34 @@ pub fn code_case_attribute(
             levels: Vec::new(),
         });
     }
+    // Mostly numbers with a few number-*like* entries -- `SeqCvge`'s
+    // "36+39L" beside "40" -- would otherwise be coded as categories, and
+    // GEMMA then fails on the resulting dummies with a message about its own
+    // LU decomposition rather than about the covariate. Refuse it here and
+    // say what to send instead. Entries with no digits at all ("unknown")
+    // stay categorical: that is a real level, not a dirty number.
+    let numeric = levels.iter().filter(|level| level.parse::<f64>().is_ok()).count();
+    if numeric > 0 && numeric < levels.len() {
+        let dirty: Vec<&str> = levels
+            .iter()
+            .copied()
+            .filter(|level| level.parse::<f64>().is_err() && level.chars().any(|c| c.is_ascii_digit()))
+            .collect();
+        if !dirty.is_empty() {
+            let example = dirty.first().copied().unwrap_or_default();
+            return Err(format!(
+                "the attribute mixes numbers with number-like text ({} of {} values look \
+                 numeric, but e.g. '{example}' does not), so coding it as categories would fit \
+                 a quantity as a factor -- and GEMMA fails on the result. Pass an explicit \
+                 `map` giving each of those a number (there are {} such value(s): {}), or send \
+                 the numbers inline",
+                numeric,
+                levels.len(),
+                dirty.len(),
+                dirty.join(", ")
+            ));
+        }
+    }
     if levels.len() > MAX_AUTO_LEVELS {
         return Err(format!(
             "the attribute has {} distinct values, too many to code automatically (limit \
@@ -800,6 +828,32 @@ mod tests {
     }
 
     #[test]
+    fn a_mostly_numeric_attribute_asks_for_a_map_instead_of_fitting_a_factor() {
+        // SeqCvge in BXD: plain depths beside two-library values.
+        let values = texts(&["40", "36+39L", "41", "40+31L"]);
+        let error = code_case_attribute(&values, None).unwrap_err();
+        assert!(error.contains("mixes numbers with number-like text"), "{error}");
+        assert!(error.contains("36+39L") && error.contains("40+31L"), "{error}");
+        assert!(error.contains("`map`"), "{error}");
+
+        // An explicit map settles it.
+        let mut map = BTreeMap::new();
+        map.insert("40".to_string(), 40.0);
+        map.insert("41".to_string(), 41.0);
+        map.insert("36+39L".to_string(), 75.0);
+        map.insert("40+31L".to_string(), 71.0);
+        let coded = code_case_attribute(&values, Some(&map)).unwrap();
+        assert_eq!(coded.kind, CovariateKind::Categorical);
+        assert_eq!(coded.values[1], Some(75.0));
+
+        // A level that is not a dirty number stays a level.
+        let mixed = texts(&["1", "2", "unknown"]);
+        let coded = code_case_attribute(&mixed, None).unwrap();
+        assert_eq!(coded.kind, CovariateKind::Categorical);
+        assert_eq!(coded.values.len(), 3);
+    }
+
+    #[test]
     fn too_many_levels_asks_for_an_explicit_map() {
         let values: Vec<Option<String>> =
             (0..MAX_AUTO_LEVELS + 1).map(|i| Some(format!("level-{i}"))).collect();