about summary refs log tree commit diff
path: root/src/gemma_io.cpp
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-11-18 11:30:59 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-11-18 11:30:59 +0000
commitf026a7b808fd2d61e64d4d53c540fd66fffcff81 (patch)
tree0b5ca159775cba3a6d7a847d1c03cc6e22a061f7 /src/gemma_io.cpp
parentcebca3b5cc041731df65bb5008fd3e7d5202c299 (diff)
downloadpangemma-f026a7b808fd2d61e64d4d53c540fd66fffcff81.tar.gz
mdb: apply the marker filters, count missing, and give K the same marker set
An lmdb scan and a BIMBAM scan of the same data were not answering the same
question. Measured on HLC (358454 markers, 23 chromosomes, gemma-wrapper's
LOCO recipe), before this commit:

  -maf 0.05  gemma 20246 markers on chr1, pangemma 27940 (every marker in the
             store): mdb_calc_gwa computed maf and put it in MarkerInfo but
             nothing ever compared it with maf_level, and -miss was ignored too
  af/n_miss  nan / 0 for every marker carrying a missing call, because
             compute_maf was called with n_miss hard-wired to 0 ("count NAs:
             FIXME") and summed NaNs into the frequency
  Gb stores  255 is the missing sentinel; read as 255/127.0 it became 2.0079, a
             valid-looking genotype that silently poisoned af and the test
  K           mdb_calc_kin had no missing handling at all (its mask marked every
             individual, so the imputation branch never fired) and applied no
             filters, so HLC's K was built from 330514 markers where GEMMA uses
             273349 -- GEMMA's K reader skips whatever its genotype reader
             rejected (indicator_snp[i] == 0) -- and every p-value moved
  log line   mdb_calc_kin never reported the count, so "number of analyzed
             SNPs/var" printed an uninitialised ns_test (140048413575759)

All four readers of gemma_io.cpp apply the same five filters in a fixed order;
the lmdb path now does the same, in the same order, with GEMMA's own allele
bins: -miss, -maf (maf_level == -1 disables), monomorphic (every observed
genotype equal), -hwe, and -r2 against the covariates (missing imputed with
maf*2 as the readers do, W'W inverted once per scan).  Missing calls arrive as
NaN in both formats, compute_maf counts what it sees, and the K path takes the
same filtered marker set and imputes the observed mean exactly as
gemma_io.cpp's ReadFile_geno does for K.

Verified, HLC LOCO on the slurm cluster (23 tasks, 1 CPU each, all 23 RUNNING
together), pangemma lmdb (Gb, missing kept) against gemma 0.98.6 on BIMBAM:
same row count on all 23 chromosomes (260251 rows), same top SNP on 23/23 (was
12/23), and no differing numeric cell in beta, se, l_mle or p_lrt (was 260251
rows differing with l_mle pinned at the 1e5 bound).  The BIMBAM route is
unchanged: 23/23 and 0 differing cells, as before.  Cost of the filtering:
job time for the lmdb route fell from 179 s to 141 s for the 23 chromosomes
(gemma BIMBAM: 958 s), because 30 percent of the markers are no longer
analysed.

On BXD.8 (20453 markers, 198 individuals, 1460 missing calls) the Gf store now
reproduces gemma's output column for column -- af and n_miss included -- with
the only differences being 33 `ps` values, where GEMMA's annotation reader
truncates R's "1.76e+08" to 1 and the store has the rounded position.
Diffstat (limited to 'src/gemma_io.cpp')
-rw-r--r--src/gemma_io.cpp68
1 files changed, 61 insertions, 7 deletions
diff --git a/src/gemma_io.cpp b/src/gemma_io.cpp
index e630f64..11e2746 100644
--- a/src/gemma_io.cpp
+++ b/src/gemma_io.cpp
@@ -1478,7 +1478,9 @@ void ReadFile_eigenD(const string &file_kd, bool &error, gsl_vector *eval) {
 
 
 // Read bimbam mean genotype file and calculate kinship matrix.
-gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco) {
+gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco,
+                          size_t &n_markers_used, const double maf_level,
+                          const double miss_level) {
   checkpoint("mdb-kin",file_geno);
   bool is_loco = !loco.empty();
 
@@ -1589,7 +1591,11 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string
       const uint8_t* gs_bbuf = reinterpret_cast<const uint8_t*>(value.data());
       gs.reserve(size);
       for (size_t i = 0; i < size; ++i) {
-        double g = static_cast<double>(gs_bbuf[i])/127.0;
+        // 255 is the Gb missing sentinel; read as 255/127.0 it would be 2.0079,
+        // a real genotype, and every missing call would silently enter the GRM.
+        double g = (gs_bbuf[i] == 255)
+                     ? std::numeric_limits<double>::quiet_NaN()
+                     : static_cast<double>(gs_bbuf[i])/127.0;
         gs.push_back(g);
       }
     } else {
@@ -1603,27 +1609,74 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string
       }
     }
 
+    // Missing genotypes: the mask used to be set to 1 for every individual (so
+    // the imputation below never fired) and n_miss was never counted, which is
+    // why a store with missing calls produced a GRM that looked fine and was not
+    // the one GEMMA computes. Now: NaN is missing, the mean and variance are
+    // taken over the observed calls, and the missing ones are plugged with the
+    // mean, exactly as gemma_io.cpp's ReadFile_geno does for K.
+    // The same marker filters the scan applies (see mdb_calc_gwa): GEMMA's K
+    // reader skips every marker its genotype reader rejected
+    // (indicator_snp[i] == 0, gemma_io.cpp:1775), so K and GWA must see one
+    // marker set.  Without this, HLC's K was built from 330514 markers where
+    // GEMMA uses 273349, and every p-value in the scan moved.
+    {
+      size_t n_miss0 = 0, observed0 = 0;
+      double total0 = 0.0, first0 = 0.0;
+      bool first_seen = false, poly = false;
+      for (size_t i = 0; i < ni_total; ++i) {
+        double g = gs[i];
+        if (std::isnan(g)) {
+          n_miss0++;
+          continue;
+        }
+        if (!first_seen) {
+          first0 = g;
+          first_seen = true;
+        } else if (g != first0) {
+          poly = true;
+        }
+        total0 += g;
+        observed0++;
+      }
+      double maf0 = observed0 ? total0 / (2.0 * (double)observed0) : 0.0;
+      if ((double)n_miss0 / (double)ni_total > miss_level)
+        continue;
+      if ((maf0 < maf_level || maf0 > (1.0 - maf_level)) && maf_level != -1)
+        continue;
+      if (!poly)
+        continue;
+    }
+
     size_t n_miss = 0;
-    double geno_mean = 0.0;
+    double geno_sum = 0.0;
     double geno_var = 0.0;
     gsl_vector_set_all(geno_miss, 0);
 
     for (size_t i = 0; i < ni_total; ++i) {
       double d = gs[i];
+      if (std::isnan(d)) {
+        gsl_vector_set(geno, i, 0.0); // placeholder, patched below
+        gsl_vector_set(geno_miss, i, 1);
+        n_miss++;
+        continue;
+      }
       gsl_vector_set(geno, i, d);
-      gsl_vector_set(geno_miss, i, 1);
-      geno_mean += d;
+      geno_sum += d;
       geno_var += d * d;
     }
 
-    geno_mean /= (double)(ni_total - n_miss);
+    const size_t observed = ni_total - n_miss;
+    const double geno_mean = observed ? geno_sum / (double)observed : 0.0;
+    // Variance over all individuals with the missing ones at the mean, as GEMMA
+    // does: sum(d^2) over observed + n_miss*mean^2, divided by ni_total.
     geno_var += geno_mean * geno_mean * (double)n_miss;
     geno_var /= (double)ni_total;
     geno_var -= geno_mean * geno_mean;
 
     // impute missing values by plugging in the mean
     for (size_t i = 0; i < ni_total; ++i) {
-      if (gsl_vector_get(geno_miss, i) == 0) {
+      if (gsl_vector_get(geno_miss, i) == 1) {
         gsl_vector_set(geno, i, geno_mean);
       }
     }
@@ -1664,6 +1717,7 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string
   gsl_vector_free(geno_miss);
   gsl_matrix_free(Xlarge);
 
+  n_markers_used = ns_test;
   return matrix_kin;
 }