about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/gemma_io.cpp68
-rw-r--r--src/gemma_io.h4
-rw-r--r--src/lmm.cpp120
-rw-r--r--src/lmm.h8
-rw-r--r--src/mathfunc.cpp23
-rw-r--r--src/param.cpp9
6 files changed, 211 insertions, 21 deletions
diff --git a/src/gemma_io.cpp b/src/gemma_io.cpp
index e630f64..11e2746 100644
--- a/src/gemma_io.cpp
+++ b/src/gemma_io.cpp
@@ -1478,7 +1478,9 @@ void ReadFile_eigenD(const string &file_kd, bool &error, gsl_vector *eval) {
 
 
 // Read bimbam mean genotype file and calculate kinship matrix.
-gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco) {
+gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco,
+                          size_t &n_markers_used, const double maf_level,
+                          const double miss_level) {
   checkpoint("mdb-kin",file_geno);
   bool is_loco = !loco.empty();
 
@@ -1589,7 +1591,11 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string
       const uint8_t* gs_bbuf = reinterpret_cast<const uint8_t*>(value.data());
       gs.reserve(size);
       for (size_t i = 0; i < size; ++i) {
-        double g = static_cast<double>(gs_bbuf[i])/127.0;
+        // 255 is the Gb missing sentinel; read as 255/127.0 it would be 2.0079,
+        // a real genotype, and every missing call would silently enter the GRM.
+        double g = (gs_bbuf[i] == 255)
+                     ? std::numeric_limits<double>::quiet_NaN()
+                     : static_cast<double>(gs_bbuf[i])/127.0;
         gs.push_back(g);
       }
     } else {
@@ -1603,27 +1609,74 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string
       }
     }
 
+    // Missing genotypes: the mask used to be set to 1 for every individual (so
+    // the imputation below never fired) and n_miss was never counted, which is
+    // why a store with missing calls produced a GRM that looked fine and was not
+    // the one GEMMA computes. Now: NaN is missing, the mean and variance are
+    // taken over the observed calls, and the missing ones are plugged with the
+    // mean, exactly as gemma_io.cpp's ReadFile_geno does for K.
+    // The same marker filters the scan applies (see mdb_calc_gwa): GEMMA's K
+    // reader skips every marker its genotype reader rejected
+    // (indicator_snp[i] == 0, gemma_io.cpp:1775), so K and GWA must see one
+    // marker set.  Without this, HLC's K was built from 330514 markers where
+    // GEMMA uses 273349, and every p-value in the scan moved.
+    {
+      size_t n_miss0 = 0, observed0 = 0;
+      double total0 = 0.0, first0 = 0.0;
+      bool first_seen = false, poly = false;
+      for (size_t i = 0; i < ni_total; ++i) {
+        double g = gs[i];
+        if (std::isnan(g)) {
+          n_miss0++;
+          continue;
+        }
+        if (!first_seen) {
+          first0 = g;
+          first_seen = true;
+        } else if (g != first0) {
+          poly = true;
+        }
+        total0 += g;
+        observed0++;
+      }
+      double maf0 = observed0 ? total0 / (2.0 * (double)observed0) : 0.0;
+      if ((double)n_miss0 / (double)ni_total > miss_level)
+        continue;
+      if ((maf0 < maf_level || maf0 > (1.0 - maf_level)) && maf_level != -1)
+        continue;
+      if (!poly)
+        continue;
+    }
+
     size_t n_miss = 0;
-    double geno_mean = 0.0;
+    double geno_sum = 0.0;
     double geno_var = 0.0;
     gsl_vector_set_all(geno_miss, 0);
 
     for (size_t i = 0; i < ni_total; ++i) {
       double d = gs[i];
+      if (std::isnan(d)) {
+        gsl_vector_set(geno, i, 0.0); // placeholder, patched below
+        gsl_vector_set(geno_miss, i, 1);
+        n_miss++;
+        continue;
+      }
       gsl_vector_set(geno, i, d);
-      gsl_vector_set(geno_miss, i, 1);
-      geno_mean += d;
+      geno_sum += d;
       geno_var += d * d;
     }
 
-    geno_mean /= (double)(ni_total - n_miss);
+    const size_t observed = ni_total - n_miss;
+    const double geno_mean = observed ? geno_sum / (double)observed : 0.0;
+    // Variance over all individuals with the missing ones at the mean, as GEMMA
+    // does: sum(d^2) over observed + n_miss*mean^2, divided by ni_total.
     geno_var += geno_mean * geno_mean * (double)n_miss;
     geno_var /= (double)ni_total;
     geno_var -= geno_mean * geno_mean;
 
     // impute missing values by plugging in the mean
     for (size_t i = 0; i < ni_total; ++i) {
-      if (gsl_vector_get(geno_miss, i) == 0) {
+      if (gsl_vector_get(geno_miss, i) == 1) {
         gsl_vector_set(geno, i, geno_mean);
       }
     }
@@ -1664,6 +1717,7 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string
   gsl_vector_free(geno_miss);
   gsl_matrix_free(Xlarge);
 
+  n_markers_used = ns_test;
   return matrix_kin;
 }
 
diff --git a/src/gemma_io.h b/src/gemma_io.h
index 06cd0ee..fb58027 100644
--- a/src/gemma_io.h
+++ b/src/gemma_io.h
@@ -92,7 +92,9 @@ void ReadFile_mk(const string &file_mk, vector<int> &indicator_idv,
 void ReadFile_eigenU(const string &file_u, bool &error, gsl_matrix *U);
 void ReadFile_eigenD(const string &file_d, bool &error, gsl_vector *eval);
 
-gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco);
+gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco,
+                          size_t &n_markers_used, const double maf_level,
+                          const double miss_level);
 
 bool BimbamKin(const string file_geno, const set<string> ksnps,
                vector<int> &indicator_snp, const int k_mode,
diff --git a/src/lmm.cpp b/src/lmm.cpp
index 9fe6bbb..9c66cc4 100644
--- a/src/lmm.cpp
+++ b/src/lmm.cpp
@@ -20,6 +20,7 @@
 
 #include <fstream>
 #include <iostream>
+#include <limits>
 #include <sstream>
 
 #include <assert.h>
@@ -70,6 +71,10 @@ void LMM::CopyFromParam(PARAM &cPar) {
 
   l_min = cPar.l_min;
   l_max = cPar.l_max;
+  maf_level = cPar.maf_level;
+  miss_level = cPar.miss_level;
+  hwe_level = cPar.hwe_level;
+  r2_level = cPar.r2_level;
   n_region = cPar.n_region;
   l_mle_null = cPar.l_mle_null;
   logl_mle_H0 = cPar.logl_mle_H0;
@@ -2239,7 +2244,23 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval,
   auto cursor = lmdb::cursor::open(rtxn, geno_mdb);
   cout << "## number of total individuals = " << ni_total << endl;
   cout << "## number of analyzed individuals = " << ni_test << endl;
-  cout << "## number of analyzed SNPs/var = " << num_markers << endl;
+  cout << "## number of total SNPs/var = " << num_markers << endl;
+
+  // -r2 needs (W'W)^-1, which the readers of gemma_io.cpp precompute once per
+  // read; here it is once per scan. W is the covariate matrix (intercept + -c
+  // columns) with ni_test rows, exactly what the readers get.
+  gsl_matrix *WtW = gsl_matrix_safe_alloc(W->size2, W->size2);
+  gsl_matrix *WtWi = gsl_matrix_safe_alloc(W->size2, W->size2);
+  gsl_permutation *pmt = gsl_permutation_alloc(W->size2);
+  gsl_vector *Wtx = gsl_vector_safe_alloc(W->size2);
+  gsl_vector *WtWiWtx = gsl_vector_safe_alloc(W->size2);
+  gsl_vector *x_r2 = gsl_vector_safe_alloc(ni_test);
+  gsl_blas_dgemm(CblasTrans, CblasNoTrans, 1.0, W, W, 0.0, WtW);
+  int sig = 0;
+  LUDecomp(WtW, pmt, &sig);
+  LUInvert(WtW, pmt, WtWi);
+
+  size_t n_analyzed = 0; // markers that survive the filters, for the log line
 
   std::function<SnpNameValues2(size_t)>  fetch_snp = [&](size_t num) {
 
@@ -2262,7 +2283,14 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval,
         const uint8_t* gs_bbuf = reinterpret_cast<const uint8_t*>(value.data());
         gs.reserve(size);
         for (size_t i = 0; i < size; ++i) {
-          double g = static_cast<double>(gs_bbuf[i])/127.0;
+          // 255 is the Gb missing sentinel (gemma-wrapper's geno2mdb and
+          // genenetwork4's make-mdb.sh both write it). Read as 255/127.0 it
+          // becomes 2.0079 -- a valid-looking genotype that silently poisons af
+          // and the analysis -- so it has to arrive as NaN, which is what
+          // mdb_analyze already knows how to drop and mean-substitute.
+          double g = (gs_bbuf[i] == 255)
+                       ? std::numeric_limits<double>::quiet_NaN()
+                       : static_cast<double>(gs_bbuf[i])/127.0;
           gs.push_back(g);
         }
       } else {
@@ -2306,11 +2334,79 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval,
       // 1       rs13476251      174792257
       // cout << static_cast<int>(chr) << ":" << pos2 << " line " << rest2 << ":" << marker << endl ;
 
-      // compute maf and n_miss (NAs)
-      size_t n_miss = 0; // count NAs: FIXME
-      double maf = compute_maf(ni_total, ni_test, n_miss, gs.data(), indicator_idv);
+      // Same marker statistics and the same filters as gemma_io.cpp's
+      // ReadFile_geno/ReadFile_bimbam_geno, in the same order, so that an lmdb
+      // scan answers the question a BIMBAM scan answers.  Before this, every
+      // marker in the store was analysed: -maf 0.05 on HLC still returned all
+      // 27940 markers of chromosome 1 where GEMMA returns 20246, and af/n_miss
+      // came out nan/0 for any marker with a missing call.
+      size_t n_miss = 0, n_0 = 0, n_1 = 0, n_2 = 0, observed = 0;
+      double geno_total = 0.0;
+      bool first = true, polymorphic = false;
+      double geno_first = 0.0;
+      for (size_t i = 0; i < ni_total; ++i) {
+        if (indicator_idv[i] == 0)
+          continue;
+        double geno = gs[i];
+        if (std::isnan(geno)) {
+          n_miss++;
+          continue;
+        }
+        // The allele-count bins are GEMMA's own (gemma_io.cpp:765-773).
+        if (geno >= 0 && geno <= 0.5)
+          n_0++;
+        if (geno > 0.5 && geno < 1.5)
+          n_1++;
+        if (geno >= 1.5 && geno <= 2.0)
+          n_2++;
+        if (first) {
+          geno_first = geno;
+          first = false;
+        } else if (geno != geno_first) {
+          polymorphic = true;
+        }
+        geno_total += geno;
+        observed++;
+      }
 
-      markerinfo = MarkerInfo { .name=marker,.chr=chr,.pos=pos,.line_no=num,.n_miss=n_miss,.maf=maf };
+      double maf = observed ? geno_total / (2.0 * (double)observed) : 0.0;
+      markerinfo = MarkerInfo { .name=marker,.chr=chr,.pos=pos,.line_no=num,
+                                .n_miss=n_miss,.maf=maf };
+
+      // -miss
+      if ((double)n_miss / (double)ni_test > miss_level)
+        return make_tuple(SKIP, markerinfo, gs);
+      // -maf (maf_level == -1 disables it, as in GEMMA)
+      if ((maf < maf_level || maf > (1.0 - maf_level)) && maf_level != -1)
+        return make_tuple(SKIP, markerinfo, gs);
+      // monomorphic: every observed genotype equal
+      if (!polymorphic)
+        return make_tuple(SKIP, markerinfo, gs);
+      // -hwe
+      if (hwe_level != 0 && maf_level != -1) {
+        if (CalcHWE(n_0, n_2, n_1) < hwe_level)
+          return make_tuple(SKIP, markerinfo, gs);
+      }
+      // -r2: correlation with the covariates, computed on the mean-substituted
+      // genotypes exactly as the readers do (missing -> maf*2).
+      if (W->size2 != 1) {
+        size_t pos2 = 0;
+        for (size_t i = 0; i < ni_total; ++i) {
+          if (indicator_idv[i] == 0)
+            continue;
+          double geno = gs[i];
+          gsl_vector_set(x_r2, pos2, std::isnan(geno) ? maf * 2.0 : geno);
+          pos2++;
+        }
+        double v_x = 0.0, v_w = 0.0;
+        gsl_blas_dgemv(CblasTrans, 1.0, W, x_r2, 0.0, Wtx);
+        gsl_blas_dgemv(CblasNoTrans, 1.0, WtWi, Wtx, 0.0, WtWiWtx);
+        gsl_blas_ddot(x_r2, x_r2, &v_x);
+        gsl_blas_ddot(Wtx, WtWiWtx, &v_w);
+        if (v_x > 0.0 && v_w / v_x > r2_level)
+          return make_tuple(SKIP, markerinfo, gs);
+      }
+      n_analyzed++;
 
       // cout << "!!!!" << size << marker << ": af" << maf << " " << gs[0] << "," << gs[1] << "," << gs[2] << "," << gs[3] << endl;
     }
@@ -2318,7 +2414,17 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval,
   };
   LMM::mdb_analyze(fetch_snp,U,eval,UtW,Uty,W,y,num_markers);
 
-  ns_total = ns_test = num_markers; // track global number of snps in original gemma (goes to cPar)
+  gsl_matrix_free(WtW);
+  gsl_matrix_free(WtWi);
+  gsl_permutation_free(pmt);
+  gsl_vector_free(Wtx);
+  gsl_vector_free(WtWiWtx);
+  gsl_vector_free(x_r2);
+
+  cout << "## number of analyzed SNPs/var = " << n_analyzed << endl;
+
+  ns_total = num_markers;
+  ns_test = n_analyzed; // what actually reaches the output (goes to cPar)
 }
 
 
diff --git a/src/lmm.h b/src/lmm.h
index 295602a..86b16c4 100644
--- a/src/lmm.h
+++ b/src/lmm.h
@@ -124,6 +124,14 @@ public:
   // Not included in PARAM.
   vector<SUMSTAT> sumStat; // Output SNPSummary Data.
 
+  // The marker filters, copied from PARAM by CopyFromParam so that the LMDB
+  // path can apply the same ones the BIMBAM and PLINK readers do (it used to
+  // ignore them entirely: every marker in the store was analysed).
+  double maf_level = 0.0;
+  double miss_level = 0.0;
+  double hwe_level = 0.0;
+  double r2_level = 1.0;
+
   // Main functions.
   void CopyFromParam(PARAM &cPar);
   void CopyToParam(PARAM &cPar);
diff --git a/src/mathfunc.cpp b/src/mathfunc.cpp
index c82872e..ad29982 100644
--- a/src/mathfunc.cpp
+++ b/src/mathfunc.cpp
@@ -667,14 +667,27 @@ std::tuple<size_t, size_t, size_t> compute_ratio(size_t ni_total, const gsl_vect
 }
 
 double compute_maf(size_t ni_total, size_t ni_test, size_t n_miss, const double *gs, const vector<int> &indicator) {
-  double maf = 0.0;
-
+  // Missing genotypes arrive here as NaN (Gf stores) or as the 255 sentinel that
+  // the Gb decoder turns into NaN, so they must be left out of the sum; the
+  // denominator is the number of *observed* genotypes, which is what GEMMA uses:
+  //   maf = sum(geno) / (2 * (ni_test - n_miss))
+  // It used to add NaNs into the sum, so af came out NaN for any marker with a
+  // missing call, and it trusted the caller's n_miss, which the LMDB path passed
+  // as 0 (see mdb_calc_gwa).
+  double total = 0.0;
+  size_t observed = 0;
   for (size_t i = 0; i < ni_total; ++i) { // read genotypes
     if (indicator[i]) {
       double geno = gs[i];
-      maf += geno; // 0..2 range
+      if (!std::isfinite(geno))
+        continue;
+      total += geno; // 0..2 range
+      observed++;
     }
   }
-  maf /= 2.0 * (double)(ni_test - n_miss); // Assumption is that geno value is between 0 and 2. FIXME
-  return maf;
+  (void)ni_test;
+  (void)n_miss; // kept for the signature: `observed' is the same number, counted here
+  if (observed == 0)
+    return 0.0;
+  return total / (2.0 * (double)observed);
 }
diff --git a/src/param.cpp b/src/param.cpp
index 7e34830..b2098ce 100644
--- a/src/param.cpp
+++ b/src/param.cpp
@@ -1309,7 +1309,14 @@ void PARAM::ReadBIMBAMGenotypes(gsl_matrix *UtX, gsl_matrix *K, const bool calc_
 }
 
 gsl_matrix *PARAM::MdbCalcKin() {
-  return mdb_calc_kin(file_geno, a_mode - 20, loco);
+  size_t n_used = 0;
+  gsl_matrix *K = mdb_calc_kin(file_geno, a_mode - 20, loco, n_used,
+                               maf_level, miss_level);
+  // Feed the log: ns_test was never set on this path, so "number of analyzed
+  // SNPs/var" printed whatever the PARAM object happened to hold.
+  ns_total = n_used;
+  ns_test = n_used;
+  return K;
 }
 
 void PARAM::CalcKin(gsl_matrix *matrix_kin) {