diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/gemma_io.cpp | 68 | ||||
| -rw-r--r-- | src/gemma_io.h | 4 | ||||
| -rw-r--r-- | src/lmm.cpp | 120 | ||||
| -rw-r--r-- | src/lmm.h | 8 | ||||
| -rw-r--r-- | src/mathfunc.cpp | 23 | ||||
| -rw-r--r-- | src/param.cpp | 9 |
6 files changed, 211 insertions, 21 deletions
diff --git a/src/gemma_io.cpp b/src/gemma_io.cpp index e630f64..11e2746 100644 --- a/src/gemma_io.cpp +++ b/src/gemma_io.cpp @@ -1478,7 +1478,9 @@ void ReadFile_eigenD(const string &file_kd, bool &error, gsl_vector *eval) { // Read bimbam mean genotype file and calculate kinship matrix. -gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco) { +gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco, + size_t &n_markers_used, const double maf_level, + const double miss_level) { checkpoint("mdb-kin",file_geno); bool is_loco = !loco.empty(); @@ -1589,7 +1591,11 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string const uint8_t* gs_bbuf = reinterpret_cast<const uint8_t*>(value.data()); gs.reserve(size); for (size_t i = 0; i < size; ++i) { - double g = static_cast<double>(gs_bbuf[i])/127.0; + // 255 is the Gb missing sentinel; read as 255/127.0 it would be 2.0079, + // a real genotype, and every missing call would silently enter the GRM. + double g = (gs_bbuf[i] == 255) + ? std::numeric_limits<double>::quiet_NaN() + : static_cast<double>(gs_bbuf[i])/127.0; gs.push_back(g); } } else { @@ -1603,27 +1609,74 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string } } + // Missing genotypes: the mask used to be set to 1 for every individual (so + // the imputation below never fired) and n_miss was never counted, which is + // why a store with missing calls produced a GRM that looked fine and was not + // the one GEMMA computes. Now: NaN is missing, the mean and variance are + // taken over the observed calls, and the missing ones are plugged with the + // mean, exactly as gemma_io.cpp's ReadFile_geno does for K. + // The same marker filters the scan applies (see mdb_calc_gwa): GEMMA's K + // reader skips every marker its genotype reader rejected + // (indicator_snp[i] == 0, gemma_io.cpp:1775), so K and GWA must see one + // marker set. Without this, HLC's K was built from 330514 markers where + // GEMMA uses 273349, and every p-value in the scan moved. + { + size_t n_miss0 = 0, observed0 = 0; + double total0 = 0.0, first0 = 0.0; + bool first_seen = false, poly = false; + for (size_t i = 0; i < ni_total; ++i) { + double g = gs[i]; + if (std::isnan(g)) { + n_miss0++; + continue; + } + if (!first_seen) { + first0 = g; + first_seen = true; + } else if (g != first0) { + poly = true; + } + total0 += g; + observed0++; + } + double maf0 = observed0 ? total0 / (2.0 * (double)observed0) : 0.0; + if ((double)n_miss0 / (double)ni_total > miss_level) + continue; + if ((maf0 < maf_level || maf0 > (1.0 - maf_level)) && maf_level != -1) + continue; + if (!poly) + continue; + } + size_t n_miss = 0; - double geno_mean = 0.0; + double geno_sum = 0.0; double geno_var = 0.0; gsl_vector_set_all(geno_miss, 0); for (size_t i = 0; i < ni_total; ++i) { double d = gs[i]; + if (std::isnan(d)) { + gsl_vector_set(geno, i, 0.0); // placeholder, patched below + gsl_vector_set(geno_miss, i, 1); + n_miss++; + continue; + } gsl_vector_set(geno, i, d); - gsl_vector_set(geno_miss, i, 1); - geno_mean += d; + geno_sum += d; geno_var += d * d; } - geno_mean /= (double)(ni_total - n_miss); + const size_t observed = ni_total - n_miss; + const double geno_mean = observed ? geno_sum / (double)observed : 0.0; + // Variance over all individuals with the missing ones at the mean, as GEMMA + // does: sum(d^2) over observed + n_miss*mean^2, divided by ni_total. geno_var += geno_mean * geno_mean * (double)n_miss; geno_var /= (double)ni_total; geno_var -= geno_mean * geno_mean; // impute missing values by plugging in the mean for (size_t i = 0; i < ni_total; ++i) { - if (gsl_vector_get(geno_miss, i) == 0) { + if (gsl_vector_get(geno_miss, i) == 1) { gsl_vector_set(geno, i, geno_mean); } } @@ -1664,6 +1717,7 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string gsl_vector_free(geno_miss); gsl_matrix_free(Xlarge); + n_markers_used = ns_test; return matrix_kin; } diff --git a/src/gemma_io.h b/src/gemma_io.h index 06cd0ee..fb58027 100644 --- a/src/gemma_io.h +++ b/src/gemma_io.h @@ -92,7 +92,9 @@ void ReadFile_mk(const string &file_mk, vector<int> &indicator_idv, void ReadFile_eigenU(const string &file_u, bool &error, gsl_matrix *U); void ReadFile_eigenD(const string &file_d, bool &error, gsl_vector *eval); -gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco); +gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco, + size_t &n_markers_used, const double maf_level, + const double miss_level); bool BimbamKin(const string file_geno, const set<string> ksnps, vector<int> &indicator_snp, const int k_mode, diff --git a/src/lmm.cpp b/src/lmm.cpp index 9fe6bbb..9c66cc4 100644 --- a/src/lmm.cpp +++ b/src/lmm.cpp @@ -20,6 +20,7 @@ #include <fstream> #include <iostream> +#include <limits> #include <sstream> #include <assert.h> @@ -70,6 +71,10 @@ void LMM::CopyFromParam(PARAM &cPar) { l_min = cPar.l_min; l_max = cPar.l_max; + maf_level = cPar.maf_level; + miss_level = cPar.miss_level; + hwe_level = cPar.hwe_level; + r2_level = cPar.r2_level; n_region = cPar.n_region; l_mle_null = cPar.l_mle_null; logl_mle_H0 = cPar.logl_mle_H0; @@ -2239,7 +2244,23 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval, auto cursor = lmdb::cursor::open(rtxn, geno_mdb); cout << "## number of total individuals = " << ni_total << endl; cout << "## number of analyzed individuals = " << ni_test << endl; - cout << "## number of analyzed SNPs/var = " << num_markers << endl; + cout << "## number of total SNPs/var = " << num_markers << endl; + + // -r2 needs (W'W)^-1, which the readers of gemma_io.cpp precompute once per + // read; here it is once per scan. W is the covariate matrix (intercept + -c + // columns) with ni_test rows, exactly what the readers get. + gsl_matrix *WtW = gsl_matrix_safe_alloc(W->size2, W->size2); + gsl_matrix *WtWi = gsl_matrix_safe_alloc(W->size2, W->size2); + gsl_permutation *pmt = gsl_permutation_alloc(W->size2); + gsl_vector *Wtx = gsl_vector_safe_alloc(W->size2); + gsl_vector *WtWiWtx = gsl_vector_safe_alloc(W->size2); + gsl_vector *x_r2 = gsl_vector_safe_alloc(ni_test); + gsl_blas_dgemm(CblasTrans, CblasNoTrans, 1.0, W, W, 0.0, WtW); + int sig = 0; + LUDecomp(WtW, pmt, &sig); + LUInvert(WtW, pmt, WtWi); + + size_t n_analyzed = 0; // markers that survive the filters, for the log line std::function<SnpNameValues2(size_t)> fetch_snp = [&](size_t num) { @@ -2262,7 +2283,14 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval, const uint8_t* gs_bbuf = reinterpret_cast<const uint8_t*>(value.data()); gs.reserve(size); for (size_t i = 0; i < size; ++i) { - double g = static_cast<double>(gs_bbuf[i])/127.0; + // 255 is the Gb missing sentinel (gemma-wrapper's geno2mdb and + // genenetwork4's make-mdb.sh both write it). Read as 255/127.0 it + // becomes 2.0079 -- a valid-looking genotype that silently poisons af + // and the analysis -- so it has to arrive as NaN, which is what + // mdb_analyze already knows how to drop and mean-substitute. + double g = (gs_bbuf[i] == 255) + ? std::numeric_limits<double>::quiet_NaN() + : static_cast<double>(gs_bbuf[i])/127.0; gs.push_back(g); } } else { @@ -2306,11 +2334,79 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval, // 1 rs13476251 174792257 // cout << static_cast<int>(chr) << ":" << pos2 << " line " << rest2 << ":" << marker << endl ; - // compute maf and n_miss (NAs) - size_t n_miss = 0; // count NAs: FIXME - double maf = compute_maf(ni_total, ni_test, n_miss, gs.data(), indicator_idv); + // Same marker statistics and the same filters as gemma_io.cpp's + // ReadFile_geno/ReadFile_bimbam_geno, in the same order, so that an lmdb + // scan answers the question a BIMBAM scan answers. Before this, every + // marker in the store was analysed: -maf 0.05 on HLC still returned all + // 27940 markers of chromosome 1 where GEMMA returns 20246, and af/n_miss + // came out nan/0 for any marker with a missing call. + size_t n_miss = 0, n_0 = 0, n_1 = 0, n_2 = 0, observed = 0; + double geno_total = 0.0; + bool first = true, polymorphic = false; + double geno_first = 0.0; + for (size_t i = 0; i < ni_total; ++i) { + if (indicator_idv[i] == 0) + continue; + double geno = gs[i]; + if (std::isnan(geno)) { + n_miss++; + continue; + } + // The allele-count bins are GEMMA's own (gemma_io.cpp:765-773). + if (geno >= 0 && geno <= 0.5) + n_0++; + if (geno > 0.5 && geno < 1.5) + n_1++; + if (geno >= 1.5 && geno <= 2.0) + n_2++; + if (first) { + geno_first = geno; + first = false; + } else if (geno != geno_first) { + polymorphic = true; + } + geno_total += geno; + observed++; + } - markerinfo = MarkerInfo { .name=marker,.chr=chr,.pos=pos,.line_no=num,.n_miss=n_miss,.maf=maf }; + double maf = observed ? geno_total / (2.0 * (double)observed) : 0.0; + markerinfo = MarkerInfo { .name=marker,.chr=chr,.pos=pos,.line_no=num, + .n_miss=n_miss,.maf=maf }; + + // -miss + if ((double)n_miss / (double)ni_test > miss_level) + return make_tuple(SKIP, markerinfo, gs); + // -maf (maf_level == -1 disables it, as in GEMMA) + if ((maf < maf_level || maf > (1.0 - maf_level)) && maf_level != -1) + return make_tuple(SKIP, markerinfo, gs); + // monomorphic: every observed genotype equal + if (!polymorphic) + return make_tuple(SKIP, markerinfo, gs); + // -hwe + if (hwe_level != 0 && maf_level != -1) { + if (CalcHWE(n_0, n_2, n_1) < hwe_level) + return make_tuple(SKIP, markerinfo, gs); + } + // -r2: correlation with the covariates, computed on the mean-substituted + // genotypes exactly as the readers do (missing -> maf*2). + if (W->size2 != 1) { + size_t pos2 = 0; + for (size_t i = 0; i < ni_total; ++i) { + if (indicator_idv[i] == 0) + continue; + double geno = gs[i]; + gsl_vector_set(x_r2, pos2, std::isnan(geno) ? maf * 2.0 : geno); + pos2++; + } + double v_x = 0.0, v_w = 0.0; + gsl_blas_dgemv(CblasTrans, 1.0, W, x_r2, 0.0, Wtx); + gsl_blas_dgemv(CblasNoTrans, 1.0, WtWi, Wtx, 0.0, WtWiWtx); + gsl_blas_ddot(x_r2, x_r2, &v_x); + gsl_blas_ddot(Wtx, WtWiWtx, &v_w); + if (v_x > 0.0 && v_w / v_x > r2_level) + return make_tuple(SKIP, markerinfo, gs); + } + n_analyzed++; // cout << "!!!!" << size << marker << ": af" << maf << " " << gs[0] << "," << gs[1] << "," << gs[2] << "," << gs[3] << endl; } @@ -2318,7 +2414,17 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval, }; LMM::mdb_analyze(fetch_snp,U,eval,UtW,Uty,W,y,num_markers); - ns_total = ns_test = num_markers; // track global number of snps in original gemma (goes to cPar) + gsl_matrix_free(WtW); + gsl_matrix_free(WtWi); + gsl_permutation_free(pmt); + gsl_vector_free(Wtx); + gsl_vector_free(WtWiWtx); + gsl_vector_free(x_r2); + + cout << "## number of analyzed SNPs/var = " << n_analyzed << endl; + + ns_total = num_markers; + ns_test = n_analyzed; // what actually reaches the output (goes to cPar) } diff --git a/src/lmm.h b/src/lmm.h index 295602a..86b16c4 100644 --- a/src/lmm.h +++ b/src/lmm.h @@ -124,6 +124,14 @@ public: // Not included in PARAM. vector<SUMSTAT> sumStat; // Output SNPSummary Data. + // The marker filters, copied from PARAM by CopyFromParam so that the LMDB + // path can apply the same ones the BIMBAM and PLINK readers do (it used to + // ignore them entirely: every marker in the store was analysed). + double maf_level = 0.0; + double miss_level = 0.0; + double hwe_level = 0.0; + double r2_level = 1.0; + // Main functions. void CopyFromParam(PARAM &cPar); void CopyToParam(PARAM &cPar); diff --git a/src/mathfunc.cpp b/src/mathfunc.cpp index c82872e..ad29982 100644 --- a/src/mathfunc.cpp +++ b/src/mathfunc.cpp @@ -667,14 +667,27 @@ std::tuple<size_t, size_t, size_t> compute_ratio(size_t ni_total, const gsl_vect } double compute_maf(size_t ni_total, size_t ni_test, size_t n_miss, const double *gs, const vector<int> &indicator) { - double maf = 0.0; - + // Missing genotypes arrive here as NaN (Gf stores) or as the 255 sentinel that + // the Gb decoder turns into NaN, so they must be left out of the sum; the + // denominator is the number of *observed* genotypes, which is what GEMMA uses: + // maf = sum(geno) / (2 * (ni_test - n_miss)) + // It used to add NaNs into the sum, so af came out NaN for any marker with a + // missing call, and it trusted the caller's n_miss, which the LMDB path passed + // as 0 (see mdb_calc_gwa). + double total = 0.0; + size_t observed = 0; for (size_t i = 0; i < ni_total; ++i) { // read genotypes if (indicator[i]) { double geno = gs[i]; - maf += geno; // 0..2 range + if (!std::isfinite(geno)) + continue; + total += geno; // 0..2 range + observed++; } } - maf /= 2.0 * (double)(ni_test - n_miss); // Assumption is that geno value is between 0 and 2. FIXME - return maf; + (void)ni_test; + (void)n_miss; // kept for the signature: `observed' is the same number, counted here + if (observed == 0) + return 0.0; + return total / (2.0 * (double)observed); } diff --git a/src/param.cpp b/src/param.cpp index 7e34830..b2098ce 100644 --- a/src/param.cpp +++ b/src/param.cpp @@ -1309,7 +1309,14 @@ void PARAM::ReadBIMBAMGenotypes(gsl_matrix *UtX, gsl_matrix *K, const bool calc_ } gsl_matrix *PARAM::MdbCalcKin() { - return mdb_calc_kin(file_geno, a_mode - 20, loco); + size_t n_used = 0; + gsl_matrix *K = mdb_calc_kin(file_geno, a_mode - 20, loco, n_used, + maf_level, miss_level); + // Feed the log: ns_test was never set on this path, so "number of analyzed + // SNPs/var" printed whatever the PARAM object happened to hold. + ns_total = n_used; + ns_test = n_used; + return K; } void PARAM::CalcKin(gsl_matrix *matrix_kin) { |
