diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-18 11:30:59 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-18 11:30:59 +0000 |
| commit | f026a7b808fd2d61e64d4d53c540fd66fffcff81 (patch) | |
| tree | 0b5ca159775cba3a6d7a847d1c03cc6e22a061f7 /src/lmm.cpp | |
| parent | cebca3b5cc041731df65bb5008fd3e7d5202c299 (diff) | |
| download | pangemma-f026a7b808fd2d61e64d4d53c540fd66fffcff81.tar.gz | |
mdb: apply the marker filters, count missing, and give K the same marker set
An lmdb scan and a BIMBAM scan of the same data were not answering the same
question. Measured on HLC (358454 markers, 23 chromosomes, gemma-wrapper's
LOCO recipe), before this commit:
-maf 0.05 gemma 20246 markers on chr1, pangemma 27940 (every marker in the
store): mdb_calc_gwa computed maf and put it in MarkerInfo but
nothing ever compared it with maf_level, and -miss was ignored too
af/n_miss nan / 0 for every marker carrying a missing call, because
compute_maf was called with n_miss hard-wired to 0 ("count NAs:
FIXME") and summed NaNs into the frequency
Gb stores 255 is the missing sentinel; read as 255/127.0 it became 2.0079, a
valid-looking genotype that silently poisoned af and the test
K mdb_calc_kin had no missing handling at all (its mask marked every
individual, so the imputation branch never fired) and applied no
filters, so HLC's K was built from 330514 markers where GEMMA uses
273349 -- GEMMA's K reader skips whatever its genotype reader
rejected (indicator_snp[i] == 0) -- and every p-value moved
log line mdb_calc_kin never reported the count, so "number of analyzed
SNPs/var" printed an uninitialised ns_test (140048413575759)
All four readers of gemma_io.cpp apply the same five filters in a fixed order;
the lmdb path now does the same, in the same order, with GEMMA's own allele
bins: -miss, -maf (maf_level == -1 disables), monomorphic (every observed
genotype equal), -hwe, and -r2 against the covariates (missing imputed with
maf*2 as the readers do, W'W inverted once per scan). Missing calls arrive as
NaN in both formats, compute_maf counts what it sees, and the K path takes the
same filtered marker set and imputes the observed mean exactly as
gemma_io.cpp's ReadFile_geno does for K.
Verified, HLC LOCO on the slurm cluster (23 tasks, 1 CPU each, all 23 RUNNING
together), pangemma lmdb (Gb, missing kept) against gemma 0.98.6 on BIMBAM:
same row count on all 23 chromosomes (260251 rows), same top SNP on 23/23 (was
12/23), and no differing numeric cell in beta, se, l_mle or p_lrt (was 260251
rows differing with l_mle pinned at the 1e5 bound). The BIMBAM route is
unchanged: 23/23 and 0 differing cells, as before. Cost of the filtering:
job time for the lmdb route fell from 179 s to 141 s for the 23 chromosomes
(gemma BIMBAM: 958 s), because 30 percent of the markers are no longer
analysed.
On BXD.8 (20453 markers, 198 individuals, 1460 missing calls) the Gf store now
reproduces gemma's output column for column -- af and n_miss included -- with
the only differences being 33 `ps` values, where GEMMA's annotation reader
truncates R's "1.76e+08" to 1 and the store has the rounded position.
Diffstat (limited to 'src/lmm.cpp')
| -rw-r--r-- | src/lmm.cpp | 120 |
1 files changed, 113 insertions, 7 deletions
diff --git a/src/lmm.cpp b/src/lmm.cpp index 9fe6bbb..9c66cc4 100644 --- a/src/lmm.cpp +++ b/src/lmm.cpp @@ -20,6 +20,7 @@ #include <fstream> #include <iostream> +#include <limits> #include <sstream> #include <assert.h> @@ -70,6 +71,10 @@ void LMM::CopyFromParam(PARAM &cPar) { l_min = cPar.l_min; l_max = cPar.l_max; + maf_level = cPar.maf_level; + miss_level = cPar.miss_level; + hwe_level = cPar.hwe_level; + r2_level = cPar.r2_level; n_region = cPar.n_region; l_mle_null = cPar.l_mle_null; logl_mle_H0 = cPar.logl_mle_H0; @@ -2239,7 +2244,23 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval, auto cursor = lmdb::cursor::open(rtxn, geno_mdb); cout << "## number of total individuals = " << ni_total << endl; cout << "## number of analyzed individuals = " << ni_test << endl; - cout << "## number of analyzed SNPs/var = " << num_markers << endl; + cout << "## number of total SNPs/var = " << num_markers << endl; + + // -r2 needs (W'W)^-1, which the readers of gemma_io.cpp precompute once per + // read; here it is once per scan. W is the covariate matrix (intercept + -c + // columns) with ni_test rows, exactly what the readers get. + gsl_matrix *WtW = gsl_matrix_safe_alloc(W->size2, W->size2); + gsl_matrix *WtWi = gsl_matrix_safe_alloc(W->size2, W->size2); + gsl_permutation *pmt = gsl_permutation_alloc(W->size2); + gsl_vector *Wtx = gsl_vector_safe_alloc(W->size2); + gsl_vector *WtWiWtx = gsl_vector_safe_alloc(W->size2); + gsl_vector *x_r2 = gsl_vector_safe_alloc(ni_test); + gsl_blas_dgemm(CblasTrans, CblasNoTrans, 1.0, W, W, 0.0, WtW); + int sig = 0; + LUDecomp(WtW, pmt, &sig); + LUInvert(WtW, pmt, WtWi); + + size_t n_analyzed = 0; // markers that survive the filters, for the log line std::function<SnpNameValues2(size_t)> fetch_snp = [&](size_t num) { @@ -2262,7 +2283,14 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval, const uint8_t* gs_bbuf = reinterpret_cast<const uint8_t*>(value.data()); gs.reserve(size); for (size_t i = 0; i < size; ++i) { - double g = static_cast<double>(gs_bbuf[i])/127.0; + // 255 is the Gb missing sentinel (gemma-wrapper's geno2mdb and + // genenetwork4's make-mdb.sh both write it). Read as 255/127.0 it + // becomes 2.0079 -- a valid-looking genotype that silently poisons af + // and the analysis -- so it has to arrive as NaN, which is what + // mdb_analyze already knows how to drop and mean-substitute. + double g = (gs_bbuf[i] == 255) + ? std::numeric_limits<double>::quiet_NaN() + : static_cast<double>(gs_bbuf[i])/127.0; gs.push_back(g); } } else { @@ -2306,11 +2334,79 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval, // 1 rs13476251 174792257 // cout << static_cast<int>(chr) << ":" << pos2 << " line " << rest2 << ":" << marker << endl ; - // compute maf and n_miss (NAs) - size_t n_miss = 0; // count NAs: FIXME - double maf = compute_maf(ni_total, ni_test, n_miss, gs.data(), indicator_idv); + // Same marker statistics and the same filters as gemma_io.cpp's + // ReadFile_geno/ReadFile_bimbam_geno, in the same order, so that an lmdb + // scan answers the question a BIMBAM scan answers. Before this, every + // marker in the store was analysed: -maf 0.05 on HLC still returned all + // 27940 markers of chromosome 1 where GEMMA returns 20246, and af/n_miss + // came out nan/0 for any marker with a missing call. + size_t n_miss = 0, n_0 = 0, n_1 = 0, n_2 = 0, observed = 0; + double geno_total = 0.0; + bool first = true, polymorphic = false; + double geno_first = 0.0; + for (size_t i = 0; i < ni_total; ++i) { + if (indicator_idv[i] == 0) + continue; + double geno = gs[i]; + if (std::isnan(geno)) { + n_miss++; + continue; + } + // The allele-count bins are GEMMA's own (gemma_io.cpp:765-773). + if (geno >= 0 && geno <= 0.5) + n_0++; + if (geno > 0.5 && geno < 1.5) + n_1++; + if (geno >= 1.5 && geno <= 2.0) + n_2++; + if (first) { + geno_first = geno; + first = false; + } else if (geno != geno_first) { + polymorphic = true; + } + geno_total += geno; + observed++; + } - markerinfo = MarkerInfo { .name=marker,.chr=chr,.pos=pos,.line_no=num,.n_miss=n_miss,.maf=maf }; + double maf = observed ? geno_total / (2.0 * (double)observed) : 0.0; + markerinfo = MarkerInfo { .name=marker,.chr=chr,.pos=pos,.line_no=num, + .n_miss=n_miss,.maf=maf }; + + // -miss + if ((double)n_miss / (double)ni_test > miss_level) + return make_tuple(SKIP, markerinfo, gs); + // -maf (maf_level == -1 disables it, as in GEMMA) + if ((maf < maf_level || maf > (1.0 - maf_level)) && maf_level != -1) + return make_tuple(SKIP, markerinfo, gs); + // monomorphic: every observed genotype equal + if (!polymorphic) + return make_tuple(SKIP, markerinfo, gs); + // -hwe + if (hwe_level != 0 && maf_level != -1) { + if (CalcHWE(n_0, n_2, n_1) < hwe_level) + return make_tuple(SKIP, markerinfo, gs); + } + // -r2: correlation with the covariates, computed on the mean-substituted + // genotypes exactly as the readers do (missing -> maf*2). + if (W->size2 != 1) { + size_t pos2 = 0; + for (size_t i = 0; i < ni_total; ++i) { + if (indicator_idv[i] == 0) + continue; + double geno = gs[i]; + gsl_vector_set(x_r2, pos2, std::isnan(geno) ? maf * 2.0 : geno); + pos2++; + } + double v_x = 0.0, v_w = 0.0; + gsl_blas_dgemv(CblasTrans, 1.0, W, x_r2, 0.0, Wtx); + gsl_blas_dgemv(CblasNoTrans, 1.0, WtWi, Wtx, 0.0, WtWiWtx); + gsl_blas_ddot(x_r2, x_r2, &v_x); + gsl_blas_ddot(Wtx, WtWiWtx, &v_w); + if (v_x > 0.0 && v_w / v_x > r2_level) + return make_tuple(SKIP, markerinfo, gs); + } + n_analyzed++; // cout << "!!!!" << size << marker << ": af" << maf << " " << gs[0] << "," << gs[1] << "," << gs[2] << "," << gs[3] << endl; } @@ -2318,7 +2414,17 @@ void LMM::mdb_calc_gwa(const gsl_matrix *U, const gsl_vector *eval, }; LMM::mdb_analyze(fetch_snp,U,eval,UtW,Uty,W,y,num_markers); - ns_total = ns_test = num_markers; // track global number of snps in original gemma (goes to cPar) + gsl_matrix_free(WtW); + gsl_matrix_free(WtWi); + gsl_permutation_free(pmt); + gsl_vector_free(Wtx); + gsl_vector_free(WtWiWtx); + gsl_vector_free(x_r2); + + cout << "## number of analyzed SNPs/var = " << n_analyzed << endl; + + ns_total = num_markers; + ns_test = n_analyzed; // what actually reaches the output (goes to cPar) } |
