From f026a7b808fd2d61e64d4d53c540fd66fffcff81 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Nov 2026 11:30:59 +0000 Subject: mdb: apply the marker filters, count missing, and give K the same marker set An lmdb scan and a BIMBAM scan of the same data were not answering the same question. Measured on HLC (358454 markers, 23 chromosomes, gemma-wrapper's LOCO recipe), before this commit: -maf 0.05 gemma 20246 markers on chr1, pangemma 27940 (every marker in the store): mdb_calc_gwa computed maf and put it in MarkerInfo but nothing ever compared it with maf_level, and -miss was ignored too af/n_miss nan / 0 for every marker carrying a missing call, because compute_maf was called with n_miss hard-wired to 0 ("count NAs: FIXME") and summed NaNs into the frequency Gb stores 255 is the missing sentinel; read as 255/127.0 it became 2.0079, a valid-looking genotype that silently poisoned af and the test K mdb_calc_kin had no missing handling at all (its mask marked every individual, so the imputation branch never fired) and applied no filters, so HLC's K was built from 330514 markers where GEMMA uses 273349 -- GEMMA's K reader skips whatever its genotype reader rejected (indicator_snp[i] == 0) -- and every p-value moved log line mdb_calc_kin never reported the count, so "number of analyzed SNPs/var" printed an uninitialised ns_test (140048413575759) All four readers of gemma_io.cpp apply the same five filters in a fixed order; the lmdb path now does the same, in the same order, with GEMMA's own allele bins: -miss, -maf (maf_level == -1 disables), monomorphic (every observed genotype equal), -hwe, and -r2 against the covariates (missing imputed with maf*2 as the readers do, W'W inverted once per scan). Missing calls arrive as NaN in both formats, compute_maf counts what it sees, and the K path takes the same filtered marker set and imputes the observed mean exactly as gemma_io.cpp's ReadFile_geno does for K. Verified, HLC LOCO on the slurm cluster (23 tasks, 1 CPU each, all 23 RUNNING together), pangemma lmdb (Gb, missing kept) against gemma 0.98.6 on BIMBAM: same row count on all 23 chromosomes (260251 rows), same top SNP on 23/23 (was 12/23), and no differing numeric cell in beta, se, l_mle or p_lrt (was 260251 rows differing with l_mle pinned at the 1e5 bound). The BIMBAM route is unchanged: 23/23 and 0 differing cells, as before. Cost of the filtering: job time for the lmdb route fell from 179 s to 141 s for the 23 chromosomes (gemma BIMBAM: 958 s), because 30 percent of the markers are no longer analysed. On BXD.8 (20453 markers, 198 individuals, 1460 missing calls) the Gf store now reproduces gemma's output column for column -- af and n_miss included -- with the only differences being 33 `ps` values, where GEMMA's annotation reader truncates R's "1.76e+08" to 1 and the store has the rounded position. --- src/gemma_io.cpp | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 7 deletions(-) (limited to 'src/gemma_io.cpp') diff --git a/src/gemma_io.cpp b/src/gemma_io.cpp index e630f64..11e2746 100644 --- a/src/gemma_io.cpp +++ b/src/gemma_io.cpp @@ -1478,7 +1478,9 @@ void ReadFile_eigenD(const string &file_kd, bool &error, gsl_vector *eval) { // Read bimbam mean genotype file and calculate kinship matrix. -gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco) { +gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string loco, + size_t &n_markers_used, const double maf_level, + const double miss_level) { checkpoint("mdb-kin",file_geno); bool is_loco = !loco.empty(); @@ -1589,7 +1591,11 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string const uint8_t* gs_bbuf = reinterpret_cast(value.data()); gs.reserve(size); for (size_t i = 0; i < size; ++i) { - double g = static_cast(gs_bbuf[i])/127.0; + // 255 is the Gb missing sentinel; read as 255/127.0 it would be 2.0079, + // a real genotype, and every missing call would silently enter the GRM. + double g = (gs_bbuf[i] == 255) + ? std::numeric_limits::quiet_NaN() + : static_cast(gs_bbuf[i])/127.0; gs.push_back(g); } } else { @@ -1603,27 +1609,74 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string } } + // Missing genotypes: the mask used to be set to 1 for every individual (so + // the imputation below never fired) and n_miss was never counted, which is + // why a store with missing calls produced a GRM that looked fine and was not + // the one GEMMA computes. Now: NaN is missing, the mean and variance are + // taken over the observed calls, and the missing ones are plugged with the + // mean, exactly as gemma_io.cpp's ReadFile_geno does for K. + // The same marker filters the scan applies (see mdb_calc_gwa): GEMMA's K + // reader skips every marker its genotype reader rejected + // (indicator_snp[i] == 0, gemma_io.cpp:1775), so K and GWA must see one + // marker set. Without this, HLC's K was built from 330514 markers where + // GEMMA uses 273349, and every p-value in the scan moved. + { + size_t n_miss0 = 0, observed0 = 0; + double total0 = 0.0, first0 = 0.0; + bool first_seen = false, poly = false; + for (size_t i = 0; i < ni_total; ++i) { + double g = gs[i]; + if (std::isnan(g)) { + n_miss0++; + continue; + } + if (!first_seen) { + first0 = g; + first_seen = true; + } else if (g != first0) { + poly = true; + } + total0 += g; + observed0++; + } + double maf0 = observed0 ? total0 / (2.0 * (double)observed0) : 0.0; + if ((double)n_miss0 / (double)ni_total > miss_level) + continue; + if ((maf0 < maf_level || maf0 > (1.0 - maf_level)) && maf_level != -1) + continue; + if (!poly) + continue; + } + size_t n_miss = 0; - double geno_mean = 0.0; + double geno_sum = 0.0; double geno_var = 0.0; gsl_vector_set_all(geno_miss, 0); for (size_t i = 0; i < ni_total; ++i) { double d = gs[i]; + if (std::isnan(d)) { + gsl_vector_set(geno, i, 0.0); // placeholder, patched below + gsl_vector_set(geno_miss, i, 1); + n_miss++; + continue; + } gsl_vector_set(geno, i, d); - gsl_vector_set(geno_miss, i, 1); - geno_mean += d; + geno_sum += d; geno_var += d * d; } - geno_mean /= (double)(ni_total - n_miss); + const size_t observed = ni_total - n_miss; + const double geno_mean = observed ? geno_sum / (double)observed : 0.0; + // Variance over all individuals with the missing ones at the mean, as GEMMA + // does: sum(d^2) over observed + n_miss*mean^2, divided by ni_total. geno_var += geno_mean * geno_mean * (double)n_miss; geno_var /= (double)ni_total; geno_var -= geno_mean * geno_mean; // impute missing values by plugging in the mean for (size_t i = 0; i < ni_total; ++i) { - if (gsl_vector_get(geno_miss, i) == 0) { + if (gsl_vector_get(geno_miss, i) == 1) { gsl_vector_set(geno, i, geno_mean); } } @@ -1664,6 +1717,7 @@ gsl_matrix *mdb_calc_kin(const string file_geno, const int k_mode, const string gsl_vector_free(geno_miss); gsl_matrix_free(Xlarge); + n_markers_used = ns_test; return matrix_kin; } -- cgit 1.4.1