about summary refs log tree commit diff
path: root/scripts
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-11-09 19:35:41 +0000
committerPjotr <wrk@bosa.lan>2026-11-09 19:35:41 +0000
commit8dfb6fe64aa815dbe836a461458fa12208d77c2a (patch)
tree293c07a566ba02bf134d9b69468d1fd8c9572c68 /scripts
parentadb75296ecd9ef24f4abed0f45a6b8f6dd657d9b (diff)
downloadgenenetwork4-8dfb6fe64aa815dbe836a461458fa12208d77c2a.tar.gz
qtl2: the engine script and its packaging
Phase 0 of the R/qtl2 work, so the engine can be exercised before any
endpoint exists. `r-qtl2` (0.42 in this channel) plus `r-digest`/`r-jsonlite`
go into the dev shell's propagated inputs; without them R has to be handed a
list of 594 store paths by hand to load qtl2 at all.

`scripts/qtl2_scan.R` is written for this API rather than copied from
GeneNetwork3's `scripts/rqtl2_wrapper.R`, which cannot be trusted as a
reference: its `"LMM"` and `"LOCO"` branches call `scan1` with identical
arguments, its covariates are a `covar <- NULL; # TODO` stub, and its only
tests mock the file-writing helpers -- nothing ever reads a cross.

The script reads a qtl2 control file, a two-column phenotype file and
optional covariates, and writes `marker,chr,pos,lod` (Mb, so the shape
matches /compute/rqtl and /plot/rqtl draws it unchanged). Genotype
probabilities and the kinship matrices are cached in --cache-dir keyed by
the control file's content, because a DO cross costs 536 s of genotype
probabilities on four cores against 11 s of kinship.

Four things the cross-file plumbing cost, all found by running it:

* `read_cross2` joins the control file's directory onto whatever paths it
  finds, so an absolute path becomes `<dir>/<absolute>` and dies. The server
  has to write the control file *beside* its data;
* our generated genotypes are markers-in-rows, which needs
  `"geno_transposed": true` -- without it qtl2 reads the individuals as
  markers (93 individuals appear as 93 extra markers, and every marker is
  "not in both genotypes and maps");
* the genotype strings need a `"genotypes"` map (the GN letters
  @mat/@het/@pat -> A/H/B -> 1/2/3); without it every value is "treated as
  missing" and the scan comes back flat (peak LOD 0.14);
* with those fixed, a cross converted from `genotype/BXD.4.geno` scans in
  ~1 s and agrees with /compute/rqtl on the same panel and trait: qtl2's LMM
  peaks on chr8 at ~111 Mb, LOD 3.50, where R/qtl's HK `scanone` peaks on
  chr8 at ~69 Mb, LOD 2.93 -- same chromosome, different model, which is the
  cross-check the plan asks for.

Not yet: the Rust side (`src/qtl2.rs`), the `/compute/qtl2` route, the
genotypes-listing `qtl2` flag and the converter that writes the control
file. Those are Phase 1-3.
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/qtl2_scan.R193
1 files changed, 193 insertions, 0 deletions
diff --git a/scripts/qtl2_scan.R b/scripts/qtl2_scan.R
new file mode 100755
index 0000000..258eae0
--- /dev/null
+++ b/scripts/qtl2_scan.R
@@ -0,0 +1,193 @@
+#!/usr/bin/env Rscript
+# Genome scan with R/qtl2, including the LMM that multiparent populations
+# (DO, CC, 4-way) need and R/qtl's scanone cannot do.
+#
+# Reads a qtl2 *control file* (the JSON `read_cross2` understands, with
+# absolute paths -- the server writes it that way rather than depending on
+# the working directory), a two-column phenotype file (`Sample` and the
+# trait, `x` for missing) and optionally a covariate file, and writes one
+# CSV row per marker:
+#
+#   marker,chr,pos,lod
+#
+# `pos` is in Mb (qtl2's physical map is in base pairs), so the output has
+# the same shape as /compute/rqtl's and /plot/rqtl draws it unchanged.
+#
+# The two expensive steps are cached in --cache-dir, keyed by the control
+# file's own content hash: genotype probabilities per chromosome (the
+# dominant cost -- 9 minutes for the DO cross on four cores) and the
+# kinship matrices. They depend only on the cross, not on the phenotype, so
+# a second trait costs seconds.
+#
+# Written for this API rather than copied from GeneNetwork3's
+# scripts/rqtl2_wrapper.R: that one calls scan1 with the same arguments in
+# its "LMM" and "LOCO" branches, and leaves covariates as a TODO.
+
+suppressPackageStartupMessages({
+  library(qtl2)
+  library(optparse)
+})
+
+option_list = list(
+  make_option(c("--control"), type="character",
+              help="qtl2 control file (absolute paths inside)"),
+  make_option(c("--pheno"), type="character",
+              help="two-column phenotype file: Sample, trait"),
+  make_option(c("--covariates"), type="character", default=NULL,
+              help="phenotype-file-shaped covariate columns (Sample, c1, c2, ...)"),
+  make_option(c("--covarstruct"), type="character", default=NULL,
+              help="covariate name/kind pairs, for dummy expansion"),
+  make_option(c("--kinship"), type="character", default="loco",
+              help="loco (default), overall, or none"),
+  make_option(c("--model"), type="character", default="normal",
+              help="normal (default) or binary"),
+  make_option(c("--cores"), type="integer", default=1),
+  make_option(c("--cache-dir"), type="character", default=NULL,
+              help="where to keep genotype probabilities and kinship"),
+  make_option(c("--out"), type="character", required=TRUE),
+  make_option(c("--quiet"), action="store_true", default=FALSE)
+)
+opt = parse_args(OptionParser(option_list=option_list))
+if (is.null(opt$control) || is.null(opt$pheno)) {
+  stop("--control and --pheno are required", call.=FALSE)
+}
+
+say <- function(...) if (!opt$quiet) cat(..., "\n")
+
+# ---- the cache key: the control file's content -------------------------
+# Every input the cross needs is named in the control file, so hashing it
+# plus each referenced file's size and mtime identifies the cross without
+# reading gigabytes of CSV on every request.
+cache_stem <- NULL
+if (!is.null(opt[["cache-dir"]])) {
+  control <- jsonlite::fromJSON(opt$control, simplifyVector = FALSE)
+  parts <- list(control)
+  for (key in c("geno", "founder_geno", "gmap", "pmap", "pheno", "covar")) {
+    path <- control[[key]]
+    if (!is.null(path)) {
+      info <- file.info(path)
+      parts[[key]] <- list(path, info$size, as.numeric(info$mtime))
+    }
+  }
+  key <- digest::digest(parts, algo = "sha256")
+  cache_stem <- file.path(opt[["cache-dir"]], substr(key, 1, 16))
+  dir.create(cache_stem, recursive = TRUE, showWarnings = FALSE)
+  say("cache key:", basename(cache_stem))
+}
+
+# ---- the cross --------------------------------------------------------
+cross <- read_cross2(opt$control, quiet = opt$quiet)
+say("cross:", cross$crosstype, "| individuals:", n_ind(cross),
+    "| markers:", sum(n_mar(cross)), "| chromosomes:", n_chr(cross))
+
+# ---- genotype probabilities, cached per chromosome --------------------
+# Multiparent means more than two founder alleles -- eight for the DO. The
+# founder genotypes are the honest test; a biallelic cross has none.
+founders <- tryCatch(n_founders(cross), error = function(e) 0)
+multiparent <- founders > 2
+say("founders:", founders, "| multiparent:", multiparent)
+
+probs <- NULL
+if (!is.null(cache_stem)) {
+  cached <- file.path(cache_stem, "probs.rds")
+  if (file.exists(cached)) {
+    probs <- readRDS(cached)
+    say("genotype probabilities: cached")
+  }
+}
+if (is.null(probs)) {
+  started <- Sys.time()
+  probs <- calc_genoprob(cross, error_prob = 0.002, cores = opt$cores, quiet = opt$quiet)
+  say("genotype probabilities:", round(as.numeric(difftime(Sys.time(), started, units="secs")), 1), "s")
+  if (!is.null(cache_stem)) saveRDS(probs, file.path(cache_stem, "probs.rds"))
+}
+
+# Multiparent populations are scanned on allele probabilities (8 founders
+# for DO); biallelic crosses scan the genotype probabilities directly.
+scan_probs <- if (multiparent) genoprob_to_alleleprob(probs, cores = opt$cores,
+                                                     quiet = opt$quiet) else probs
+if (multiparent) say("allele probabilities: states =", dim(scan_probs[[1]])[3])
+
+# ---- kinship, cached --------------------------------------------------
+kinship <- NULL
+if (opt$kinship != "none") {
+  type <- if (opt$kinship == "overall") "overall" else "loco"
+  cached <- if (!is.null(cache_stem)) file.path(cache_stem, paste0("kinship-", type, ".rds")) else NULL
+  if (!is.null(cached) && file.exists(cached)) {
+    kinship <- readRDS(cached)
+    say("kinship (", type, "): cached", sep = "")
+  } else {
+    started <- Sys.time()
+    kinship <- calc_kinship(scan_probs, type, cores = opt$cores, quiet = opt$quiet)
+    say("kinship (", type, "):", round(as.numeric(difftime(Sys.time(), started, units="secs")), 1), "s")
+    if (!is.null(cached)) saveRDS(kinship, cached)
+  }
+}
+
+# ---- phenotype and covariates, aligned by name ------------------------
+df <- read.table(opt$pheno, na.strings = "x", header = TRUE, check.names = FALSE)
+trait_name <- colnames(df)[2]
+ind <- rownames(cross$pheno)
+values <- df[[2]][match(ind, df$Sample)]
+if (all(is.na(values))) {
+  stop("none of the phenotype file's samples appear in the cross", call.=FALSE)
+}
+say("phenotype:", trait_name, "|", sum(!is.na(values)), "of", length(values), "individuals have a value")
+pheno <- matrix(values, ncol = 1, dimnames = list(ind, trait_name))
+
+addcovar <- NULL
+if (!is.null(opt$covariates)) {
+  covar_df <- read.table(opt$covariates, header = TRUE, check.names = FALSE, na.strings = "x")
+  struct <- read.table(opt$covarstruct, header = FALSE, sep = "\t", stringsAsFactors = FALSE,
+                       col.names = c("name", "kind"), quote = "")
+  addcovar <- data.frame(row.names = ind)
+  for (i in seq_len(nrow(struct))) {
+    column <- covar_df[[1 + i]][match(ind, covar_df$Sample)]
+    if (identical(struct$kind[i], "categorical")) {
+      levels_all <- sort(unique(column[!is.na(column)]))
+      if (length(levels_all) < 2) next
+      for (j in 2:length(levels_all)) {
+        addcovar[[paste0(struct$name[i], ".", levels_all[j])]] <-
+          ifelse(is.na(column), NA, as.numeric(column == levels_all[j]))
+      }
+    } else {
+      addcovar[[struct$name[i]]] <- as.numeric(column)
+    }
+  }
+  # A covariate that does not vary among the phenotyped individuals is
+  # singular in the fit, and contributes nothing anyway.
+  with_pheno <- !is.na(values)
+  varies <- vapply(addcovar, function(column) {
+    vals <- column[with_pheno]
+    length(unique(vals[!is.na(vals)])) > 1
+  }, logical(1))
+  if (any(!varies)) say("covariate column(s) left out:", paste(colnames(addcovar)[!varies], collapse=", "))
+  addcovar <- addcovar[, varies, drop = FALSE]
+  if (ncol(addcovar) == 0) addcovar <- NULL else
+    say("covariates:", paste(colnames(addcovar), collapse = ", "))
+}
+
+# ---- the scan ---------------------------------------------------------
+started <- Sys.time()
+scan <- scan1(scan_probs, pheno, kinship = kinship, addcovar = addcovar,
+              model = opt$model, cores = opt$cores)
+say("scan1:", round(as.numeric(difftime(Sys.time(), started, units="secs")), 1), "s")
+
+# ---- rows: marker, chr, pos(Mb), lod ----------------------------------
+map <- if (!is.null(cross$pmap)) cross$pmap else cross$gmap
+in_mb <- !is.null(cross$pmap)
+rows <- list()
+for (chr_name in names(map)) {
+  positions <- map[[chr_name]]
+  markers <- names(positions)
+  lod <- scan[markers, 1]
+  if (in_mb) positions <- positions / 1e6
+  rows[[length(rows) + 1]] <- data.frame(
+    marker = markers, chr = chr_name, pos = round(as.numeric(positions), 6),
+    lod = round(as.numeric(lod), 6), stringsAsFactors = FALSE)
+}
+result <- do.call(rbind, rows)
+write.csv(result, opt$out, row.names = FALSE, quote = FALSE)
+best <- result[which.max(result$lod), ]
+say(sprintf("wrote %s: %d markers; strongest %s chr%s %.1f Mb LOD %.2f",
+            opt$out, nrow(result), best$marker, best$chr, best$pos, best$lod))