#!/usr/bin/env Rscript # Genome scan with R/qtl2, including the LMM that multiparent populations # (DO, CC, 4-way) need and R/qtl's scanone cannot do. # # Reads a qtl2 *control file* (the JSON `read_cross2` understands, with # absolute paths -- the server writes it that way rather than depending on # the working directory), a two-column phenotype file (`Sample` and the # trait, `x` for missing) and optionally a covariate file, and writes one # CSV row per marker: # # marker,chr,pos,lod # # `pos` is in Mb (qtl2's physical map is in base pairs), so the output has # the same shape as /compute/rqtl's and /plot/rqtl draws it unchanged. # # The two expensive steps are cached in --cache-dir, keyed by the control # file's own content hash: genotype probabilities per chromosome (the # dominant cost -- 9 minutes for the DO cross on four cores) and the # kinship matrices. They depend only on the cross, not on the phenotype, so # a second trait costs seconds. # # Written for this API rather than copied from GeneNetwork3's # scripts/rqtl2_wrapper.R: that one calls scan1 with the same arguments in # its "LMM" and "LOCO" branches, and leaves covariates as a TODO. suppressPackageStartupMessages({ library(qtl2) library(optparse) }) option_list = list( make_option(c("--control"), type="character", help="qtl2 control file (absolute paths inside)"), make_option(c("--pheno"), type="character", help="two-column phenotype file: Sample, trait"), make_option(c("--covariates"), type="character", default=NULL, help="phenotype-file-shaped covariate columns (Sample, c1, c2, ...)"), make_option(c("--covarstruct"), type="character", default=NULL, help="covariate name/kind pairs, for dummy expansion"), make_option(c("--kinship"), type="character", default="loco", help="loco (default), overall, or none"), make_option(c("--model"), type="character", default="normal", help="normal (default) or binary"), make_option(c("--cores"), type="integer", default=1), make_option(c("--cache-dir"), type="character", default=NULL, help="where to keep genotype probabilities and kinship"), make_option(c("--nperm"), type="integer", default=0, help="permutations for the LOD thresholds; 0 (default) skips them"), make_option(c("--seed"), type="integer", default=NULL, help="seed for the permutations, so a run is reproducible"), make_option(c("--strata"), type="character", default=NULL, help="permute within strata (e.g. sex): a phenotype-file-shaped column, Sample and stratum"), make_option(c("--perm-out"), type="character", default=NULL, help="where to write one LOD per permutation (default .perm)"), make_option(c("--out"), type="character", required=TRUE), make_option(c("--quiet"), action="store_true", default=FALSE) ) opt = parse_args(OptionParser(option_list=option_list)) if (is.null(opt$control) || is.null(opt$pheno)) { stop("--control and --pheno are required", call.=FALSE) } say <- function(...) if (!opt$quiet) cat(..., "\n") # ---- the cache key: the control file's content ------------------------- # Every input the cross needs is named in the control file, so hashing it # plus each referenced file's size and mtime identifies the cross without # reading gigabytes of CSV on every request. cache_stem <- NULL if (!is.null(opt[["cache-dir"]])) { control <- jsonlite::fromJSON(opt$control, simplifyVector = FALSE) # The control file's own directory is part of the key: two conversions of # the same panel (an older one in another cache directory, say) can have # identical control files and identical file sizes, and reusing the first # one's probabilities silently scans the wrong cross. # The rules this script applies to the cross (the autosome subset) are # part of the key too: caching probabilities that were computed under a # different rule would silently change what a scan means. parts <- list("scan-rules-v1", dirname(normalizePath(opt$control, mustWork = FALSE)), control) for (key in c("geno", "founder_geno", "gmap", "pmap", "pheno", "covar")) { path <- control[[key]] if (!is.null(path)) { info <- file.info(path) parts[[key]] <- list(path, info$size, as.numeric(info$mtime)) } } key <- digest::digest(parts, algo = "sha256") cache_stem <- file.path(opt[["cache-dir"]], substr(key, 1, 16)) dir.create(cache_stem, recursive = TRUE, showWarnings = FALSE) say("cache key:", basename(cache_stem)) } # ---- the cross -------------------------------------------------------- cross <- read_cross2(opt$control, quiet = opt$quiet) say("cross:", cross$crosstype, "| individuals:", n_ind(cross), "| markers:", sum(n_mar(cross)), "| chromosomes:", n_chr(cross)) # ---- autosomes only --------------------------------------------------- # A cross this API converted has its non-autosomal markers removed already; # one read from a control file (the DO, say) carries X, Y and MT as the panel # does. X needs a sex covariate to be fitted properly and Y/MT degenerate the # per-chromosome kinship, so both sources get the same treatment here and the # number left out is printed rather than silently changing the scan. chromosomes <- names(cross$gmap) if (is.null(chromosomes)) chromosomes <- names(cross$pmap) if (!is.null(chromosomes)) { autosomal <- chromosomes[grepl("^[0-9]+$", chromosomes)] left_out <- setdiff(chromosomes, autosomal) if (length(left_out) > 0 && length(autosomal) > 0) { before <- sum(n_mar(cross)) cross <- cross[, chr = autosomal] say("autosomes only: left out", paste(left_out, collapse = ","), "|", before - sum(n_mar(cross)), "markers") } } # ---- genotype probabilities, cached per chromosome -------------------- # Multiparent means more than two founder alleles -- eight for the DO. The # founder genotypes are the honest test; a biallelic cross has none. founders <- tryCatch(n_founders(cross), error = function(e) 0) multiparent <- founders > 2 say("founders:", founders, "| multiparent:", multiparent) probs <- NULL if (!is.null(cache_stem)) { cached <- file.path(cache_stem, "probs.rds") if (file.exists(cached)) { probs <- readRDS(cached) say("genotype probabilities: cached") } } if (is.null(probs)) { started <- Sys.time() probs <- calc_genoprob(cross, error_prob = 0.002, cores = opt$cores, quiet = opt$quiet) say("genotype probabilities:", round(as.numeric(difftime(Sys.time(), started, units="secs")), 1), "s") if (!is.null(cache_stem)) saveRDS(probs, file.path(cache_stem, "probs.rds")) } # Multiparent populations are scanned on allele probabilities (8 founders # for DO); biallelic crosses scan the genotype probabilities directly. scan_probs <- if (multiparent) genoprob_to_alleleprob(probs, cores = opt$cores, quiet = opt$quiet) else probs if (multiparent) say("allele probabilities: states =", dim(scan_probs[[1]])[3]) # ---- kinship, cached -------------------------------------------------- kinship <- NULL if (opt$kinship != "none") { type <- if (opt$kinship == "overall") "overall" else "loco" cached <- if (!is.null(cache_stem)) file.path(cache_stem, paste0("kinship-", type, ".rds")) else NULL if (!is.null(cached) && file.exists(cached)) { kinship <- readRDS(cached) say("kinship (", type, "): cached", sep = "") } else { started <- Sys.time() kinship <- calc_kinship(scan_probs, type, cores = opt$cores, quiet = opt$quiet) say("kinship (", type, "):", round(as.numeric(difftime(Sys.time(), started, units="secs")), 1), "s") if (!is.null(cached)) saveRDS(kinship, cached) } } # ---- phenotype and covariates, aligned by name ------------------------ # sep = "\t" because a trait label may contain a space: with the default # whitespace separator, "DO::10001 (WBC)" splits into three fields and # read.table stops with "line 1 did not have 3 elements", which reads like a # malformed file rather than a label with a space in it. df <- read.table(opt$pheno, sep = "\t", na.strings = "x", header = TRUE, check.names = FALSE) trait_name <- colnames(df)[2] # The cross's individuals, whatever shape its pheno slot takes: a matrix # with rownames when the cross file carries several phenotypes, a named # vector for one, and neither when the pheno file holds a single column -- # in which case the genotype matrix's row names are the individuals. An # empty `ind` produced an empty phenotype and scan1's "No individuals in # common", which reads like a data problem rather than a shape one. ind <- rownames(cross$pheno) if (is.null(ind)) ind <- names(cross$pheno) if (is.null(ind)) ind <- rownames(cross$geno[[1]]) values <- df[[2]][match(ind, df$Sample)] if (all(is.na(values))) { stop("none of the phenotype file's samples appear in the cross", call.=FALSE) } say("phenotype:", trait_name, "|", sum(!is.na(values)), "of", length(values), "individuals have a value") pheno <- matrix(values, ncol = 1, dimnames = list(ind, trait_name)) addcovar <- NULL if (!is.null(opt$covariates)) { covar_df <- read.table(sep = "\t", opt$covariates, header = TRUE, check.names = FALSE, na.strings = "x") struct <- read.table(opt$covarstruct, header = FALSE, sep = "\t", stringsAsFactors = FALSE, col.names = c("name", "kind"), quote = "") addcovar <- data.frame(row.names = ind) for (i in seq_len(nrow(struct))) { column <- covar_df[[1 + i]][match(ind, covar_df$Sample)] if (identical(struct$kind[i], "categorical")) { levels_all <- sort(unique(column[!is.na(column)])) if (length(levels_all) < 2) next for (j in 2:length(levels_all)) { addcovar[[paste0(struct$name[i], ".", levels_all[j])]] <- ifelse(is.na(column), NA, as.numeric(column == levels_all[j])) } } else { addcovar[[struct$name[i]]] <- as.numeric(column) } } # A covariate that does not vary among the phenotyped individuals is # singular in the fit, and contributes nothing anyway. with_pheno <- !is.na(values) varies <- vapply(addcovar, function(column) { vals <- column[with_pheno] length(unique(vals[!is.na(vals)])) > 1 }, logical(1)) if (any(!varies)) say("covariate column(s) left out:", paste(colnames(addcovar)[!varies], collapse=", ")) addcovar <- addcovar[, varies, drop = FALSE] if (ncol(addcovar) == 0) addcovar <- NULL else say("covariates:", paste(colnames(addcovar), collapse = ", ")) } # ---- the scan --------------------------------------------------------- started <- Sys.time() # Multiparent crosses are scanned on one core, deliberately. scan1's forked # path stalls here on allele probabilities -- the parent sits in waitpid with # the children idle (measured: 25+ minutes with cores=8, no CPU consumed), # while cores=1 scans the whole DO genome in 21 s. The expensive step, the # genotype probabilities, is still parallel; only the scan is serial, and it # is the cheap one. scan_cores <- if (multiparent) 1L else opt$cores if (multiparent && opt$cores > 1) { say("scan1 on 1 core: the forked path stalls on allele probabilities") } scan <- scan1(scan_probs, pheno, kinship = kinship, addcovar = addcovar, model = opt$model, cores = scan_cores) say("scan1:", round(as.numeric(difftime(Sys.time(), started, units="secs")), 1), "s") # ---- rows: marker, chr, pos(Mb), lod ---------------------------------- map <- if (!is.null(cross$pmap)) cross$pmap else cross$gmap in_mb <- !is.null(cross$pmap) # qtl2 documents a physical map in base pairs, and the control files this API # writes do hold bp -- but a map that ships with a panel may be in Mb (the # DO's is: 74.22 for a chr6 marker, not 74223530). Dividing by 1e6 twice put # every DO peak at 0.0 Mb, which plots as a peak at the telomere of nothing. # A whole mouse chromosome is ~2e8 bp and ~200 Mb, so the largest position # says which unit the map is in. if (in_mb) { largest <- suppressWarnings(max(unlist(cross$pmap), na.rm = TRUE)) if (is.finite(largest) && largest < 1e6) { say("physical map: already in Mb (largest position", round(largest, 1), ")") in_mb <- FALSE } } rows <- list() for (chr_name in names(map)) { positions <- map[[chr_name]] # qtl2 drops markers it cannot use (all-missing genotypes, say); scanning # them by name would be "subscript out of bounds". markers <- intersect(names(positions), rownames(scan)) if (length(markers) == 0) next positions <- positions[markers] lod <- scan[markers, 1] if (in_mb) positions <- positions / 1e6 rows[[length(rows) + 1]] <- data.frame( marker = markers, chr = chr_name, pos = round(as.numeric(positions), 6), lod = round(as.numeric(lod), 6), stringsAsFactors = FALSE) } result <- do.call(rbind, rows) write.csv(result, opt$out, row.names = FALSE, quote = FALSE) best <- result[which.max(result$lod), ] say(sprintf("wrote %s: %d markers; strongest %s chr%s %.1f Mb LOD %.2f", opt$out, nrow(result), best$marker, best$chr, best$pos, best$lod)) # ---- permutations: the null distribution of the genome-wide maximum --- # One permutation is a whole genome scan of a shuffled phenotype, so the # cost is nperm x the scan above -- which is why the caller decides how # many, and why qtl2's own `cores` argument is used here (the permutations # fork cleanly, unlike scan1 on allele probabilities). # # The output is deliberately the same two-column shape R/qtl's scanone # permutations produce ("", "lod"), so one parser reads both. Two things # that are NOT the same as R/qtl, and are the caller's business: # * scan1perm permutes phenotypes across individuals, which is the right # null for an outbred cross and only approximate for a selfing RIL # panel (qtl2perm is the statistically correct package for those); # * thresholds at alpha depend on how many permutations there are: with # fewer than ~100 the 0.05 quantile is a coin toss. if (opt$nperm > 0) { perm_started <- Sys.time() if (!is.null(opt$seed)) set.seed(opt$seed) strata <- NULL if (!is.null(opt$strata)) { strata_df <- read.table(opt$strata, sep = "\t", header = TRUE, check.names = FALSE, stringsAsFactors = FALSE) strata_values <- strata_df[[2]][match(ind, strata_df$Sample)] if (all(is.na(strata_values))) { stop("none of the stratum file's samples appear in the cross", call. = FALSE) } # qtl2's perm_strata is a vector *of strata per individual*, named by # individual: a factor reaches get_common_ids and it answers "Not sure # what to do with object of class factor". Characters, names, and the # unassigned left as NA so qtl2 drops those individuals from the # permutation rather than refusing. strata <- stats::setNames(as.character(strata_values), ind) say("permutation strata:", paste(sort(unique(strata[!is.na(strata)])), collapse = ","), "|", sum(!is.na(strata)), "individuals assigned") } # chr_lengths from the map: without it scan1perm derives chromosome # lengths from the marker positions, and the thresholds stop being # comparable with R/qtl's on the same cross. lengths <- qtl2::chr_lengths(map) perm <- scan1perm(scan_probs, pheno, kinship = kinship, addcovar = addcovar, n_perm = opt$nperm, perm_strata = strata, chr_lengths = lengths, cores = scan_cores, model = opt$model) perm_values <- as.numeric(perm) if (all(is.na(perm_values))) { # A placeholder or constant phenotype makes every permutation NaN # without an error, and summary() then reports "0 permutations" with NA # thresholds -- a plausible answer that means nothing. Refuse instead. stop("every permutation returned NA (a constant or placeholder phenotype?)", call. = FALSE) } # optparse keeps the dash in the name (`--perm-out` is opt[["perm-out"]]), # so opt$perm_out is always NULL: the fallback silently took over and the # file landed at .perm while the caller read -perm.csv. perm_path <- if (is.null(opt[["perm-out"]])) paste0(opt$out, ".perm") else opt[["perm-out"]] write.csv(data.frame(lod = perm_values), perm_path, row.names = FALSE, quote = FALSE) thresholds <- stats::quantile(perm_values, probs = c(0.99, 0.95, 0.63), na.rm = TRUE, names = FALSE) say(sprintf("permutations: %d in %.1f s (%.3f s each); thresholds 0.01=%.3f 0.05=%.3f 0.37=%.3f", opt$nperm, as.numeric(difftime(Sys.time(), perm_started, units = "secs")), as.numeric(difftime(Sys.time(), perm_started, units = "secs")) / opt$nperm, thresholds[1], thresholds[2], thresholds[3])) say("wrote ", perm_path, ": ", length(perm_values), " permutation maxima") }