about summary refs log tree commit diff
path: root/scripts/qtl2_scan.R
blob: 724eb69c504e1ba858c2629ace3f52154ffdf3dc (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
#!/usr/bin/env Rscript
# Genome scan with R/qtl2, including the LMM that multiparent populations
# (DO, CC, 4-way) need and R/qtl's scanone cannot do.
#
# Reads a qtl2 *control file* (the JSON `read_cross2` understands, with
# absolute paths -- the server writes it that way rather than depending on
# the working directory), a two-column phenotype file (`Sample` and the
# trait, `x` for missing) and optionally a covariate file, and writes one
# CSV row per marker:
#
#   marker,chr,pos,lod
#
# `pos` is in Mb (qtl2's physical map is in base pairs), so the output has
# the same shape as /compute/rqtl's and /plot/rqtl draws it unchanged.
#
# The two expensive steps are cached in --cache-dir, keyed by the control
# file's own content hash: genotype probabilities per chromosome (the
# dominant cost -- 9 minutes for the DO cross on four cores) and the
# kinship matrices. They depend only on the cross, not on the phenotype, so
# a second trait costs seconds.
#
# Written for this API rather than copied from GeneNetwork3's
# scripts/rqtl2_wrapper.R: that one calls scan1 with the same arguments in
# its "LMM" and "LOCO" branches, and leaves covariates as a TODO.

suppressPackageStartupMessages({
  library(qtl2)
  library(optparse)
})

option_list = list(
  make_option(c("--control"), type="character",
              help="qtl2 control file (absolute paths inside)"),
  make_option(c("--pheno"), type="character",
              help="two-column phenotype file: Sample, trait"),
  make_option(c("--covariates"), type="character", default=NULL,
              help="phenotype-file-shaped covariate columns (Sample, c1, c2, ...)"),
  make_option(c("--covarstruct"), type="character", default=NULL,
              help="covariate name/kind pairs, for dummy expansion"),
  make_option(c("--kinship"), type="character", default="loco",
              help="loco (default), overall, or none"),
  make_option(c("--model"), type="character", default="normal",
              help="normal (default) or binary"),
  make_option(c("--cores"), type="integer", default=1),
  make_option(c("--cache-dir"), type="character", default=NULL,
              help="where to keep genotype probabilities and kinship"),
  make_option(c("--nperm"), type="integer", default=0,
              help="permutations for the LOD thresholds; 0 (default) skips them"),
  make_option(c("--seed"), type="integer", default=NULL,
              help="seed for the permutations, so a run is reproducible"),
  make_option(c("--strata"), type="character", default=NULL,
              help="permute within strata (e.g. sex): a phenotype-file-shaped column, Sample and stratum"),
  make_option(c("--perm-out"), type="character", default=NULL,
              help="where to write one LOD per permutation (default <out>.perm)"),
  make_option(c("--out"), type="character", required=TRUE),
  make_option(c("--quiet"), action="store_true", default=FALSE)
)
opt = parse_args(OptionParser(option_list=option_list))
if (is.null(opt$control) || is.null(opt$pheno)) {
  stop("--control and --pheno are required", call.=FALSE)
}

say <- function(...) if (!opt$quiet) cat(..., "\n")

# ---- the cache key: the control file's content -------------------------
# Every input the cross needs is named in the control file, so hashing it
# plus each referenced file's size and mtime identifies the cross without
# reading gigabytes of CSV on every request.
cache_stem <- NULL
if (!is.null(opt[["cache-dir"]])) {
  control <- jsonlite::fromJSON(opt$control, simplifyVector = FALSE)
  # The control file's own directory is part of the key: two conversions of
  # the same panel (an older one in another cache directory, say) can have
  # identical control files and identical file sizes, and reusing the first
  # one's probabilities silently scans the wrong cross.
  # The rules this script applies to the cross (the autosome subset) are
  # part of the key too: caching probabilities that were computed under a
  # different rule would silently change what a scan means.
  parts <- list("scan-rules-v1", dirname(normalizePath(opt$control, mustWork = FALSE)), control)
  for (key in c("geno", "founder_geno", "gmap", "pmap", "pheno", "covar")) {
    path <- control[[key]]
    if (!is.null(path)) {
      info <- file.info(path)
      parts[[key]] <- list(path, info$size, as.numeric(info$mtime))
    }
  }
  key <- digest::digest(parts, algo = "sha256")
  cache_stem <- file.path(opt[["cache-dir"]], substr(key, 1, 16))
  dir.create(cache_stem, recursive = TRUE, showWarnings = FALSE)
  say("cache key:", basename(cache_stem))
}

# ---- the cross --------------------------------------------------------
cross <- read_cross2(opt$control, quiet = opt$quiet)
say("cross:", cross$crosstype, "| individuals:", n_ind(cross),
    "| markers:", sum(n_mar(cross)), "| chromosomes:", n_chr(cross))

# ---- autosomes only ---------------------------------------------------
# A cross this API converted has its non-autosomal markers removed already;
# one read from a control file (the DO, say) carries X, Y and MT as the panel
# does. X needs a sex covariate to be fitted properly and Y/MT degenerate the
# per-chromosome kinship, so both sources get the same treatment here and the
# number left out is printed rather than silently changing the scan.
chromosomes <- names(cross$gmap)
if (is.null(chromosomes)) chromosomes <- names(cross$pmap)
if (!is.null(chromosomes)) {
  autosomal <- chromosomes[grepl("^[0-9]+$", chromosomes)]
  left_out <- setdiff(chromosomes, autosomal)
  if (length(left_out) > 0 && length(autosomal) > 0) {
    before <- sum(n_mar(cross))
    cross <- cross[, chr = autosomal]
    say("autosomes only: left out", paste(left_out, collapse = ","), "|",
        before - sum(n_mar(cross)), "markers")
  }
}

# ---- genotype probabilities, cached per chromosome --------------------
# Multiparent means more than two founder alleles -- eight for the DO. The
# founder genotypes are the honest test; a biallelic cross has none.
founders <- tryCatch(n_founders(cross), error = function(e) 0)
multiparent <- founders > 2
say("founders:", founders, "| multiparent:", multiparent)

probs <- NULL
if (!is.null(cache_stem)) {
  cached <- file.path(cache_stem, "probs.rds")
  if (file.exists(cached)) {
    probs <- readRDS(cached)
    say("genotype probabilities: cached")
  }
}
if (is.null(probs)) {
  started <- Sys.time()
  probs <- calc_genoprob(cross, error_prob = 0.002, cores = opt$cores, quiet = opt$quiet)
  say("genotype probabilities:", round(as.numeric(difftime(Sys.time(), started, units="secs")), 1), "s")
  if (!is.null(cache_stem)) saveRDS(probs, file.path(cache_stem, "probs.rds"))
}

# Multiparent populations are scanned on allele probabilities (8 founders
# for DO); biallelic crosses scan the genotype probabilities directly.
scan_probs <- if (multiparent) genoprob_to_alleleprob(probs, cores = opt$cores,
                                                     quiet = opt$quiet) else probs
if (multiparent) say("allele probabilities: states =", dim(scan_probs[[1]])[3])

# ---- kinship, cached --------------------------------------------------
kinship <- NULL
if (opt$kinship != "none") {
  type <- if (opt$kinship == "overall") "overall" else "loco"
  cached <- if (!is.null(cache_stem)) file.path(cache_stem, paste0("kinship-", type, ".rds")) else NULL
  if (!is.null(cached) && file.exists(cached)) {
    kinship <- readRDS(cached)
    say("kinship (", type, "): cached", sep = "")
  } else {
    started <- Sys.time()
    kinship <- calc_kinship(scan_probs, type, cores = opt$cores, quiet = opt$quiet)
    say("kinship (", type, "):", round(as.numeric(difftime(Sys.time(), started, units="secs")), 1), "s")
    if (!is.null(cached)) saveRDS(kinship, cached)
  }
}

# ---- phenotype and covariates, aligned by name ------------------------
# sep = "\t" because a trait label may contain a space: with the default
# whitespace separator, "DO::10001 (WBC)" splits into three fields and
# read.table stops with "line 1 did not have 3 elements", which reads like a
# malformed file rather than a label with a space in it.
df <- read.table(opt$pheno, sep = "\t", na.strings = "x", header = TRUE, check.names = FALSE)
trait_name <- colnames(df)[2]
# The cross's individuals, whatever shape its pheno slot takes: a matrix
# with rownames when the cross file carries several phenotypes, a named
# vector for one, and neither when the pheno file holds a single column --
# in which case the genotype matrix's row names are the individuals. An
# empty `ind` produced an empty phenotype and scan1's "No individuals in
# common", which reads like a data problem rather than a shape one.
ind <- rownames(cross$pheno)
if (is.null(ind)) ind <- names(cross$pheno)
if (is.null(ind)) ind <- rownames(cross$geno[[1]])
values <- df[[2]][match(ind, df$Sample)]
if (all(is.na(values))) {
  stop("none of the phenotype file's samples appear in the cross", call.=FALSE)
}
say("phenotype:", trait_name, "|", sum(!is.na(values)), "of", length(values), "individuals have a value")
pheno <- matrix(values, ncol = 1, dimnames = list(ind, trait_name))

addcovar <- NULL
if (!is.null(opt$covariates)) {
  covar_df <- read.table(sep = "\t", opt$covariates, header = TRUE, check.names = FALSE, na.strings = "x")
  struct <- read.table(opt$covarstruct, header = FALSE, sep = "\t", stringsAsFactors = FALSE,
                       col.names = c("name", "kind"), quote = "")
  addcovar <- data.frame(row.names = ind)
  for (i in seq_len(nrow(struct))) {
    column <- covar_df[[1 + i]][match(ind, covar_df$Sample)]
    if (identical(struct$kind[i], "categorical")) {
      levels_all <- sort(unique(column[!is.na(column)]))
      if (length(levels_all) < 2) next
      for (j in 2:length(levels_all)) {
        addcovar[[paste0(struct$name[i], ".", levels_all[j])]] <-
          ifelse(is.na(column), NA, as.numeric(column == levels_all[j]))
      }
    } else {
      addcovar[[struct$name[i]]] <- as.numeric(column)
    }
  }
  # A covariate that does not vary among the phenotyped individuals is
  # singular in the fit, and contributes nothing anyway.
  with_pheno <- !is.na(values)
  varies <- vapply(addcovar, function(column) {
    vals <- column[with_pheno]
    length(unique(vals[!is.na(vals)])) > 1
  }, logical(1))
  if (any(!varies)) say("covariate column(s) left out:", paste(colnames(addcovar)[!varies], collapse=", "))
  addcovar <- addcovar[, varies, drop = FALSE]
  if (ncol(addcovar) == 0) addcovar <- NULL else
    say("covariates:", paste(colnames(addcovar), collapse = ", "))
}

# ---- the scan ---------------------------------------------------------
started <- Sys.time()
# Multiparent crosses are scanned on one core, deliberately. scan1's forked
# path stalls here on allele probabilities -- the parent sits in waitpid with
# the children idle (measured: 25+ minutes with cores=8, no CPU consumed),
# while cores=1 scans the whole DO genome in 21 s. The expensive step, the
# genotype probabilities, is still parallel; only the scan is serial, and it
# is the cheap one.
scan_cores <- if (multiparent) 1L else opt$cores
if (multiparent && opt$cores > 1) {
  say("scan1 on 1 core: the forked path stalls on allele probabilities")
}
scan <- scan1(scan_probs, pheno, kinship = kinship, addcovar = addcovar,
              model = opt$model, cores = scan_cores)
say("scan1:", round(as.numeric(difftime(Sys.time(), started, units="secs")), 1), "s")

# ---- rows: marker, chr, pos(Mb), lod ----------------------------------
map <- if (!is.null(cross$pmap)) cross$pmap else cross$gmap
in_mb <- !is.null(cross$pmap)
# qtl2 documents a physical map in base pairs, and the control files this API
# writes do hold bp -- but a map that ships with a panel may be in Mb (the
# DO's is: 74.22 for a chr6 marker, not 74223530). Dividing by 1e6 twice put
# every DO peak at 0.0 Mb, which plots as a peak at the telomere of nothing.
# A whole mouse chromosome is ~2e8 bp and ~200 Mb, so the largest position
# says which unit the map is in.
if (in_mb) {
  largest <- suppressWarnings(max(unlist(cross$pmap), na.rm = TRUE))
  if (is.finite(largest) && largest < 1e6) {
    say("physical map: already in Mb (largest position", round(largest, 1), ")")
    in_mb <- FALSE
  }
}
rows <- list()
for (chr_name in names(map)) {
  positions <- map[[chr_name]]
  # qtl2 drops markers it cannot use (all-missing genotypes, say); scanning
  # them by name would be "subscript out of bounds".
  markers <- intersect(names(positions), rownames(scan))
  if (length(markers) == 0) next
  positions <- positions[markers]
  lod <- scan[markers, 1]
  if (in_mb) positions <- positions / 1e6
  rows[[length(rows) + 1]] <- data.frame(
    marker = markers, chr = chr_name, pos = round(as.numeric(positions), 6),
    lod = round(as.numeric(lod), 6), stringsAsFactors = FALSE)
}
result <- do.call(rbind, rows)
write.csv(result, opt$out, row.names = FALSE, quote = FALSE)
best <- result[which.max(result$lod), ]
say(sprintf("wrote %s: %d markers; strongest %s chr%s %.1f Mb LOD %.2f",
            opt$out, nrow(result), best$marker, best$chr, best$pos, best$lod))

# ---- permutations: the null distribution of the genome-wide maximum ---
# One permutation is a whole genome scan of a shuffled phenotype, so the
# cost is nperm x the scan above -- which is why the caller decides how
# many, and why qtl2's own `cores` argument is used here (the permutations
# fork cleanly, unlike scan1 on allele probabilities).
#
# The output is deliberately the same two-column shape R/qtl's scanone
# permutations produce ("", "lod"), so one parser reads both. Two things
# that are NOT the same as R/qtl, and are the caller's business:
#   * scan1perm permutes phenotypes across individuals, which is the right
#     null for an outbred cross and only approximate for a selfing RIL
#     panel (qtl2perm is the statistically correct package for those);
#   * thresholds at alpha depend on how many permutations there are: with
#     fewer than ~100 the 0.05 quantile is a coin toss.
if (opt$nperm > 0) {
  perm_started <- Sys.time()
  if (!is.null(opt$seed)) set.seed(opt$seed)
  strata <- NULL
  if (!is.null(opt$strata)) {
    strata_df <- read.table(opt$strata, sep = "\t", header = TRUE,
                            check.names = FALSE, stringsAsFactors = FALSE)
    strata_values <- strata_df[[2]][match(ind, strata_df$Sample)]
    if (all(is.na(strata_values))) {
      stop("none of the stratum file's samples appear in the cross", call. = FALSE)
    }
    # qtl2's perm_strata is a vector *of strata per individual*, named by
    # individual: a factor reaches get_common_ids and it answers "Not sure
    # what to do with object of class factor". Characters, names, and the
    # unassigned left as NA so qtl2 drops those individuals from the
    # permutation rather than refusing.
    strata <- stats::setNames(as.character(strata_values), ind)
    say("permutation strata:", paste(sort(unique(strata[!is.na(strata)])), collapse = ","),
        "|", sum(!is.na(strata)), "individuals assigned")
  }
  # chr_lengths from the map: without it scan1perm derives chromosome
  # lengths from the marker positions, and the thresholds stop being
  # comparable with R/qtl's on the same cross.
  lengths <- qtl2::chr_lengths(map)
  perm <- scan1perm(scan_probs, pheno, kinship = kinship, addcovar = addcovar,
                    n_perm = opt$nperm, perm_strata = strata,
                    chr_lengths = lengths, cores = scan_cores,
                    model = opt$model)
  perm_values <- as.numeric(perm)
  if (all(is.na(perm_values))) {
    # A placeholder or constant phenotype makes every permutation NaN
    # without an error, and summary() then reports "0 permutations" with NA
    # thresholds -- a plausible answer that means nothing. Refuse instead.
    stop("every permutation returned NA (a constant or placeholder phenotype?)",
         call. = FALSE)
  }
  # optparse keeps the dash in the name (`--perm-out` is opt[["perm-out"]]),
  # so opt$perm_out is always NULL: the fallback silently took over and the
  # file landed at <out>.perm while the caller read <out>-perm.csv.
  perm_path <- if (is.null(opt[["perm-out"]])) paste0(opt$out, ".perm") else opt[["perm-out"]]
  write.csv(data.frame(lod = perm_values), perm_path, row.names = FALSE,
            quote = FALSE)
  thresholds <- stats::quantile(perm_values, probs = c(0.99, 0.95, 0.63),
                                na.rm = TRUE, names = FALSE)
  say(sprintf("permutations: %d in %.1f s (%.3f s each); thresholds 0.01=%.3f 0.05=%.3f 0.37=%.3f",
              opt$nperm,
              as.numeric(difftime(Sys.time(), perm_started, units = "secs")),
              as.numeric(difftime(Sys.time(), perm_started, units = "secs")) / opt$nperm,
              thresholds[1], thresholds[2], thresholds[3]))
  say("wrote ", perm_path, ": ", length(perm_values), " permutation maxima")
}