1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
|
//! `POST /compute/qtl2`: a genome scan with R/qtl2.
//!
//! The middle ground between `/compute/rqtl` (Haley-Knott, no kinship) and
//! `/compute/gemma` (an LMM, but biallelic BIMBAM with mean-imputed
//! genotypes): genotype probabilities from a hidden Markov model and a
//! linear mixed model with leave-one-chromosome-out kinship. That is how
//! multiparent populations are analysed, and it is a better-conditioned
//! model than either for a plain riset panel.
//!
//! On DOL trait 10003 the three engines tell the same story with different
//! emphasis: R/qtl's HK scan peaks on chr9 at LOD 5.86 and misses chr16
//! entirely; GEMMA's LMM prefers chr16 at -log10 p 7.15; R/qtl2's LMM puts
//! chr9 (LOD 4.85), chr19 (4.76) and chr16 (4.24) on one footing. The two
//! LMMs agree on which loci exist, which is what modeling relatedness buys.
use axum::Json;
use axum::extract::Query;
use axum::http::StatusCode;
use serde::Deserialize;
use gnauth_macros::{compute_gate, gnauth};
use crate::auth::AuthToken;
use crate::compute_gate as gate;
use crate::config::Config;
use crate::covariates;
use crate::error::{ApiError, Unauthorized};
use crate::genotypes;
use crate::normalize;
use crate::qtl2;
use crate::response::{Ctx, Envelope, RequestInfo, path_segment};
use crate::rqtl::Sample;
/// Each permutation is a whole genome scan, so this is not a knob to leave
/// open: 10 000 is already thousands of scans (about five minutes measured on
/// a 34-individual panel, and far more on a multiparent cross).
const MAX_PERMUTATIONS: u32 = 10_000;
/// Below this the 0.05 threshold is a quantile of too few values to lean on.
/// A note says so; the request is still answered.
const FEW_PERMUTATIONS: u32 = 100;
/// A permutation null without a seed is not reproducible, so one is chosen and
/// reported when the caller does not supply it.
fn default_perm_seed() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|elapsed| elapsed.as_secs())
.unwrap_or(0)
}
#[derive(Deserialize, utoipa::IntoParams)]
pub struct ComputeQtl2Params {
/// Log progress for `GET /compute/rqtl/status/{id}` (shared with
/// `/compute/rqtl`, since both run R).
pub id: Option<String>,
}
/// One trait value, by individual name.
#[derive(Clone, Deserialize, utoipa::ToSchema)]
pub struct Qtl2Sample {
pub name: String,
pub value: Option<f64>,
}
#[derive(Deserialize, utoipa::ToSchema)]
pub struct Qtl2Request {
/// Group to scan (an InbredSet name, e.g. "BXD"); the server resolves
/// and converts its `.geno` file.
pub group: Option<String>,
/// A version from `/groups/{group}/genotypes`.
pub genotype_version: Option<String>,
/// Explicit `.geno` path on the server, instead of `group`.
pub genotype_file: Option<String>,
/// An uploaded genotype from `/uploads/genotypes`, in place of
/// `genotype_file`: the server writes it out and maps over it.
pub genotype_upload: Option<String>,
/// A cross that ships its own R/qtl2 control file, by name (e.g. "do").
/// This is the only route to a multiparent scan: a `.geno` panel carries
/// two codes per marker, not the founder alleles a multiparent model
/// fits. The cross's individuals are the population, so `samples` are
/// matched to it by name and anything outside it is left out.
pub rqtl2_cross: Option<String>,
/// Include parents and F1s; off by default, as elsewhere.
pub include_parents_f1: Option<bool>,
/// Individuals to keep or drop on top of the parent/F1 rule
/// (`genotypes::Individuals`); `exclude` wins over `include`.
#[serde(default)]
pub individuals: genotypes::Individuals,
/// The trait's values by individual name, or `trait_values` positionally
/// in the genotype file's order.
pub samples: Option<Vec<Qtl2Sample>>,
/// A basket from `/baskets`: a stored column of values to map instead of
/// posting `samples`. Its revision and digest are recorded in `notes`, so
/// a run on an edited basket is distinguishable from an earlier one.
pub basket: Option<String>,
/// Positional values, for a `genotype_file` request.
pub trait_values: Option<Vec<Option<f64>>>,
/// Label for the trait, e.g. "DOL::10003"; reported in `notes`.
pub trait_name: Option<String>,
/// `loco` (default), `overall` or `none` for the kinship the LMM uses;
/// `none` makes the scan a Haley-Knott scan on genotype probabilities.
pub kinship: Option<String>,
/// `normal` (default) or `binary`.
pub model: Option<String>,
/// How heterozygous calls on a `riset` panel are modelled: `riself`
/// (default -- R/qtl's `convert2riself` coding, the calls are dropped and
/// the HMM re-imputes them) or `f2` (the calls are kept as a third
/// genotype). `f2` is a diagnostic: it fits het as a real genotype but
/// also assumes an outbred intercross design, which an inbred line is not.
/// Ignored for a cross that ships its own control file.
pub genotype_model: Option<String>,
/// Deprecated spelling of `genotype_model`: `drop` -> `riself`,
/// `keep` -> `f2`.
pub het: Option<String>,
/// Cores for the per-chromosome steps (default 4, capped at 16).
pub cores: Option<u32>,
/// Covariates, as on `/compute/rqtl`: a case attribute of the group by
/// name, or inline values.
#[serde(default)]
pub covariates: Vec<covariates::Spec>,
/// Normalisation of the trait values, as on `/compute/rqtl`.
pub normalize: Option<String>,
/// Winsorise at a tail fraction, as on `/compute/rqtl`.
pub winsor_quantile: Option<f64>,
/// Winsorise at Tukey fences, as on `/compute/rqtl`.
pub winsor_fences: Option<f64>,
/// Genome-wide LOD thresholds from permutations: every permutation is
/// another whole scan of a shuffled phenotype, so the cost is this many
/// scans. 0 (default) skips them and reports no thresholds. Measured on a
/// 34-individual BXD panel over 20 chromosomes with a LOCO kinship: about
/// 0.03 s per permutation on four cores, i.e. 1000 in half a minute.
/// Fewer than ~100 makes the 0.05 quantile a coin toss, so that is
/// reported as a note rather than refused.
pub nperm: Option<u32>,
/// Seed for the permutations. A permutation null is random, so without one
/// two identical requests give different thresholds; the seed used is
/// reported in `notes` either way. Ignored when `nperm` is 0.
pub perm_seed: Option<u64>,
/// Permute within strata instead of across all individuals: a case
/// attribute of the group by name (e.g. `sex`). Permuting across sexes
/// inflates the null, so a mixed-sex panel wants this. Ignored when
/// `nperm` is 0.
pub perm_strata: Option<String>,
}
#[utoipa::path(post, path = "/compute/qtl2",
summary = "Scan a panel with R/qtl2: genotype probabilities and an LMM with LOCO kinship",
description = "Fits R/qtl2's `scan1` over genotype probabilities, with a kinship matrix \
(`loco` by default), which is the model multiparent populations need and a better-conditioned \
one than R/qtl's Haley-Knott scan. Autosomes only: the X needs a sex covariate and the Y and \
mitochondrion degenerate the per-chromosome kinship.",
params(ComputeQtl2Params),
request_body = Qtl2Request,
responses(
(status = 200, description = "One row per marker, positional (column names in meta.columns)", body = Envelope<serde_json::Value>),
Unauthorized,
(status = 409, description = "A compute job is already running for this caller", body = ApiError),
(status = 422, description = "No usable cross, trait values or normalisation", body = ApiError),
(status = 502, description = "R/qtl2 failed", body = ApiError)
)
)]
#[gnauth(token_only)]
#[compute_gate(qtl2, label = "R/qtl2", run_id = params.id.as_deref())]
pub async fn compute_qtl2(
ctx: Ctx,
token: AuthToken,
Query(params): Query<ComputeQtl2Params>,
Json(mut request): Json<Qtl2Request>,
) -> Result<Json<Envelope<serde_json::Value>>, ApiError> {
let rid = ctx.request_info();
// An uploaded genotype is just another `.geno`: materialise it and let the
// path branch below do the rest.
if let Some(upload) = request.genotype_upload.clone() {
if request.genotype_file.is_some() || request.group.is_some() {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"ambiguous_genotype",
"Ambiguous genotype input",
"give genotype_upload, genotype_file or group -- one of the three.".to_string(),
"compute_qtl2",
)
.with_request_id(rid));
}
let config = Config::load();
let (path, _) = crate::routes::genoupload::resolve_upload(
token.sub.clone(),
config.basket_dir,
config.genotype_files_dir,
upload,
"compute_qtl2",
rid.clone(),
false,
)
.await?;
request.genotype_file = Some(path);
}
let config = Config::load();
let trait_label = request
.trait_name
.clone()
.filter(|name| !name.trim().is_empty())
.unwrap_or_else(|| format!("{}::qtl2", request.group.clone().unwrap_or_default()));
let pipeline = normalize::Pipeline::parse(
request.normalize.as_deref(),
request.winsor_quantile,
request.winsor_fences,
)
.map_err(|detail| {
ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"invalid_normalization",
"Invalid normalization",
detail,
"compute_qtl2",
)
.with_request_id(rid.clone())
})?;
let kinship = request.kinship.clone().unwrap_or_else(|| "loco".to_string());
if !matches!(kinship.as_str(), "loco" | "overall" | "none") {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"invalid_kinship",
"Invalid kinship",
format!("kinship '{kinship}' not recognized; use 'loco', 'overall' or 'none'."),
"compute_qtl2",
)
.with_request_id(rid));
}
let model = request.model.clone().unwrap_or_else(|| "normal".to_string());
if !matches!(model.as_str(), "normal" | "binary") {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"invalid_model",
"Invalid model",
format!("model '{model}' not recognized; use 'normal' or 'binary'."),
"compute_qtl2",
)
.with_request_id(rid));
}
let model_word = request
.genotype_model
.clone()
.or_else(|| request.het.clone());
let het = qtl2::Het::parse(model_word.as_deref().map(|word| match word {
"riself" => "drop",
"f2" => "keep",
other => other,
}))
.map_err(|detail| {
ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"invalid_het",
"Invalid heterozygous-call policy",
detail,
"compute_qtl2",
)
.with_request_id(rid.clone())
})?;
if het == qtl2::Het::Keep && request.rqtl2_cross.is_some() {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"invalid_het",
"Invalid heterozygous-call policy",
"`het` applies to a panel this server converts; a shipped cross carries its own \
genotype coding."
.to_string(),
"compute_qtl2",
)
.with_request_id(rid));
}
let cores = request.cores.unwrap_or(4).clamp(1, 16);
let request_group = request.group.clone();
let include_parents_f1 = request.include_parents_f1.unwrap_or(false);
let individuals = request.individuals.clone();
let mut notes: Vec<String> = Vec::new();
/// Where the scan's genotypes come from.
#[derive(Clone)]
enum Source {
/// A panel the server converts into a cross and caches.
Convert(genotypes::Resolved),
/// A cross that ships a qtl2 control file of its own.
Control(String),
}
// A basket is another source for `samples`, resolved before anything else
// looks at them: a stored column and a posted one are the same thing to
// the mapping code below.
let mut basket_note: Option<String> = None;
if let Some(name) = request.basket.clone() {
if request.samples.is_some() {
return Err(crate::routes::baskets::ambiguous("compute_qtl2", rid));
}
let (resolved, note) = crate::routes::baskets::resolve_for_mapping(
token.sub.clone(),
Config::load().basket_dir,
name,
"compute_qtl2",
rid.clone(),
)
.await?;
basket_note = Some(note);
request.samples = Some(
resolved
.values
.iter()
.map(|(name, value)| Qtl2Sample {
name: name.clone(),
value: *value,
})
.collect(),
);
}
let (source, samples) = if let Some(name) = request.rqtl2_cross.clone() {
if request.group.is_some() || request.genotype_file.is_some() {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"ambiguous_genotype",
"Ambiguous genotype input",
"give rqtl2_cross, or group, or genotype_file -- one of the three.".to_string(),
"compute_qtl2",
)
.with_request_id(rid));
}
let given: Vec<Sample> = request
.samples
.clone()
.unwrap_or_default()
.into_iter()
.map(|sample| Sample {
name: sample.name,
value: sample.value,
})
.collect();
if given.is_empty() {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"missing_samples",
"No trait values",
"a shipped cross names its own individuals, so give `samples` (name and value, as /groups/{group}/traits/{id}/values returns them)."
.to_string(),
"compute_qtl2",
)
.with_request_id(rid));
}
(Source::Control(name), given)
} else {
match (request.genotype_file.clone(), request.group.clone()) {
(None, Some(group)) => {
let wanted = group.clone();
let version = request.genotype_version.clone();
let values = request
.samples
.clone()
.map(|samples| {
samples
.into_iter()
.map(|sample| (sample.name, sample.value))
.collect::<Vec<_>>()
})
.unwrap_or_default();
let directory = config.genotype_files_dir.clone();
let (resolved, aligned, matched, kept_names) =
tokio::task::spawn_blocking(move || {
let resolved = genotypes::resolve(
std::path::Path::new(&directory),
&wanted,
version.as_deref(),
)?;
let parents_f1s = genotypes::non_strains(
std::path::Path::new(&directory),
&wanted,
&resolved.panel,
);
let selection = genotypes::select_panel(
&resolved.panel,
&parents_f1s,
include_parents_f1,
&individuals,
)?;
let keep: Vec<usize> = selection.keep.clone();
let (aligned, matched) = genotypes::align(&resolved.panel, &keep, &values);
let kept_names: Vec<String> = keep
.iter()
.filter_map(|index| resolved.panel.get(*index).cloned())
.collect();
Ok::<_, genotypes::GenotypeError>((resolved, aligned, matched, kept_names))
})
.await
.map_err(|e| {
ApiError::new(
StatusCode::INTERNAL_SERVER_ERROR,
"internal_error",
"Internal error",
e.to_string(),
"compute_qtl2",
)
.with_request_id(rid.clone())
})?
.map_err(|e| genotypes_error(e, rid.clone()))?;
if matched == 0 {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"no_samples_aligned",
"No samples aligned",
format!(
"none of the supplied samples appear in genotype version '{}'.",
resolved.version
),
"compute_qtl2",
)
.with_request_id(rid));
}
notes.push(format!(
"Genotype: {} (version {}, {} individuals); {} values used.",
resolved.relative(std::path::Path::new(&config.genotype_files_dir), &resolved.geno),
resolved.version,
kept_names.len(),
aligned.iter().filter(|value| value.is_some()).count()
));
let samples: Vec<Sample> = kept_names
.into_iter()
.zip(aligned)
.map(|(name, value)| Sample { name, value })
.collect();
(Source::Convert(resolved), samples)
}
(Some(file), None) => {
let values = request.trait_values.clone().unwrap_or_default();
// Values by name are the natural thing to give for an uploaded
// genotype: the caller has a trait's values for its animals, not
// the file's column order. Positional `trait_values` still work,
// for a file whose order the caller does know.
let given: Vec<Sample> = request
.samples
.clone()
.unwrap_or_default()
.into_iter()
.map(|sample| Sample {
name: sample.name,
value: sample.value,
})
.collect();
let by_name = !given.is_empty();
if values.is_empty() && !by_name {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"missing_trait_values",
"No trait values",
"with an explicit genotype_file, give `samples` (name and value), a `basket`, \
or `trait_values` (one per individual, in the file's order)."
.to_string(),
"compute_qtl2",
)
.with_request_id(rid));
}
// A path request has no panel metadata; the cross carries its own
// individual order, so names are placeholders here.
let resolved = genotypes::Resolved {
version: std::path::Path::new(&file)
.file_stem()
.map(|stem| stem.to_string_lossy().into_owned())
.unwrap_or_default(),
title: None,
geno: std::path::PathBuf::from(&file),
bimbam_geno: None,
bimbam_snps: None,
panel: Vec::new(),
};
// The cross's own individual order is the only name source a
// path request has; the values are positional over it, so they
// are named after the conversion (below), not here.
let samples: Vec<Sample> = if by_name {
given
} else {
values
.into_iter()
.enumerate()
.map(|(index, value)| Sample {
name: format!("individual-{}", index + 1),
value,
})
.collect()
};
(Source::Convert(resolved), samples)
}
(Some(_), Some(_)) => {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"ambiguous_genotype",
"Ambiguous genotype input",
"give either genotype_file or group, not both.".to_string(),
"compute_qtl2",
)
.with_request_id(rid));
}
(None, None) => {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"missing_genotype",
"No genotype input",
"give rqtl2_cross, genotype_file, or group (see /groups/{group}/genotypes)."
.to_string(),
"compute_qtl2",
)
.with_request_id(rid));
}
}
};
// Normalise after alignment, over the individuals that enter the scan.
let samples = if pipeline.is_identity() {
samples
} else {
let raw: Vec<Option<f64>> = samples.iter().map(|sample| sample.value).collect();
let (normalized, applied) = pipeline.apply(&raw).map_err(|detail| {
ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"invalid_normalization",
"Invalid normalization",
detail,
"compute_qtl2",
)
.with_request_id(rid.clone())
})?;
notes.push(pipeline.describe(&applied));
samples
.into_iter()
.zip(normalized)
.map(|(sample, value)| Sample { name: sample.name, value })
.collect()
};
let (covariate_list, covariate_notes) = covariates::resolve(
&request.covariates,
&samples.iter().map(|s| s.name.clone()).collect::<Vec<_>>(),
request_group.as_deref(),
Some(&samples.iter().map(|s| s.value.is_some()).collect::<Vec<_>>()),
rid.clone(),
)
.await?;
notes.extend(covariate_notes);
// ---- permutations: how many, which seed, and permute within what -----
let permutations = request.nperm.unwrap_or(0);
if permutations > MAX_PERMUTATIONS {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"too_many_permutations",
"Too many permutations",
format!(
"{} permutations is more than the {} allowed: each one is a whole genome \
scan, so this is already thousands of scans.",
permutations, MAX_PERMUTATIONS
),
"compute_qtl2",
)
.with_request_id(rid.clone()));
}
let perm_seed = request.perm_seed.unwrap_or_else(default_perm_seed);
let mut strata_pairs: Vec<(String, String)> = Vec::new();
let mut strata_name = String::new();
if permutations > 0 {
if permutations < FEW_PERMUTATIONS {
notes.push(format!(
"{permutations} permutations is few: the 0.05 threshold is a quantile of only {permutations} values, so treat it as indicative. 1000 is the usual number."
));
}
if let Some(attribute) = request.perm_strata.as_deref() {
let group = request_group.as_deref().ok_or_else(|| {
ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"missing_group",
"No group for permutation strata",
"`perm_strata` names a case attribute of the group, so `group` is required."
.to_string(),
"compute_qtl2",
)
.with_request_id(rid.clone())
})?;
let group_owned = group.to_string();
let attribute_owned = attribute.to_string();
let (mut values, available) = tokio::task::spawn_blocking(move || {
let found = crate::db::get_group_by_name(&group_owned)?;
let Some(found) = found else {
return Ok::<(std::collections::HashMap<String, String>, Vec<String>),
crate::db::DbError>(
(std::collections::HashMap::new(), Vec::new()),
);
};
let attributes = crate::db::get_group_case_attributes(found.id)?;
let names = attributes.names.clone();
let mut by_strain = std::collections::HashMap::new();
for entry in attributes.values {
if let Some(value) = entry.case_attributes.get(&attribute_owned) {
// An empty cell is not a stratum, it is a missing value.
if !value.trim().is_empty() {
by_strain.insert(entry.strain, value.trim().to_string());
}
}
}
Ok((by_strain, names))
})
.await
.map_err(|error| {
ApiError::new(
StatusCode::INTERNAL_SERVER_ERROR,
"db_failure",
"Database failure",
format!("could not read the group's case attributes: {error}"),
"compute_qtl2",
)
.with_request_id(rid.clone())
})?
.map_err(|error| {
ApiError::new(
StatusCode::BAD_GATEWAY,
"db_failure",
"Database failure",
error.to_string(),
"compute_qtl2",
)
.with_request_id(rid.clone())
})?;
// Only individuals with a value are scanned, so only they need a
// stratum: counting the whole sample list made the note claim
// "195 of 198" for a request whose scan used 34.
let phenotyped: Vec<&Sample> =
samples.iter().filter(|sample| sample.value.is_some()).collect();
for sample in &phenotyped {
if let Some(stratum) = values.remove(&sample.name) {
strata_pairs.push((sample.name.clone(), stratum));
}
}
if strata_pairs.is_empty() {
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"invalid_perm_strata",
"Invalid permutation strata",
format!(
"no individual in this request has a value for case attribute '{}'. \
The group's attributes are: {}.",
attribute,
if available.is_empty() {
"none".to_string()
} else {
available.join(", ")
}
),
"compute_qtl2",
)
.with_request_id(rid.clone()));
}
strata_name = attribute.to_string();
notes.push(format!(
"permutations are stratified by '{}': {} of {} individuals have a value, and an \
individual without one is permuted with the rest.",
attribute,
strata_pairs.len(),
samples.len()
));
}
}
let strata_pairs_for_job = strata_pairs.clone();
let strata_name_for_job = strata_name.clone();
// The cross: converted from the .geno once, then cached.
let cache_dir = config.qtl2_cache_dir.clone();
let source_for_cross = source.clone();
let genotype_root = config.genotype_files_dir.clone();
let cache_dir_for_cross = cache_dir.clone();
let het_for_cross = het;
let cross = tokio::task::spawn_blocking(move || match source_for_cross {
Source::Convert(resolved) => {
qtl2::prepare_cross(&resolved, std::path::Path::new(&cache_dir_for_cross), het_for_cross)
}
Source::Control(name) => {
qtl2::prepare_control_cross(std::path::Path::new(&genotype_root), &name)
}
})
.await
.map_err(|e| {
ApiError::new(
StatusCode::INTERNAL_SERVER_ERROR,
"internal_error",
"Internal error",
e.to_string(),
"compute_qtl2",
)
.with_request_id(rid.clone())
})?
.map_err(|e| match e {
qtl2::Qtl2Error::Unsupported(detail) => ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"unsupported_cross",
"Unsupported cross",
detail,
"compute_qtl2",
)
.with_request_id(rid.clone()),
other => ApiError::new(
StatusCode::BAD_GATEWAY,
"qtl2_failed",
"R/qtl2 failed",
other.to_string(),
"compute_qtl2",
)
.with_request_id(rid.clone()),
})?;
let samples: Vec<Sample> = match &source {
// A path request's values were positional: give them the cross's own
// individual names, which is what the script aligns on.
// A path request's values were positional (or already named, for a
// basket or `samples`): align them onto the cross either way, so the
// script sees one entry per individual and missing ones as `None`.
Source::Convert(_) if request_group.is_none() => {
let positional = samples
.iter()
.all(|sample| sample.name.starts_with("individual-"));
if positional && samples.len() == cross.individuals.len() {
samples
.into_iter()
.zip(cross.individuals.iter())
.map(|(sample, name)| Sample {
name: name.clone(),
value: sample.value,
})
.collect()
} else {
let by_name: std::collections::HashMap<&str, Option<f64>> = samples
.iter()
.map(|sample| (sample.name.as_str(), sample.value))
.collect();
cross
.individuals
.iter()
.map(|name| Sample {
name: name.clone(),
value: by_name.get(name.as_str()).copied().flatten(),
})
.collect()
}
}
Source::Convert(_) => samples,
// A shipped cross is its own population: keep the values for the
// individuals it holds, and say how many fell outside it.
Source::Control(_) => {
let in_cross: std::collections::HashSet<&str> =
cross.individuals.iter().map(String::as_str).collect();
let given = samples.len();
let kept: Vec<Sample> = samples
.into_iter()
.filter(|sample| in_cross.contains(sample.name.as_str()))
.collect();
if kept.is_empty() {
let example = cross
.individuals
.iter()
.take(3)
.cloned()
.collect::<Vec<_>>()
.join(", ");
return Err(ApiError::new(
StatusCode::UNPROCESSABLE_ENTITY,
"no_samples_in_cross",
"No trait values for this cross",
format!(
"none of the {given} sample name(s) are individuals of this cross; they look like {example}"
),
"compute_qtl2",
)
.with_request_id(rid));
}
if kept.len() < given {
notes.push(format!(
"{} of {given} sample(s) are not in the cross and were left out; the scan \
fits {} individuals",
given - kept.len(),
kept.len()
));
}
kept
}
};
let mut cross_note = format!(
"Cross: {} ({} individuals, {} autosomal markers{}",
cross.crosstype,
cross.individuals.len(),
cross.markers,
if cross.non_autosomal > 0 {
format!(", {} non-autosomal left out", cross.non_autosomal)
} else {
String::new()
}
);
if cross.multiparent {
// Worth saying plainly: a multiparent LOD is a different claim from a
// biallelic one, and reading it as a strain effect is the usual mistake.
cross_note.push_str(&format!(
"; {} founder alleles, so every marker is fitted with one coefficient per founder -- \
the LOD asks whether any founder differs, not which. Per-founder effect sizes need \
scan1coef, which this endpoint does not return yet",
cross.founders
));
}
cross_note.push(')');
notes.push(cross_note);
if let Some(note) = basket_note {
notes.push(note);
}
if het == qtl2::Het::Drop && cross.het_dropped > 0 && cross.cells > 0 {
// The one fact that explains most disagreements with R/qtl on a riset
// panel, stated with this panel's own numbers.
let share = cross.het_dropped as f64 / cross.cells as f64 * 100.0;
notes.push(format!(
"genotype_model riself: {} heterozygous call(s) ({share:.1} %) were dropped before the \
scan -- R/qtl's convert2riself keeps two of the three codes and drops the third the \
same way -- and the HMM re-imputes them from flanking markers. On a marker that is \
mostly H the two engines then fit different predictors; genotype_model: \"f2\" keeps \
the calls as a third genotype and shows what that costs or recovers.",
cross.het_dropped
));
}
if het == qtl2::Het::Keep {
notes.push(
"Heterozygous calls were kept as a third genotype, so the cross is fitted as `f2` \
rather than `riself`. That is a diagnostic: an inbred line has no heterozygotes, \
and a het state modeled as real is only meaningful where the calls are."
.to_string(),
);
}
// A multiparent cross is fitted per founder allele: the DO's scan is
// eight times the work of the same panel scanned biallelically, so the
// gate is handed that weight rather than the individual count.
let weight = if cross.multiparent {
samples.len() * cross.founders.max(1)
} else {
samples.len()
};
__gn4_claim.claim_slot_or_busy(weight, rid.clone()).await?;
let script = config.qtl2_script_path.clone();
let kinship_for_note = kinship.clone();
let model_for_note = model.clone();
let trait_label_for_job = trait_label.clone();
let rid_for_job = rid.clone();
let output = gate::qtl2()
.run(__gn4_claim, async move {
qtl2::compute(
&script,
&cache_dir,
&cross,
&trait_label_for_job,
&samples,
&covariate_list,
&kinship,
&model,
cores,
qtl2::Permutations {
count: permutations,
seed: perm_seed,
samples: &strata_pairs_for_job,
strata_name: &strata_name_for_job,
},
)
.await
.map_err(|e| {
ApiError::new(
StatusCode::BAD_GATEWAY,
"qtl2_failed",
"R/qtl2 failed",
e.to_string(),
"compute_qtl2",
)
.with_request_id(rid_for_job)
})
})
.await?;
let num_rows = output.markers;
// The strongest marker, for a PheWAS link that answers 200: a bare
// /phewas wants a locus.
let peak_marker = crate::suggest::peak_marker_of(&output.rows, 0, 3);
let mut envelope = ctx
.envelope(output.rows)
.with_columns(
qtl2::COLUMNS.iter().map(|name| name.to_string()).collect(),
num_rows,
)
.with_note(&format!("Trait: {trait_label}"))
.with_note(&format!(
"Rows are positional: marker, chr, pos (Mb), lod. R/qtl2's scan1 over genotype \
probabilities, with kinship '{kinship_for_note}' and model '{model_for_note}'; the \
genotype probabilities use {cores} core(s). LOD is -log10 of the LMM's p-value at \
that marker. Autosomes only -- the X needs a sex covariate and the Y and \
mitochondrion degenerate the per-chromosome kinship."
))
.with_note(&output.command);
if let Some(permutation) = &output.permutation {
envelope = envelope.with_meta(
"permutation",
serde_json::json!({
"n": permutation.values.len(),
"seed": perm_seed,
"significant": permutation.significant,
"suggestive": permutation.suggestive,
"values": permutation.values,
}),
);
envelope = envelope.with_note(&format!(
"Permutation thresholds: significant (0.05) LOD {:.3}, suggestive (0.37) LOD {:.3}, from {} genome-wide maxima. `permutation` carries every maximum, so a histogram can be drawn from this response alone.",
permutation.significant,
permutation.suggestive,
permutation.values.len()
));
envelope = envelope.with_note(
"The permutations are qtl2's scan1perm: phenotypes are shuffled across individuals, \
which is the right null for an outbred cross and only approximate for a selfing RIL \
panel. `perm_strata` restricts the shuffle to one stratum (e.g. sex); the seed is \
reported so the run can be repeated.",
);
}
if let Some(note) = &output.permutation_note {
envelope = envelope.with_note(&format!("permutations: {note}"));
}
if !output.summary.trim().is_empty() {
envelope = envelope.with_note(&format!("qtl2: {}", output.summary.trim()));
}
envelope = envelope.with_note(
"On a riself (riset) panel heterozygous calls are dropped before the scan -- R/qtl's \
convert2riself -- and the HMM re-imputes them from flanking markers; GEMMA keeps them \
as dosage 0.5. A marker that is mostly H therefore cannot produce a peak here, and two \
engines can name different markers of one region without either being wrong. The \
classes link in `next` shows the per-class counts.",
);
for note in ¬es {
envelope = envelope.with_note(note);
}
if let Some(group) = request_group {
envelope = envelope.with_action(crate::response::SuggestedCall {
method: "POST".to_string(),
href: "/plot/rqtl".to_string(),
description: "Draw this scan: the rows are the same shape /plot/rqtl takes (marker, \
chr, pos, lod), so posting this response back gives a Manhattan plot \
with no reshaping."
.to_string(),
body: None,
});
if let Some(peak) = &peak_marker {
envelope = envelope.with_next(
&format!(
"/groups/{}/phewas?locus={}",
path_segment(&group),
path_segment(peak)
),
"PheWAS: every published trait peaking at this scan's strongest marker",
);
}
let version = request
.genotype_version
.clone()
.unwrap_or_else(|| "{version}".to_string());
if let Some(peak) = &peak_marker {
let (href, description) =
crate::suggest::class_hint(&group, &version, peak);
envelope = envelope.with_next(&href, description);
}
envelope = envelope.with_links(crate::suggest::group_links(&group));
}
Ok(Json(envelope))
}
fn genotypes_error(error: genotypes::GenotypeError, rid: RequestInfo) -> ApiError {
let (status, code, title) = match error {
genotypes::GenotypeError::Invalid(_) => (
StatusCode::UNPROCESSABLE_ENTITY,
"invalid_individuals",
"Unusable individual selection",
),
genotypes::GenotypeError::NotFound(_) => (
StatusCode::NOT_FOUND,
"genotypes_not_found",
"Genotype file not found",
),
genotypes::GenotypeError::Io(_) => (
StatusCode::INTERNAL_SERVER_ERROR,
"genotypes_unreadable",
"Genotype files could not be read",
),
};
ApiError::new(status, code, title, error.to_string(), "compute_qtl2").with_request_id(rid)
}
|