1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
|
//! One compute job at a time, per endpoint.
//!
//! The gate is a fixed number of slots plus a per-subject claim, which
//! gives the policy the deployment wants:
//!
//! - **a fixed number of jobs run at a time**, across users: the slots are
//! a [`tokio::sync::Semaphore`], and a waiter takes a permit in arrival
//! order, so another user's request *waits* for one to come free rather
//! than being refused. One slot for GEMMA (`gemma`, because two runs
//! would fight over the same cache and the same cores), eight for R/qtl
//! (`rqtl`, `RQTL_MAX_CONCURRENT`);
//! - **and only *large* jobs take a slot.** A panel below the endpoint's
//! threshold ([`GEMMA_LARGE_PANEL`], [`RQTL_LARGE_PANEL`]) runs
//! alongside everything else: the gate is for protecting the machine,
//! and a two-second request that waits behind a ten-minute one is worse
//! off than the machine would be;
//! - **one job per subject**: a second submission from somebody who
//! already holds a slot *or is queued for one* is refused immediately
//! (`409 compute_busy`), because queueing it would only make them wait
//! twice;
//! - the refusal names the job that is in the way and how long it has
//! been running, so a caller knows whether to poll or to give up.
//!
//! The claim outlives the request deliberately: `/compute/gemma` runs a
//! child process that a client disconnect does not kill (see
//! [`crate::streaming`]), so the caller moves the [`Claim`] into the task
//! that awaits the job (`compute_gemma`'s own comment), and the slot is
//! held for exactly as long as the work is.
//!
//! What this is *not*: it is process-local, so it stops being the whole
//! truth the moment more than one server process serves the API (there,
//! the scheduler or a database row is the right home for the state), and
//! it only guards the endpoints that ask it to. Each endpoint gets its own
//! [`Gate`], which is what keeps the limit per endpoint rather than per
//! server.
use std::collections::HashSet;
use std::sync::{Mutex, OnceLock};
use std::time::Instant;
use tokio::sync::{Semaphore, SemaphorePermit};
/// Where R/qtl's panel size stops being small, and starts taking one of
/// its slots.
///
/// What the gate is for is **large computations**, not computation as
/// such: a 93-strain version or an exploratory subset finishes in seconds,
/// and two of them at once on this box cost nothing worth serialising --
/// while refusing them is actively harmful, because a small request then
/// waits behind, or is refused by, somebody else's minutes-long run. A real
/// cross that is slow (3 281 individuals x 12 000 markers is minutes of
/// `scanone`) is well above this.
pub const RQTL_LARGE_PANEL: usize = 1000;
/// Where GEMMA's panel size stops being small -- which is almost at once.
///
/// Much lower than R/qtl's, for two reasons that have nothing to do with
/// how long a small run takes:
///
/// - **every GEMMA run goes through one shared cache directory**
/// (`GEMMA_CACHE_DIR`), where `gemma-wrapper` extracts its per-chromosome
/// kinship archives by content hash. Two runs at once are two writers in
/// that directory, and the archives are hundreds of megabytes; a
/// half-extracted kinship that another run reads is a failure that looks
/// like a GEMMA bug;
/// - a real BXD WGS run is ~19 s of every core it can get (measured), and a
/// UM-HET3 one writes 207 MB of kinship per chromosome. Overlapping those
/// buys the caller little and costs the box the database is on.
///
/// 50 rather than 0 so that a handful of individuals -- a four-way cross's
/// parents, a probe of the pipeline -- still runs without waiting.
pub const GEMMA_LARGE_PANEL: usize = 50;
/// Whether a job over `individuals` individuals is large enough to be
/// gated, for this gate's threshold. See [`RQTL_LARGE_PANEL`] and
/// [`GEMMA_LARGE_PANEL`].
pub fn is_large(individuals: usize, threshold: usize) -> bool {
individuals >= threshold
}
/// Take a lock, recovering from poisoning: the guarded data is a `HashSet`
/// or an `Option`, both still valid after a panic elsewhere, and wedging
/// the gate (or leaking an entry) is worse than trusting them.
fn lock_recover<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
mutex
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
/// A subject's place in the queue, from insertion until it becomes a
/// [`Claim`].
///
/// This exists for one reason: `claim()` inserts the subject *before*
/// awaiting the permit, and awaiting is cancellable -- a client
/// disconnect drops the handler's future. Without this guard the entry
/// would stay in `claimants` with nothing to remove it, and that subject
/// would be refused with `compute_busy` for the life of the process.
struct Reservation<'a> {
gate: &'a Gate,
subject: String,
/// False once the [`Claim`] takes over the removal.
held: bool,
}
impl Reservation<'_> {
fn hand_over(&mut self) {
self.held = false;
}
}
impl Drop for Reservation<'_> {
fn drop(&mut self) {
if self.held {
lock_recover(&self.gate.claimants).remove(&self.subject);
}
}
}
/// A claim on the slot -- in two stages.
///
/// The decorator makes a *light* claim as soon as the request arrives
/// ([`Gate::claim_light`]): cheap, and it reserves nothing, because at
/// that point nobody knows whether the job is large. Once the handler has
/// resolved the panel it calls [`Claim::claim_slot_or_busy`], which takes
/// the slot for a large job and does nothing at all for a small one.
pub struct Claim<'a> {
gate: &'a Gate,
subject: String,
/// The run id the request named, for the refusal's `next` link and for
/// a later refusal that names *this* job.
run_id: Option<String>,
/// Where the 409's `detail`, `operation_id` and `next` come from --
/// carried here so a handler promotes its claim without repeating
/// them.
endpoint: &'static str,
label: &'static str,
status_prefix: &'static str,
/// Held for the claim's lifetime, once taken; dropping it lets the
/// next waiter in. `None` for a light claim and for a small job.
permit: Option<SemaphorePermit<'a>>,
}
impl Claim<'_> {
/// Whether this claim is holding a slot.
pub fn is_held(&self) -> bool {
self.permit.is_some()
}
/// Take the endpoint's slot for a job over `individuals` individuals,
/// or stay light for a small one (see [`GEMMA_LARGE_PANEL`]).
///
/// This is the one gate call each compute handler makes, placed where
/// the panel size is known and before the work starts. A large job may
/// have to wait for another user's job to finish, and a *second* large
/// job from a caller who already holds one is refused here -- queuing
/// it would only make that caller wait twice.
pub async fn claim_slot_or_busy(
&mut self,
individuals: usize,
rid: crate::response::RequestInfo,
) -> Result<(), crate::error::ApiError> {
if self.permit.is_some() || !self.gate.is_large(individuals) {
return Ok(());
}
self.claim_slot()
.await
.map_err(|busy| busy_error(busy, self.endpoint, self.label, self.status_prefix, rid))
}
/// The slot itself: register the subject, then wait for a permit.
async fn claim_slot(&mut self) -> Result<(), Busy> {
{
let mut claimants = lock_recover(&self.gate.claimants);
if !claimants.insert(self.subject.clone()) {
let running = lock_recover(&self.gate.running);
let active = running.get(&self.subject);
return Err(Busy {
run_id: active.and_then(|active| active.run_id.clone()),
elapsed: active
.map(|active| active.started.elapsed())
.unwrap_or_default(),
});
}
}
// From here to the permit, cancellation must undo the insertion.
let mut reservation = Reservation {
gate: self.gate,
subject: self.subject.clone(),
held: true,
};
// Another subject's job may be running: this is where a different
// user waits (indefinitely -- the request timeout is the bound).
let permit = match self.gate.slot.acquire().await {
Ok(permit) => permit,
Err(_) => {
// The semaphore is never closed in this code; the
// reservation's drop undoes the insertion either way.
return Err(Busy {
run_id: None,
elapsed: std::time::Duration::ZERO,
});
}
};
reservation.hand_over();
lock_recover(&self.gate.running).insert(
self.subject.clone(),
Active {
run_id: self.run_id.clone(),
started: Instant::now(),
},
);
self.permit = Some(permit);
Ok(())
}
}
impl std::fmt::Debug for Claim<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Claim")
.field("subject", &self.subject)
.finish_non_exhaustive()
}
}
impl Drop for Claim<'_> {
fn drop(&mut self) {
// Only a *held* claim registered itself; a light one must not
// delete a registration that belongs to somebody's live job.
if self.permit.is_none() {
return;
}
// Field drop order matters here: this runs before `permit` is
// released, so the next waiter cannot see a stale entry or a
// stale `running` record.
lock_recover(&self.gate.claimants).remove(&self.subject);
lock_recover(&self.gate.running).remove(&self.subject);
}
}
/// Why a claim was refused: somebody else's job is running and this
/// subject already has one of their own in the way.
#[derive(Debug)]
pub struct Busy {
/// The run id of the job in the way, when the request that started it
/// named one (`?id=`).
pub run_id: Option<String>,
/// How long that job has been running.
pub elapsed: std::time::Duration,
}
impl Busy {
/// The sentence that goes in the error's `detail`.
pub fn describe(&self, label: &str) -> String {
let seconds = self.elapsed.as_secs();
let waited = if seconds >= 60 {
format!("{}m{:02}s", seconds / 60, seconds % 60)
} else {
format!("{seconds}s")
};
match &self.run_id {
Some(run_id) => format!(
"your {label} job (run '{run_id}', started {waited} ago) is still running; \
wait for it, or poll its status endpoint"
),
None => format!(
"your {label} job (started {waited} ago) is still running; wait for it to \
finish -- one job per caller at a time -- and pass ?id= next time so you can \
poll its status"
),
}
}
}
/// One running job, for the error message.
struct Active {
run_id: Option<String>,
started: Instant,
}
/// One endpoint's gate.
pub struct Gate {
/// Remembered because the semaphore only reports what is left.
permits: usize,
/// Panel size at which this endpoint's jobs take a slot (see
/// [`RQTL_LARGE_PANEL`], [`GEMMA_LARGE_PANEL`]).
large_panel: usize,
slot: Semaphore,
/// Subjects holding a slot *or waiting for one*.
claimants: Mutex<HashSet<String>>,
/// The running job per subject: a single record would name the wrong
/// caller's job as soon as there is more than one slot.
running: Mutex<std::collections::HashMap<String, Active>>,
}
impl Gate {
/// A gate with `permit` slots (one, for "one job at a time") that
/// starts gating at `large_panel` individuals.
fn new(permits: usize, large_panel: usize) -> Self {
Gate {
permits,
large_panel,
slot: Semaphore::new(permits),
claimants: Mutex::new(HashSet::new()),
running: Mutex::new(std::collections::HashMap::new()),
}
}
/// How many jobs this gate lets run at once.
pub fn permits(&self) -> usize {
self.permits
}
/// The panel size at which this endpoint's jobs take a slot.
pub fn large_panel(&self) -> usize {
self.large_panel
}
/// Whether a job over `individuals` individuals takes this gate's slot.
pub fn is_large(&self, individuals: usize) -> bool {
is_large(individuals, self.large_panel)
}
/// A claim that reserves nothing.
///
/// This is what the `#[compute_gate(...)]` decorator makes when the
/// request arrives: at that point the panel is not resolved, so the
/// gate cannot know whether the job is large. The handler promotes it
/// with [`Claim::claim_slot_or_busy`] where the size *is* known, and a
/// small job never touches the semaphore at all.
pub fn claim_light(
&self,
subject: &str,
run_id: Option<&str>,
endpoint: &'static str,
label: &'static str,
status_prefix: &'static str,
) -> Claim<'_> {
Claim {
gate: self,
subject: subject.to_string(),
run_id: run_id.map(str::to_string),
endpoint,
label,
status_prefix,
permit: None,
}
}
/// Take the slot for `subject` unconditionally, waiting for another
/// user's job to finish if need be.
///
/// This is where [`Claim::claim_slot_or_busy`] ends up for a large job,
/// and what this module's tests use to exercise the policy directly.
/// `None` when `subject` already holds a claim (running or queued) --
/// the caller turns that into a `409`.
pub async fn claim(&self, subject: &str, run_id: Option<&str>) -> Result<Claim<'_>, Busy> {
let mut claim = self.claim_light(subject, run_id, "compute", "job", "/compute/status");
claim.claim_slot().await?;
Ok(claim)
}
}
impl Gate {
/// Run `job` in a task that owns `claim`, so the slot is held for as
/// long as the work is -- not as long as the *request* is (a client
/// disconnect does not kill the child process, see
/// [`crate::streaming`]).
///
/// This is the last thing a gated handler does with its claim: promote
/// it if the job is large ([`Claim::claim_slot_or_busy`]), then wrap the
/// call that actually computes.
pub async fn run<T, F>(&self, claim: Claim<'static>, job: F) -> Result<T, crate::error::ApiError>
where
T: Send + 'static,
F: std::future::Future<Output = Result<T, crate::error::ApiError>> + Send + 'static,
{
let task = tokio::spawn(async move {
let _claim = claim;
job.await
});
task.await.map_err(|e| {
crate::error::ApiError::new(
axum::http::StatusCode::INTERNAL_SERVER_ERROR,
"internal_error",
"Internal error",
format!("the compute job task ended unexpectedly: {e}"),
"compute_gate",
)
})?
}
}
/// The `409` a handler should return for a [`Busy`] gate: the policy
/// (status, code, wording, and the `next` link to the job in the way)
/// lives here rather than in each endpoint, so it is covered by this
/// module's tests.
pub fn busy_error(
busy: Busy,
endpoint: &str,
label: &str,
status_prefix: &str,
rid: crate::response::RequestInfo,
) -> crate::error::ApiError {
let mut error = crate::error::ApiError::new(
axum::http::StatusCode::CONFLICT,
"compute_busy",
"A compute job is already running",
busy.describe(label),
endpoint,
)
.with_request_id(rid);
if let Some(run_id) = busy.run_id.as_deref() {
error = error.with_next(&format!("{status_prefix}/{run_id}"));
}
error
}
/// The R/qtl gate (`POST /compute/rqtl`): `RQTL_MAX_CONCURRENT` slots
/// (default 8), one job per caller.
///
/// More than one because an R/qtl scan is a single-threaded `Rscript` and
/// the box has cores to spare, fewer than the core count because the scans
/// are CPU- and memory-hungry (a 3 281-individual cross is minutes of CPU)
/// and this machine also serves the database and the API. Unlike GEMMA's
/// single slot, concurrent scans share no cache -- R/qtl's results are
/// uncached -- so the only cost is the CPU they take.
pub fn rqtl() -> &'static Gate {
static RQTL: OnceLock<Gate> = OnceLock::new();
RQTL.get_or_init(|| {
let permits = crate::config::Config::load().rqtl_max_concurrent.max(1);
Gate::new(permits, RQTL_LARGE_PANEL)
})
}
/// The ravanan gate (`POST /compute/gemma/ravanan`): one job at a time,
/// one per caller.
///
/// One because ravanan drives a whole CWL workflow of its own -- it
/// already fans the per-chromosome steps out (over Slurm, when
/// `RAVANAN_BATCH_SYSTEM=slurm-api`), so a second concurrent request would
/// only queue behind the cluster and compete for the same scratch
/// directory. Separate from [`gemma`]'s slot: the two are different
/// runners of the same computation, and running one of each at once is the
/// same kind of overlap `/compute/rqtl` is allowed, not a shared resource
/// conflict (they use different cache directories).
pub fn ravanan() -> &'static Gate {
static RAVANAN: OnceLock<Gate> = OnceLock::new();
RAVANAN.get_or_init(|| Gate::new(1, GEMMA_LARGE_PANEL))
}
/// The R/qtl2 gate (`POST /compute/qtl2`): two at a time.
///
/// Two because a scan is a single-threaded `Rscript` per chromosome and the
/// probability calculation is the expensive part; more because the whole
/// point of qtl2 here is the multiparent crosses, whose genotype
/// probabilities cost minutes and hundreds of megabytes. Gated from
/// [`RQTL_LARGE_PANEL`] upwards, like R/qtl itself.
pub fn qtl2() -> &'static Gate {
static QTL2: OnceLock<Gate> = OnceLock::new();
QTL2.get_or_init(|| Gate::new(2, RQTL_LARGE_PANEL))
}
/// The GEMMA gate (`POST /compute/gemma`).
/// BulkLMM scans: two at a time. A permutation is cheap (it rotates the
/// phenotypes rather than repeating the scan), so the gate treats it as a
/// GEMMA-sized job rather than an R/qtl nperm-sized one.
/// The BNW gate: one network search at a time, always gated.
///
/// The threshold is 0 rather than a panel size because there is no such thing
/// as a small BNW run: the cost is the structure search, which is exponential
/// in the number of parents, not the number of individuals. A request that
/// reached this gate has already passed the variable ceiling, so any single
/// run deserves the one slot.
pub fn bnw() -> &'static Gate {
static GATE: std::sync::OnceLock<Gate> = std::sync::OnceLock::new();
GATE.get_or_init(|| Gate::new(1, 0))
}
pub fn bulklmm() -> &'static Gate {
static GATE: std::sync::OnceLock<Gate> = std::sync::OnceLock::new();
GATE.get_or_init(|| Gate::new(2, GEMMA_LARGE_PANEL))
}
pub fn gemma() -> &'static Gate {
static GEMMA: OnceLock<Gate> = OnceLock::new();
GEMMA.get_or_init(|| Gate::new(1, GEMMA_LARGE_PANEL))
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
fn leaked_gate() -> &'static Gate {
Box::leak(Box::new(Gate::new(1, RQTL_LARGE_PANEL)))
}
fn leaked_gate_with(permits: usize) -> &'static Gate {
Box::leak(Box::new(Gate::new(permits, RQTL_LARGE_PANEL)))
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_second_claim_from_the_same_subject_is_refused() {
let gate = leaked_gate();
let first = gate.claim("alice", Some("a-1")).await.expect("first claim");
let busy = gate.claim("alice", None).await.expect_err("second claim");
assert_eq!(busy.run_id.as_deref(), Some("a-1"));
assert!(busy.describe("gemma").contains("run 'a-1'"));
drop(first);
// ... and once released, the same subject can claim again.
assert!(gate.claim("alice", None).await.is_ok());
}
#[test]
fn each_endpoint_gates_at_its_own_panel_size() {
// R/qtl: a cross has to be genuinely big before it is serialised
// (3 281 individuals x 12 000 markers is minutes of scanone).
for small in [1, 93, 198, 235, 999] {
assert!(
!is_large(small, RQTL_LARGE_PANEL),
"{small} individuals is not a large R/qtl cross"
);
}
for large in [RQTL_LARGE_PANEL, 3281, 6382, 24465] {
assert!(is_large(large, RQTL_LARGE_PANEL), "{large} is a large cross");
}
// GEMMA: from a real panel's worth of individuals (see
// GEMMA_LARGE_PANEL -- the shared cache directory is the reason).
for small in [1, 34, 49] {
assert!(!is_large(small, GEMMA_LARGE_PANEL), "{small} still runs free");
}
for large in [GEMMA_LARGE_PANEL, 93, 198, 235, 6382] {
assert!(is_large(large, GEMMA_LARGE_PANEL), "{large} takes the slot");
}
// And the endpoints really are wired to those numbers.
assert_eq!(super::gemma().large_panel(), GEMMA_LARGE_PANEL);
assert_eq!(super::ravanan().large_panel(), GEMMA_LARGE_PANEL);
assert_eq!(super::rqtl().large_panel(), RQTL_LARGE_PANEL);
}
/// A light claim reserves nothing: a small job never waits and never
/// refuses anybody, so two of them run at once even with one slot.
#[tokio::test]
async fn a_small_job_never_takes_the_slot() {
// A gate that starts gating at 100 individuals, to keep the sizes
// here away from the endpoints' own thresholds.
let gate = Gate::new(1, 100);
let mut first = gate.claim_light("alice", Some("a-1"), "compute", "job", "/status");
let mut second = gate.claim_light("alice", Some("a-2"), "compute", "job", "/status");
first
.claim_slot_or_busy(34, crate::response::RequestInfo::new(None))
.await
.expect("a small job is never refused");
second
.claim_slot_or_busy(93, crate::response::RequestInfo::new(None))
.await
.expect("nor is a second one from the same caller");
assert!(!first.is_held() && !second.is_held());
// Nothing was registered, so the slot is still free for a large job.
let large = gate
.claim("alice", Some("a-3"))
.await
.expect("the slot was never taken");
assert!(large.is_held());
}
/// The same caller cannot hold two large jobs, and the second one is
/// refused with a 409 naming the first.
#[tokio::test]
async fn a_second_large_job_from_one_caller_is_a_409() {
let gate = Gate::new(1, 100);
let mut first = gate.claim_light("alice", Some("a-1"), "compute_gemma", "gemma", "/compute/gemma/status");
first
.claim_slot_or_busy(6382, crate::response::RequestInfo::new(None))
.await
.expect("first large job");
assert!(first.is_held());
let mut second = gate.claim_light("alice", Some("a-2"), "compute_gemma", "gemma", "/compute/gemma/status");
let error = second
.claim_slot_or_busy(6382, crate::response::RequestInfo::new(None))
.await
.expect_err("the same caller cannot run two large jobs");
assert_eq!(error.status, axum::http::StatusCode::CONFLICT);
assert_eq!(error.code, "compute_busy");
assert_eq!(
error.next.as_deref(),
Some("/compute/gemma/status/a-1"),
"the refusal names the job in the way"
);
drop(first);
second
.claim_slot_or_busy(6382, crate::response::RequestInfo::new(None))
.await
.expect("and the slot is free once the first job ends");
}
/// Another caller's large job makes this one wait rather than fail.
#[tokio::test]
async fn a_large_job_waits_for_another_callers() {
let gate = Gate::new(1, 100);
let holder = gate.claim("alice", None).await.expect("alice's claim");
let mut bob = gate.claim_light("bob", None, "compute", "job", "/status");
// While alice holds the only slot, bob's promotion cannot finish --
// and it is not refused either (no error, just no result).
let blocked = tokio::time::timeout(
Duration::from_millis(100),
bob.claim_slot_or_busy(6382, crate::response::RequestInfo::new(None)),
)
.await;
assert!(blocked.is_err(), "bob waits for alice's job, and is not refused");
// Cancelling that wait leaves nothing behind (see the cancelled-
// waiter test), so the slot comes free to bob when alice ends.
drop(holder);
bob.claim_slot_or_busy(6382, crate::response::RequestInfo::new(None))
.await
.expect("the slot is free once alice's job ends");
assert!(bob.is_held(), "bob holds the slot while his job runs");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn another_subject_waits_instead_of_being_refused() {
let gate = leaked_gate();
let first = gate.claim("alice", None).await.expect("alice's claim");
let waiter = tokio::spawn(async move {
let started = Instant::now();
let claim = gate.claim("bob", None).await.expect("bob's claim");
(started.elapsed(), claim)
});
// Bob is parked on the slot, not refusing.
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(!waiter.is_finished(), "bob should still be waiting");
drop(first);
let (waited, _bob) = waiter.await.expect("bob's task");
assert!(waited >= Duration::from_millis(50), "bob waited: {waited:?}");
}
/// The server is a multi-threaded runtime, so the gate has to be one
/// shared object across worker threads, not a copy per thread: eight
/// tasks on four threads must never be inside the slot together.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn concurrent_subjects_on_several_threads_still_run_one_at_a_time() {
use std::sync::atomic::{AtomicUsize, Ordering};
let gate = leaked_gate();
let inside = std::sync::Arc::new(AtomicUsize::new(0));
let peak = std::sync::Arc::new(AtomicUsize::new(0));
let mut tasks = Vec::new();
for index in 0..8 {
let inside = inside.clone();
let peak = peak.clone();
tasks.push(tokio::spawn(async move {
let _claim = gate
.claim(&format!("subject-{index}"), None)
.await
.expect("each subject claims at most once");
let now = inside.fetch_add(1, Ordering::SeqCst) + 1;
peak.fetch_max(now, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(5)).await;
inside.fetch_sub(1, Ordering::SeqCst);
}));
}
for task in tasks {
task.await.expect("task");
}
assert_eq!(
peak.load(Ordering::SeqCst),
1,
"only one job inside the slot at a time, across worker threads"
);
}
/// A client that disconnects while queued must not leave its subject
/// blocked for ever: the reservation has to undo the insertion when
/// the `acquire()` future is dropped.
/// Eight slots (R/qtl's default): eight subjects run together, the
/// ninth waits, and one subject still cannot take two.
#[test]
fn each_endpoint_gets_the_slot_count_it_was_given() {
assert_eq!(super::gemma().permits(), 1, "one GEMMA job at a time");
assert_eq!(super::ravanan().permits(), 1, "one ravanan workflow at a time");
assert_eq!(
super::rqtl().permits(),
crate::config::Config::load().rqtl_max_concurrent.max(1),
"R/qtl takes RQTL_MAX_CONCURRENT slots (8 by default)"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn eight_slots_run_eight_subjects_and_the_ninth_waits() {
use std::sync::atomic::{AtomicUsize, Ordering};
let gate = leaked_gate_with(8);
let inside = std::sync::Arc::new(AtomicUsize::new(0));
let peak = std::sync::Arc::new(AtomicUsize::new(0));
let mut tasks = Vec::new();
for index in 0..8 {
let inside = inside.clone();
let peak = peak.clone();
tasks.push(tokio::spawn(async move {
let claim = gate.claim(&format!("subject-{index}"), None).await.expect("a slot");
let now = inside.fetch_add(1, Ordering::SeqCst) + 1;
peak.fetch_max(now, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(80)).await;
inside.fetch_sub(1, Ordering::SeqCst);
drop(claim);
}));
}
tokio::time::sleep(Duration::from_millis(30)).await;
// All eight are inside, so a ninth subject is parked.
let ninth = tokio::spawn(async move { gate.claim("subject-9", None).await.map(|_| ()) });
tokio::time::sleep(Duration::from_millis(30)).await;
assert!(!ninth.is_finished(), "the ninth subject must wait for a slot");
// ... and while slots are free for others, one subject still may
// not take a second one.
let duplicate = gate.claim("subject-0", None).await;
assert!(duplicate.is_err(), "one job per caller, even with slots free");
for task in tasks {
task.await.expect("task");
}
ninth.await.expect("the ninth's task").expect("the ninth eventually gets a slot");
assert_eq!(peak.load(Ordering::SeqCst), 8, "eight ran together");
}
/// A refusal must quote *that caller's* run, not whichever job most
/// recently took a slot -- with several slots, the single-record
/// version named somebody else's job.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_refusal_quotes_the_callers_own_run_id() {
let gate = leaked_gate_with(4);
let alice = gate.claim("alice", Some("alice-1")).await.expect("alice");
let bob = gate.claim("bob", Some("bob-1")).await.expect("bob");
let busy = gate.claim("alice", None).await.expect_err("alice again");
assert_eq!(busy.run_id.as_deref(), Some("alice-1"), "not bob's job");
drop(alice);
drop(bob);
// ... and bob's own refusal quotes bob's run.
let bob_claim = gate.claim("bob", Some("bob-2")).await.expect("bob again");
let alice_claim = gate.claim("alice", Some("alice-2")).await.expect("alice again");
let busy = gate.claim("bob", None).await.expect_err("bob twice");
assert_eq!(busy.run_id.as_deref(), Some("bob-2"));
drop(bob_claim);
drop(alice_claim);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_cancelled_waiter_does_not_wedge_its_subject() {
let gate = leaked_gate();
let holder = gate.claim("alice", None).await.expect("alice's claim");
let waiter = tokio::spawn(async move { gate.claim("bob", None).await.map(|_| ()) });
tokio::time::sleep(Duration::from_millis(50)).await;
waiter.abort();
let _ = waiter.await;
drop(holder);
let claim = tokio::time::timeout(Duration::from_secs(1), gate.claim("bob", None))
.await
.expect("bob must not be refused after their waiter was cancelled")
.expect("bob's claim");
drop(claim);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_queued_subject_cannot_queue_a_second_job() {
let gate = leaked_gate();
let first = gate.claim("alice", None).await.expect("alice's claim");
let waiting = tokio::spawn(async move {
let claim = gate.claim("bob", None).await.expect("bob's claim");
drop(claim);
});
tokio::time::sleep(Duration::from_millis(20)).await;
// Bob is waiting, so a second submission from bob is refused now
// rather than queued behind the first.
let busy = gate.claim("bob", None).await.expect_err("bob again");
assert!(busy.describe("gemma").contains("one job per caller"));
drop(first);
waiting.await.expect("bob's task");
}
}
|