//! One compute job at a time, per endpoint. //! //! The gate is a fixed number of slots plus a per-subject claim, which //! gives the policy the deployment wants: //! //! - **a fixed number of jobs run at a time**, across users: the slots are //! a [`tokio::sync::Semaphore`], and a waiter takes a permit in arrival //! order, so another user's request *waits* for one to come free rather //! than being refused. One slot for GEMMA (`gemma`, because two runs //! would fight over the same cache and the same cores), eight for R/qtl //! (`rqtl`, `RQTL_MAX_CONCURRENT`); //! - **and only *large* jobs take a slot.** A panel below the endpoint's //! threshold ([`GEMMA_LARGE_PANEL`], [`RQTL_LARGE_PANEL`]) runs //! alongside everything else: the gate is for protecting the machine, //! and a two-second request that waits behind a ten-minute one is worse //! off than the machine would be; //! - **one job per subject**: a second submission from somebody who //! already holds a slot *or is queued for one* is refused immediately //! (`409 compute_busy`), because queueing it would only make them wait //! twice; //! - the refusal names the job that is in the way and how long it has //! been running, so a caller knows whether to poll or to give up. //! //! The claim outlives the request deliberately: `/compute/gemma` runs a //! child process that a client disconnect does not kill (see //! [`crate::streaming`]), so the caller moves the [`Claim`] into the task //! that awaits the job (`compute_gemma`'s own comment), and the slot is //! held for exactly as long as the work is. //! //! What this is *not*: it is process-local, so it stops being the whole //! truth the moment more than one server process serves the API (there, //! the scheduler or a database row is the right home for the state), and //! it only guards the endpoints that ask it to. Each endpoint gets its own //! [`Gate`], which is what keeps the limit per endpoint rather than per //! server. use std::collections::HashSet; use std::sync::{Mutex, OnceLock}; use std::time::Instant; use tokio::sync::{Semaphore, SemaphorePermit}; /// Where R/qtl's panel size stops being small, and starts taking one of /// its slots. /// /// What the gate is for is **large computations**, not computation as /// such: a 93-strain version or an exploratory subset finishes in seconds, /// and two of them at once on this box cost nothing worth serialising -- /// while refusing them is actively harmful, because a small request then /// waits behind, or is refused by, somebody else's minutes-long run. A real /// cross that is slow (3 281 individuals x 12 000 markers is minutes of /// `scanone`) is well above this. pub const RQTL_LARGE_PANEL: usize = 1000; /// Where GEMMA's panel size stops being small -- which is almost at once. /// /// Much lower than R/qtl's, for two reasons that have nothing to do with /// how long a small run takes: /// /// - **every GEMMA run goes through one shared cache directory** /// (`GEMMA_CACHE_DIR`), where `gemma-wrapper` extracts its per-chromosome /// kinship archives by content hash. Two runs at once are two writers in /// that directory, and the archives are hundreds of megabytes; a /// half-extracted kinship that another run reads is a failure that looks /// like a GEMMA bug; /// - a real BXD WGS run is ~19 s of every core it can get (measured), and a /// UM-HET3 one writes 207 MB of kinship per chromosome. Overlapping those /// buys the caller little and costs the box the database is on. /// /// 50 rather than 0 so that a handful of individuals -- a four-way cross's /// parents, a probe of the pipeline -- still runs without waiting. pub const GEMMA_LARGE_PANEL: usize = 50; /// Whether a job over `individuals` individuals is large enough to be /// gated, for this gate's threshold. See [`RQTL_LARGE_PANEL`] and /// [`GEMMA_LARGE_PANEL`]. pub fn is_large(individuals: usize, threshold: usize) -> bool { individuals >= threshold } /// Take a lock, recovering from poisoning: the guarded data is a `HashSet` /// or an `Option`, both still valid after a panic elsewhere, and wedging /// the gate (or leaking an entry) is worse than trusting them. fn lock_recover(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { mutex .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) } /// A subject's place in the queue, from insertion until it becomes a /// [`Claim`]. /// /// This exists for one reason: `claim()` inserts the subject *before* /// awaiting the permit, and awaiting is cancellable -- a client /// disconnect drops the handler's future. Without this guard the entry /// would stay in `claimants` with nothing to remove it, and that subject /// would be refused with `compute_busy` for the life of the process. struct Reservation<'a> { gate: &'a Gate, subject: String, /// False once the [`Claim`] takes over the removal. held: bool, } impl Reservation<'_> { fn hand_over(&mut self) { self.held = false; } } impl Drop for Reservation<'_> { fn drop(&mut self) { if self.held { lock_recover(&self.gate.claimants).remove(&self.subject); } } } /// A claim on the slot -- in two stages. /// /// The decorator makes a *light* claim as soon as the request arrives /// ([`Gate::claim_light`]): cheap, and it reserves nothing, because at /// that point nobody knows whether the job is large. Once the handler has /// resolved the panel it calls [`Claim::claim_slot_or_busy`], which takes /// the slot for a large job and does nothing at all for a small one. pub struct Claim<'a> { gate: &'a Gate, subject: String, /// The run id the request named, for the refusal's `next` link and for /// a later refusal that names *this* job. run_id: Option, /// Where the 409's `detail`, `operation_id` and `next` come from -- /// carried here so a handler promotes its claim without repeating /// them. endpoint: &'static str, label: &'static str, status_prefix: &'static str, /// Held for the claim's lifetime, once taken; dropping it lets the /// next waiter in. `None` for a light claim and for a small job. permit: Option>, } impl Claim<'_> { /// Whether this claim is holding a slot. pub fn is_held(&self) -> bool { self.permit.is_some() } /// Take the endpoint's slot for a job over `individuals` individuals, /// or stay light for a small one (see [`GEMMA_LARGE_PANEL`]). /// /// This is the one gate call each compute handler makes, placed where /// the panel size is known and before the work starts. A large job may /// have to wait for another user's job to finish, and a *second* large /// job from a caller who already holds one is refused here -- queuing /// it would only make that caller wait twice. pub async fn claim_slot_or_busy( &mut self, individuals: usize, rid: crate::response::RequestInfo, ) -> Result<(), crate::error::ApiError> { if self.permit.is_some() || !self.gate.is_large(individuals) { return Ok(()); } self.claim_slot() .await .map_err(|busy| busy_error(busy, self.endpoint, self.label, self.status_prefix, rid)) } /// The slot itself: register the subject, then wait for a permit. async fn claim_slot(&mut self) -> Result<(), Busy> { { let mut claimants = lock_recover(&self.gate.claimants); if !claimants.insert(self.subject.clone()) { let running = lock_recover(&self.gate.running); let active = running.get(&self.subject); return Err(Busy { run_id: active.and_then(|active| active.run_id.clone()), elapsed: active .map(|active| active.started.elapsed()) .unwrap_or_default(), }); } } // From here to the permit, cancellation must undo the insertion. let mut reservation = Reservation { gate: self.gate, subject: self.subject.clone(), held: true, }; // Another subject's job may be running: this is where a different // user waits (indefinitely -- the request timeout is the bound). let permit = match self.gate.slot.acquire().await { Ok(permit) => permit, Err(_) => { // The semaphore is never closed in this code; the // reservation's drop undoes the insertion either way. return Err(Busy { run_id: None, elapsed: std::time::Duration::ZERO, }); } }; reservation.hand_over(); lock_recover(&self.gate.running).insert( self.subject.clone(), Active { run_id: self.run_id.clone(), started: Instant::now(), }, ); self.permit = Some(permit); Ok(()) } } impl std::fmt::Debug for Claim<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Claim") .field("subject", &self.subject) .finish_non_exhaustive() } } impl Drop for Claim<'_> { fn drop(&mut self) { // Only a *held* claim registered itself; a light one must not // delete a registration that belongs to somebody's live job. if self.permit.is_none() { return; } // Field drop order matters here: this runs before `permit` is // released, so the next waiter cannot see a stale entry or a // stale `running` record. lock_recover(&self.gate.claimants).remove(&self.subject); lock_recover(&self.gate.running).remove(&self.subject); } } /// Why a claim was refused: somebody else's job is running and this /// subject already has one of their own in the way. #[derive(Debug)] pub struct Busy { /// The run id of the job in the way, when the request that started it /// named one (`?id=`). pub run_id: Option, /// How long that job has been running. pub elapsed: std::time::Duration, } impl Busy { /// The sentence that goes in the error's `detail`. pub fn describe(&self, label: &str) -> String { let seconds = self.elapsed.as_secs(); let waited = if seconds >= 60 { format!("{}m{:02}s", seconds / 60, seconds % 60) } else { format!("{seconds}s") }; match &self.run_id { Some(run_id) => format!( "your {label} job (run '{run_id}', started {waited} ago) is still running; \ wait for it, or poll its status endpoint" ), None => format!( "your {label} job (started {waited} ago) is still running; wait for it to \ finish -- one job per caller at a time -- and pass ?id= next time so you can \ poll its status" ), } } } /// One running job, for the error message. struct Active { run_id: Option, started: Instant, } /// One endpoint's gate. pub struct Gate { /// Remembered because the semaphore only reports what is left. permits: usize, /// Panel size at which this endpoint's jobs take a slot (see /// [`RQTL_LARGE_PANEL`], [`GEMMA_LARGE_PANEL`]). large_panel: usize, slot: Semaphore, /// Subjects holding a slot *or waiting for one*. claimants: Mutex>, /// The running job per subject: a single record would name the wrong /// caller's job as soon as there is more than one slot. running: Mutex>, } impl Gate { /// A gate with `permit` slots (one, for "one job at a time") that /// starts gating at `large_panel` individuals. fn new(permits: usize, large_panel: usize) -> Self { Gate { permits, large_panel, slot: Semaphore::new(permits), claimants: Mutex::new(HashSet::new()), running: Mutex::new(std::collections::HashMap::new()), } } /// How many jobs this gate lets run at once. pub fn permits(&self) -> usize { self.permits } /// The panel size at which this endpoint's jobs take a slot. pub fn large_panel(&self) -> usize { self.large_panel } /// Whether a job over `individuals` individuals takes this gate's slot. pub fn is_large(&self, individuals: usize) -> bool { is_large(individuals, self.large_panel) } /// A claim that reserves nothing. /// /// This is what the `#[compute_gate(...)]` decorator makes when the /// request arrives: at that point the panel is not resolved, so the /// gate cannot know whether the job is large. The handler promotes it /// with [`Claim::claim_slot_or_busy`] where the size *is* known, and a /// small job never touches the semaphore at all. pub fn claim_light( &self, subject: &str, run_id: Option<&str>, endpoint: &'static str, label: &'static str, status_prefix: &'static str, ) -> Claim<'_> { Claim { gate: self, subject: subject.to_string(), run_id: run_id.map(str::to_string), endpoint, label, status_prefix, permit: None, } } /// Take the slot for `subject` unconditionally, waiting for another /// user's job to finish if need be. /// /// This is where [`Claim::claim_slot_or_busy`] ends up for a large job, /// and what this module's tests use to exercise the policy directly. /// `None` when `subject` already holds a claim (running or queued) -- /// the caller turns that into a `409`. pub async fn claim(&self, subject: &str, run_id: Option<&str>) -> Result, Busy> { let mut claim = self.claim_light(subject, run_id, "compute", "job", "/compute/status"); claim.claim_slot().await?; Ok(claim) } } impl Gate { /// Run `job` in a task that owns `claim`, so the slot is held for as /// long as the work is -- not as long as the *request* is (a client /// disconnect does not kill the child process, see /// [`crate::streaming`]). /// /// This is the last thing a gated handler does with its claim: promote /// it if the job is large ([`Claim::claim_slot_or_busy`]), then wrap the /// call that actually computes. pub async fn run(&self, claim: Claim<'static>, job: F) -> Result where T: Send + 'static, F: std::future::Future> + Send + 'static, { let task = tokio::spawn(async move { let _claim = claim; job.await }); task.await.map_err(|e| { crate::error::ApiError::new( axum::http::StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", format!("the compute job task ended unexpectedly: {e}"), "compute_gate", ) })? } } /// The `409` a handler should return for a [`Busy`] gate: the policy /// (status, code, wording, and the `next` link to the job in the way) /// lives here rather than in each endpoint, so it is covered by this /// module's tests. pub fn busy_error( busy: Busy, endpoint: &str, label: &str, status_prefix: &str, rid: crate::response::RequestInfo, ) -> crate::error::ApiError { let mut error = crate::error::ApiError::new( axum::http::StatusCode::CONFLICT, "compute_busy", "A compute job is already running", busy.describe(label), endpoint, ) .with_request_id(rid); if let Some(run_id) = busy.run_id.as_deref() { error = error.with_next(&format!("{status_prefix}/{run_id}")); } error } /// The R/qtl gate (`POST /compute/rqtl`): `RQTL_MAX_CONCURRENT` slots /// (default 8), one job per caller. /// /// More than one because an R/qtl scan is a single-threaded `Rscript` and /// the box has cores to spare, fewer than the core count because the scans /// are CPU- and memory-hungry (a 3 281-individual cross is minutes of CPU) /// and this machine also serves the database and the API. Unlike GEMMA's /// single slot, concurrent scans share no cache -- R/qtl's results are /// uncached -- so the only cost is the CPU they take. pub fn rqtl() -> &'static Gate { static RQTL: OnceLock = OnceLock::new(); RQTL.get_or_init(|| { let permits = crate::config::Config::load().rqtl_max_concurrent.max(1); Gate::new(permits, RQTL_LARGE_PANEL) }) } /// The ravanan gate (`POST /compute/gemma/ravanan`): one job at a time, /// one per caller. /// /// One because ravanan drives a whole CWL workflow of its own -- it /// already fans the per-chromosome steps out (over Slurm, when /// `RAVANAN_BATCH_SYSTEM=slurm-api`), so a second concurrent request would /// only queue behind the cluster and compete for the same scratch /// directory. Separate from [`gemma`]'s slot: the two are different /// runners of the same computation, and running one of each at once is the /// same kind of overlap `/compute/rqtl` is allowed, not a shared resource /// conflict (they use different cache directories). pub fn ravanan() -> &'static Gate { static RAVANAN: OnceLock = OnceLock::new(); RAVANAN.get_or_init(|| Gate::new(1, GEMMA_LARGE_PANEL)) } /// The R/qtl2 gate (`POST /compute/qtl2`): two at a time. /// /// Two because a scan is a single-threaded `Rscript` per chromosome and the /// probability calculation is the expensive part; more because the whole /// point of qtl2 here is the multiparent crosses, whose genotype /// probabilities cost minutes and hundreds of megabytes. Gated from /// [`RQTL_LARGE_PANEL`] upwards, like R/qtl itself. pub fn qtl2() -> &'static Gate { static QTL2: OnceLock = OnceLock::new(); QTL2.get_or_init(|| Gate::new(2, RQTL_LARGE_PANEL)) } /// The GEMMA gate (`POST /compute/gemma`). /// BulkLMM scans: two at a time. A permutation is cheap (it rotates the /// phenotypes rather than repeating the scan), so the gate treats it as a /// GEMMA-sized job rather than an R/qtl nperm-sized one. /// The BNW gate: one network search at a time, always gated. /// /// The threshold is 0 rather than a panel size because there is no such thing /// as a small BNW run: the cost is the structure search, which is exponential /// in the number of parents, not the number of individuals. A request that /// reached this gate has already passed the variable ceiling, so any single /// run deserves the one slot. pub fn bnw() -> &'static Gate { static GATE: std::sync::OnceLock = std::sync::OnceLock::new(); GATE.get_or_init(|| Gate::new(1, 0)) } pub fn bulklmm() -> &'static Gate { static GATE: std::sync::OnceLock = std::sync::OnceLock::new(); GATE.get_or_init(|| Gate::new(2, GEMMA_LARGE_PANEL)) } pub fn gemma() -> &'static Gate { static GEMMA: OnceLock = OnceLock::new(); GEMMA.get_or_init(|| Gate::new(1, GEMMA_LARGE_PANEL)) } #[cfg(test)] mod tests { use super::*; use std::time::Duration; fn leaked_gate() -> &'static Gate { Box::leak(Box::new(Gate::new(1, RQTL_LARGE_PANEL))) } fn leaked_gate_with(permits: usize) -> &'static Gate { Box::leak(Box::new(Gate::new(permits, RQTL_LARGE_PANEL))) } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_second_claim_from_the_same_subject_is_refused() { let gate = leaked_gate(); let first = gate.claim("alice", Some("a-1")).await.expect("first claim"); let busy = gate.claim("alice", None).await.expect_err("second claim"); assert_eq!(busy.run_id.as_deref(), Some("a-1")); assert!(busy.describe("gemma").contains("run 'a-1'")); drop(first); // ... and once released, the same subject can claim again. assert!(gate.claim("alice", None).await.is_ok()); } #[test] fn each_endpoint_gates_at_its_own_panel_size() { // R/qtl: a cross has to be genuinely big before it is serialised // (3 281 individuals x 12 000 markers is minutes of scanone). for small in [1, 93, 198, 235, 999] { assert!( !is_large(small, RQTL_LARGE_PANEL), "{small} individuals is not a large R/qtl cross" ); } for large in [RQTL_LARGE_PANEL, 3281, 6382, 24465] { assert!(is_large(large, RQTL_LARGE_PANEL), "{large} is a large cross"); } // GEMMA: from a real panel's worth of individuals (see // GEMMA_LARGE_PANEL -- the shared cache directory is the reason). for small in [1, 34, 49] { assert!(!is_large(small, GEMMA_LARGE_PANEL), "{small} still runs free"); } for large in [GEMMA_LARGE_PANEL, 93, 198, 235, 6382] { assert!(is_large(large, GEMMA_LARGE_PANEL), "{large} takes the slot"); } // And the endpoints really are wired to those numbers. assert_eq!(super::gemma().large_panel(), GEMMA_LARGE_PANEL); assert_eq!(super::ravanan().large_panel(), GEMMA_LARGE_PANEL); assert_eq!(super::rqtl().large_panel(), RQTL_LARGE_PANEL); } /// A light claim reserves nothing: a small job never waits and never /// refuses anybody, so two of them run at once even with one slot. #[tokio::test] async fn a_small_job_never_takes_the_slot() { // A gate that starts gating at 100 individuals, to keep the sizes // here away from the endpoints' own thresholds. let gate = Gate::new(1, 100); let mut first = gate.claim_light("alice", Some("a-1"), "compute", "job", "/status"); let mut second = gate.claim_light("alice", Some("a-2"), "compute", "job", "/status"); first .claim_slot_or_busy(34, crate::response::RequestInfo::new(None)) .await .expect("a small job is never refused"); second .claim_slot_or_busy(93, crate::response::RequestInfo::new(None)) .await .expect("nor is a second one from the same caller"); assert!(!first.is_held() && !second.is_held()); // Nothing was registered, so the slot is still free for a large job. let large = gate .claim("alice", Some("a-3")) .await .expect("the slot was never taken"); assert!(large.is_held()); } /// The same caller cannot hold two large jobs, and the second one is /// refused with a 409 naming the first. #[tokio::test] async fn a_second_large_job_from_one_caller_is_a_409() { let gate = Gate::new(1, 100); let mut first = gate.claim_light("alice", Some("a-1"), "compute_gemma", "gemma", "/compute/gemma/status"); first .claim_slot_or_busy(6382, crate::response::RequestInfo::new(None)) .await .expect("first large job"); assert!(first.is_held()); let mut second = gate.claim_light("alice", Some("a-2"), "compute_gemma", "gemma", "/compute/gemma/status"); let error = second .claim_slot_or_busy(6382, crate::response::RequestInfo::new(None)) .await .expect_err("the same caller cannot run two large jobs"); assert_eq!(error.status, axum::http::StatusCode::CONFLICT); assert_eq!(error.code, "compute_busy"); assert_eq!( error.next.as_deref(), Some("/compute/gemma/status/a-1"), "the refusal names the job in the way" ); drop(first); second .claim_slot_or_busy(6382, crate::response::RequestInfo::new(None)) .await .expect("and the slot is free once the first job ends"); } /// Another caller's large job makes this one wait rather than fail. #[tokio::test] async fn a_large_job_waits_for_another_callers() { let gate = Gate::new(1, 100); let holder = gate.claim("alice", None).await.expect("alice's claim"); let mut bob = gate.claim_light("bob", None, "compute", "job", "/status"); // While alice holds the only slot, bob's promotion cannot finish -- // and it is not refused either (no error, just no result). let blocked = tokio::time::timeout( Duration::from_millis(100), bob.claim_slot_or_busy(6382, crate::response::RequestInfo::new(None)), ) .await; assert!(blocked.is_err(), "bob waits for alice's job, and is not refused"); // Cancelling that wait leaves nothing behind (see the cancelled- // waiter test), so the slot comes free to bob when alice ends. drop(holder); bob.claim_slot_or_busy(6382, crate::response::RequestInfo::new(None)) .await .expect("the slot is free once alice's job ends"); assert!(bob.is_held(), "bob holds the slot while his job runs"); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn another_subject_waits_instead_of_being_refused() { let gate = leaked_gate(); let first = gate.claim("alice", None).await.expect("alice's claim"); let waiter = tokio::spawn(async move { let started = Instant::now(); let claim = gate.claim("bob", None).await.expect("bob's claim"); (started.elapsed(), claim) }); // Bob is parked on the slot, not refusing. tokio::time::sleep(Duration::from_millis(50)).await; assert!(!waiter.is_finished(), "bob should still be waiting"); drop(first); let (waited, _bob) = waiter.await.expect("bob's task"); assert!(waited >= Duration::from_millis(50), "bob waited: {waited:?}"); } /// The server is a multi-threaded runtime, so the gate has to be one /// shared object across worker threads, not a copy per thread: eight /// tasks on four threads must never be inside the slot together. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn concurrent_subjects_on_several_threads_still_run_one_at_a_time() { use std::sync::atomic::{AtomicUsize, Ordering}; let gate = leaked_gate(); let inside = std::sync::Arc::new(AtomicUsize::new(0)); let peak = std::sync::Arc::new(AtomicUsize::new(0)); let mut tasks = Vec::new(); for index in 0..8 { let inside = inside.clone(); let peak = peak.clone(); tasks.push(tokio::spawn(async move { let _claim = gate .claim(&format!("subject-{index}"), None) .await .expect("each subject claims at most once"); let now = inside.fetch_add(1, Ordering::SeqCst) + 1; peak.fetch_max(now, Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(5)).await; inside.fetch_sub(1, Ordering::SeqCst); })); } for task in tasks { task.await.expect("task"); } assert_eq!( peak.load(Ordering::SeqCst), 1, "only one job inside the slot at a time, across worker threads" ); } /// A client that disconnects while queued must not leave its subject /// blocked for ever: the reservation has to undo the insertion when /// the `acquire()` future is dropped. /// Eight slots (R/qtl's default): eight subjects run together, the /// ninth waits, and one subject still cannot take two. #[test] fn each_endpoint_gets_the_slot_count_it_was_given() { assert_eq!(super::gemma().permits(), 1, "one GEMMA job at a time"); assert_eq!(super::ravanan().permits(), 1, "one ravanan workflow at a time"); assert_eq!( super::rqtl().permits(), crate::config::Config::load().rqtl_max_concurrent.max(1), "R/qtl takes RQTL_MAX_CONCURRENT slots (8 by default)" ); } #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn eight_slots_run_eight_subjects_and_the_ninth_waits() { use std::sync::atomic::{AtomicUsize, Ordering}; let gate = leaked_gate_with(8); let inside = std::sync::Arc::new(AtomicUsize::new(0)); let peak = std::sync::Arc::new(AtomicUsize::new(0)); let mut tasks = Vec::new(); for index in 0..8 { let inside = inside.clone(); let peak = peak.clone(); tasks.push(tokio::spawn(async move { let claim = gate.claim(&format!("subject-{index}"), None).await.expect("a slot"); let now = inside.fetch_add(1, Ordering::SeqCst) + 1; peak.fetch_max(now, Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(80)).await; inside.fetch_sub(1, Ordering::SeqCst); drop(claim); })); } tokio::time::sleep(Duration::from_millis(30)).await; // All eight are inside, so a ninth subject is parked. let ninth = tokio::spawn(async move { gate.claim("subject-9", None).await.map(|_| ()) }); tokio::time::sleep(Duration::from_millis(30)).await; assert!(!ninth.is_finished(), "the ninth subject must wait for a slot"); // ... and while slots are free for others, one subject still may // not take a second one. let duplicate = gate.claim("subject-0", None).await; assert!(duplicate.is_err(), "one job per caller, even with slots free"); for task in tasks { task.await.expect("task"); } ninth.await.expect("the ninth's task").expect("the ninth eventually gets a slot"); assert_eq!(peak.load(Ordering::SeqCst), 8, "eight ran together"); } /// A refusal must quote *that caller's* run, not whichever job most /// recently took a slot -- with several slots, the single-record /// version named somebody else's job. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_refusal_quotes_the_callers_own_run_id() { let gate = leaked_gate_with(4); let alice = gate.claim("alice", Some("alice-1")).await.expect("alice"); let bob = gate.claim("bob", Some("bob-1")).await.expect("bob"); let busy = gate.claim("alice", None).await.expect_err("alice again"); assert_eq!(busy.run_id.as_deref(), Some("alice-1"), "not bob's job"); drop(alice); drop(bob); // ... and bob's own refusal quotes bob's run. let bob_claim = gate.claim("bob", Some("bob-2")).await.expect("bob again"); let alice_claim = gate.claim("alice", Some("alice-2")).await.expect("alice again"); let busy = gate.claim("bob", None).await.expect_err("bob twice"); assert_eq!(busy.run_id.as_deref(), Some("bob-2")); drop(bob_claim); drop(alice_claim); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_cancelled_waiter_does_not_wedge_its_subject() { let gate = leaked_gate(); let holder = gate.claim("alice", None).await.expect("alice's claim"); let waiter = tokio::spawn(async move { gate.claim("bob", None).await.map(|_| ()) }); tokio::time::sleep(Duration::from_millis(50)).await; waiter.abort(); let _ = waiter.await; drop(holder); let claim = tokio::time::timeout(Duration::from_secs(1), gate.claim("bob", None)) .await .expect("bob must not be refused after their waiter was cancelled") .expect("bob's claim"); drop(claim); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_queued_subject_cannot_queue_a_second_job() { let gate = leaked_gate(); let first = gate.claim("alice", None).await.expect("alice's claim"); let waiting = tokio::spawn(async move { let claim = gate.claim("bob", None).await.expect("bob's claim"); drop(claim); }); tokio::time::sleep(Duration::from_millis(20)).await; // Bob is waiting, so a second submission from bob is refused now // rather than queued behind the first. let busy = gate.claim("bob", None).await.expect_err("bob again"); assert!(busy.describe("gemma").contains("one job per caller")); drop(first); waiting.await.expect("bob's task"); } }