about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-11-08 14:15:13 +0000
committerPjotr <wrk@bosa.lan>2026-11-08 14:15:24 +0000
commit51739ea82f3543ac6e28c8392d6d62957c021b3a (patch)
tree7663b917022f709b343933ca036e6f420a3a460a /src
parent1f76ef2ef19ae36882236be7c016f0ab98d31f11 (diff)
downloadgenenetwork4-51739ea82f3543ac6e28c8392d6d62957c021b3a.tar.gz
compute: gate R/qtl at eight concurrent scans, one per caller
`POST /compute/rqtl` now takes up to `RQTL_MAX_CONCURRENT` slots (default
8, read when the gate is built) with the same one-job-per-caller rule as
GEMMA: a second submission from the same token while its scan is running
or queued is refused `409 compute_busy`; a submission that finds every
slot taken waits for one to come free. Eight because an R/qtl scan is a
single-threaded Rscript on a 96-core box; not more, because a scan on a
3 000-individual cross is minutes of CPU and the same machine serves the
database. Unlike GEMMA's single slot, concurrent scans share no cache
(R/qtl results are uncached), so the only cost is the CPU they take.

Two flaws only appeared once there was more than one slot:

- the "what is running" record was a single `Option<Active>`, so a
  refusal quoted whichever job had most recently taken a slot -- robw's
  409 named JoM's run id. It is now a map keyed by subject, and
  `a_refusal_quotes_the_callers_own_run_id` pins it;
- `Busy::describe` took the endpoint identifier, so the message read
  "your rqtl job"; the decorator takes a separate `label` ("R/qtl") from
  the status path.

The test suite hit the rule itself: six compute tests in tests/rqtl.rs
share one token, and cargo runs them on several threads, so three were
refused instead of run. They now serialise on a module mutex, with a
comment saying why -- the same lesson clients need, that parallelism
means more tokens, not more requests from one.

* src/compute_gate.rs (rqtl()): eight slots from `RQTL_MAX_CONCURRENT`;
`Gate::new(permits)` was already parameterised. `running` becomes a map
keyed by subject, since one record cannot name N jobs. Module doc talks
about slots rather than a single slot. Tests: eight subjects run and the
ninth waits, a caller still may not take two even with slots free, and a
refusal quotes the caller's own run id.
* src/config.rs: RQTL_MAX_CONCURRENT, default 8, fixed at first use.
* gnauth-macros/src/lib.rs: `label = "…"`, so prose and status path are
independent.
* src/routes/rqtl.rs: the decorator, and `_token` becomes `token`, which
it needs.
* tests/rqtl.rs: compute tests serialised.
* README.md (/compute/rqtl), docs/www/ai-instruction.md (4a),
docs/www/operator-checklist.md: the numbers per endpoint, and that more
parallelism means more tokens.
Diffstat (limited to 'src')
-rw-r--r--src/compute_gate.rs148
-rw-r--r--src/config.rs12
-rw-r--r--src/routes/rqtl.rs5
3 files changed, 133 insertions, 32 deletions
diff --git a/src/compute_gate.rs b/src/compute_gate.rs
index cfff61c..6b0f2c1 100644
--- a/src/compute_gate.rs
+++ b/src/compute_gate.rs
@@ -1,14 +1,16 @@
 //! One compute job at a time, per endpoint.
 //!
-//! The gate is a single global slot plus a per-subject claim, which gives
-//! the policy the deployment wants for GEMMA:
+//! The gate is a fixed number of slots plus a per-subject claim, which
+//! gives the policy the deployment wants:
 //!
-//! - **one job runs at a time**, across users: the slot is a
-//!   [`tokio::sync::Semaphore`] with one permit, and a waiter takes the
-//!   permit in arrival order, so another user's request *waits* for the
-//!   running job to finish rather than being refused;
+//! - **a fixed number of jobs run at a time**, across users: the slots are
+//!   a [`tokio::sync::Semaphore`], and a waiter takes a permit in arrival
+//!   order, so another user's request *waits* for one to come free rather
+//!   than being refused. One slot for GEMMA (`gemma`, because two runs
+//!   would fight over the same cache and the same cores), eight for R/qtl
+//!   (`rqtl`, `RQTL_MAX_CONCURRENT`);
 //! - **one job per subject**: a second submission from somebody who
-//!   already holds the slot *or is queued for it* is refused immediately
+//!   already holds a slot *or is queued for one* is refused immediately
 //!   (`409 compute_busy`), because queueing it would only make them wait
 //!   twice;
 //! - the refusal names the job that is in the way and how long it has
@@ -23,9 +25,9 @@
 //! What this is *not*: it is process-local, so it stops being the whole
 //! truth the moment more than one server process serves the API (there,
 //! the scheduler or a database row is the right home for the state), and
-//! it only guards the endpoints that ask it to -- today `/compute/gemma`
-//! alone. A second endpoint gets its own [`Gate`], which is what keeps
-//! "one at a time" honest per endpoint rather than per server.
+//! it only guards the endpoints that ask it to. Each endpoint gets its own
+//! [`Gate`], which is what keeps the limit per endpoint rather than per
+//! server.
 
 use std::collections::HashSet;
 use std::sync::{Mutex, OnceLock};
@@ -93,7 +95,7 @@ impl Drop for Claim<'_> {
         // released, so the next waiter cannot see a stale entry or a
         // stale `running` record.
         lock_recover(&self.gate.claimants).remove(&self.subject);
-        *lock_recover(&self.gate.running) = None;
+        lock_recover(&self.gate.running).remove(&self.subject);
     }
 }
 
@@ -110,7 +112,7 @@ pub struct Busy {
 
 impl Busy {
     /// The sentence that goes in the error's `detail`.
-    pub fn describe(&self, endpoint: &str) -> String {
+    pub fn describe(&self, label: &str) -> String {
         let seconds = self.elapsed.as_secs();
         let waited = if seconds >= 60 {
             format!("{}m{:02}s", seconds / 60, seconds % 60)
@@ -119,11 +121,11 @@ impl Busy {
         };
         match &self.run_id {
             Some(run_id) => format!(
-                "your {endpoint} job (run '{run_id}', started {waited} ago) is still running; \
-                 wait for it, or poll /compute/{endpoint}/status/{run_id}"
+                "your {label} job (run '{run_id}', started {waited} ago) is still running; \
+                 wait for it, or poll its status endpoint"
             ),
             None => format!(
-                "your {endpoint} job (started {waited} ago) is still running; wait for it to \
+                "your {label} job (started {waited} ago) is still running; wait for it to \
                  finish -- one job per caller at a time -- and pass ?id= next time so you can \
                  poll its status"
             ),
@@ -131,7 +133,7 @@ impl Busy {
     }
 }
 
-/// What is running, for the error message.
+/// One running job, for the error message.
 struct Active {
     run_id: Option<String>,
     started: Instant,
@@ -140,9 +142,11 @@ struct Active {
 /// One endpoint's gate.
 pub struct Gate {
     slot: Semaphore,
-    /// Subjects holding the slot *or waiting for it*.
+    /// Subjects holding a slot *or waiting for one*.
     claimants: Mutex<HashSet<String>>,
-    running: Mutex<Option<Active>>,
+    /// The running job per subject: a single record would name the wrong
+    /// caller's job as soon as there is more than one slot.
+    running: Mutex<std::collections::HashMap<String, Active>>,
 }
 
 impl Gate {
@@ -151,7 +155,7 @@ impl Gate {
         Gate {
             slot: Semaphore::new(permits),
             claimants: Mutex::new(HashSet::new()),
-            running: Mutex::new(None),
+            running: Mutex::new(std::collections::HashMap::new()),
         }
     }
 
@@ -165,12 +169,13 @@ impl Gate {
             let mut claimants = lock_recover(&self.claimants);
             if !claimants.insert(subject.to_string()) {
                 let running = lock_recover(&self.running);
-                let elapsed = running
-                    .as_ref()
-                    .map(|active| active.started.elapsed())
-                    .unwrap_or_default();
-                let run_id = running.as_ref().and_then(|active| active.run_id.clone());
-                return Err(Busy { run_id, elapsed });
+                let active = running.get(subject);
+                return Err(Busy {
+                    run_id: active.and_then(|active| active.run_id.clone()),
+                    elapsed: active
+                        .map(|active| active.started.elapsed())
+                        .unwrap_or_default(),
+                });
             }
         }
         // From here to the permit, cancellation must undo the insertion.
@@ -197,10 +202,13 @@ impl Gate {
         // entry's removal.
         reservation.hand_over();
 
-        *lock_recover(&self.running) = Some(Active {
-            run_id: run_id.map(str::to_string),
-            started: Instant::now(),
-        });
+        lock_recover(&self.running).insert(
+            subject.to_string(),
+            Active {
+                run_id: run_id.map(str::to_string),
+                started: Instant::now(),
+            },
+        );
         Ok(Claim {
             gate: self,
             subject: subject.to_string(),
@@ -224,6 +232,7 @@ impl Gate {
         run_id: Option<&str>,
         rid: crate::response::RequestInfo,
         endpoint: &str,
+        label: &str,
         status_prefix: &str,
     ) -> Result<Claim<'_>, crate::error::ApiError> {
         self.claim(subject, run_id).await.map_err(|busy| {
@@ -231,7 +240,7 @@ impl Gate {
                 axum::http::StatusCode::CONFLICT,
                 "compute_busy",
                 "A compute job is already running",
-                busy.describe(endpoint),
+                busy.describe(label),
                 endpoint,
             )
             .with_request_id(rid);
@@ -267,6 +276,23 @@ impl Gate {
     }
 }
 
+/// The R/qtl gate (`POST /compute/rqtl`): `RQTL_MAX_CONCURRENT` slots
+/// (default 8), one job per caller.
+///
+/// More than one because an R/qtl scan is a single-threaded `Rscript` and
+/// the box has cores to spare, fewer than the core count because the scans
+/// are CPU- and memory-hungry (a 3 281-individual cross is minutes of CPU)
+/// and this machine also serves the database and the API. Unlike GEMMA's
+/// single slot, concurrent scans share no cache -- R/qtl's results are
+/// uncached -- so the only cost is the CPU they take.
+pub fn rqtl() -> &'static Gate {
+    static RQTL: OnceLock<Gate> = OnceLock::new();
+    RQTL.get_or_init(|| {
+        let permits = crate::config::Config::load().rqtl_max_concurrent.max(1);
+        Gate::new(permits)
+    })
+}
+
 /// The GEMMA gate (`POST /compute/gemma`).
 pub fn gemma() -> &'static Gate {
     static GEMMA: OnceLock<Gate> = OnceLock::new();
@@ -282,6 +308,10 @@ mod tests {
         Box::leak(Box::new(Gate::new(1)))
     }
 
+    fn leaked_gate_with(permits: usize) -> &'static Gate {
+        Box::leak(Box::new(Gate::new(permits)))
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     async fn a_second_claim_from_the_same_subject_is_refused() {
         let gate = leaked_gate();
@@ -348,6 +378,64 @@ mod tests {
     /// A client that disconnects while queued must not leave its subject
     /// blocked for ever: the reservation has to undo the insertion when
     /// the `acquire()` future is dropped.
+    /// Eight slots (R/qtl's default): eight subjects run together, the
+    /// ninth waits, and one subject still cannot take two.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn eight_slots_run_eight_subjects_and_the_ninth_waits() {
+        use std::sync::atomic::{AtomicUsize, Ordering};
+        let gate = leaked_gate_with(8);
+        let inside = std::sync::Arc::new(AtomicUsize::new(0));
+        let peak = std::sync::Arc::new(AtomicUsize::new(0));
+        let mut tasks = Vec::new();
+        for index in 0..8 {
+            let inside = inside.clone();
+            let peak = peak.clone();
+            tasks.push(tokio::spawn(async move {
+                let claim = gate.claim(&format!("subject-{index}"), None).await.expect("a slot");
+                let now = inside.fetch_add(1, Ordering::SeqCst) + 1;
+                peak.fetch_max(now, Ordering::SeqCst);
+                tokio::time::sleep(Duration::from_millis(80)).await;
+                inside.fetch_sub(1, Ordering::SeqCst);
+                drop(claim);
+            }));
+        }
+        tokio::time::sleep(Duration::from_millis(30)).await;
+        // All eight are inside, so a ninth subject is parked.
+        let ninth = tokio::spawn(async move { gate.claim("subject-9", None).await.map(|_| ()) });
+        tokio::time::sleep(Duration::from_millis(30)).await;
+        assert!(!ninth.is_finished(), "the ninth subject must wait for a slot");
+        // ... and while slots are free for others, one subject still may
+        // not take a second one.
+        let duplicate = gate.claim("subject-0", None).await;
+        assert!(duplicate.is_err(), "one job per caller, even with slots free");
+        for task in tasks {
+            task.await.expect("task");
+        }
+        ninth.await.expect("the ninth's task").expect("the ninth eventually gets a slot");
+        assert_eq!(peak.load(Ordering::SeqCst), 8, "eight ran together");
+    }
+
+    /// A refusal must quote *that caller's* run, not whichever job most
+    /// recently took a slot -- with several slots, the single-record
+    /// version named somebody else's job.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn a_refusal_quotes_the_callers_own_run_id() {
+        let gate = leaked_gate_with(4);
+        let alice = gate.claim("alice", Some("alice-1")).await.expect("alice");
+        let bob = gate.claim("bob", Some("bob-1")).await.expect("bob");
+        let busy = gate.claim("alice", None).await.expect_err("alice again");
+        assert_eq!(busy.run_id.as_deref(), Some("alice-1"), "not bob's job");
+        drop(alice);
+        drop(bob);
+        // ... and bob's own refusal quotes bob's run.
+        let bob_claim = gate.claim("bob", Some("bob-2")).await.expect("bob again");
+        let alice_claim = gate.claim("alice", Some("alice-2")).await.expect("alice again");
+        let busy = gate.claim("bob", None).await.expect_err("bob twice");
+        assert_eq!(busy.run_id.as_deref(), Some("bob-2"));
+        drop(bob_claim);
+        drop(alice_claim);
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
     async fn a_cancelled_waiter_does_not_wedge_its_subject() {
         let gate = leaked_gate();
diff --git a/src/config.rs b/src/config.rs
index 6a390f1..51c16ca 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -138,6 +138,11 @@ pub struct Config {
     /// means a connection the server closed surfaces as a failed request
     /// instead of being replaced.
     pub sql_check_health: bool,
+    /// How many `/compute/rqtl` scans may run at once
+    /// (`RQTL_MAX_CONCURRENT`, default 8), one per caller. Fixed when the
+    /// gate is built, i.e. at the first compute request.
+    pub rqtl_max_concurrent: usize,
+
     pub plot_python: String,
     /// `scripts/plot-rqtl.py`: turns a /compute/rqtl response into an SVG
     /// Manhattan plot (`RQTL_PLOTTER`).
@@ -189,6 +194,7 @@ impl Config {
         );
         values.insert("GN3_TIMEOUT_SECS".to_string(), "30".to_string());
         values.insert("SQL_CHECK_HEALTH".to_string(), "true".to_string());
+        values.insert("RQTL_MAX_CONCURRENT".to_string(), "8".to_string());
         values.insert("PLOT_PYTHON".to_string(), "python3".to_string());
         values.insert(
             "RQTL_PLOTTER".to_string(),
@@ -277,6 +283,7 @@ impl Config {
             "GN3_TIMEOUT_SECS",
             "GENOTYPE_FILES",
             "SQL_CHECK_HEALTH",
+            "RQTL_MAX_CONCURRENT",
             "PLOT_PYTHON",
             "RQTL_PLOTTER",
             "GEMMA_PLOTTER",
@@ -337,6 +344,11 @@ impl Config {
             rqtl_cache_dir: values["RQTL_CACHE_DIR"].clone(),
             genotype_files_dir: values["GENOTYPE_FILES"].clone(),
             sql_check_health: parse_switch(&values["SQL_CHECK_HEALTH"]),
+            rqtl_max_concurrent: values["RQTL_MAX_CONCURRENT"]
+                .parse()
+                .ok()
+                .filter(|permits| *permits > 0)
+                .unwrap_or(8),
             plot_python: values["PLOT_PYTHON"].clone(),
             rqtl_plotter: values["RQTL_PLOTTER"].clone(),
             gemma_plotter: values["GEMMA_PLOTTER"].clone(),
diff --git a/src/routes/rqtl.rs b/src/routes/rqtl.rs
index e8baee8..bfb979f 100644
--- a/src/routes/rqtl.rs
+++ b/src/routes/rqtl.rs
@@ -11,7 +11,7 @@ use axum::extract::{Path, Query};
 use axum::http::StatusCode;
 use serde::Deserialize;
 
-use gnauth_macros::gnauth;
+use gnauth_macros::{compute_gate, gnauth};
 
 use crate::auth::AuthToken;
 use crate::config::Config;
@@ -127,9 +127,10 @@ concurrently -- same shape as /compute/gemma.",
         (status = 502, description = "Rscript or rqtl_wrapper.R failed", body = ApiError)
     ))]
 #[gnauth(token_only)]
+#[compute_gate(rqtl, label = "R/qtl", run_id = params.id.as_deref())]
 pub async fn compute_rqtl(
     ctx: Ctx,
-    _token: AuthToken,
+    token: AuthToken,
     Query(params): Query<ComputeRqtlParams>,
     Json(request): Json<RqtlRequest>,
 ) -> Result<Json<Envelope<serde_json::Value>>, ApiError> {