diff options
| author | Pjotr <wrk@bosa.lan> | 2026-11-08 13:28:27 +0000 |
|---|---|---|
| committer | Pjotr <wrk@bosa.lan> | 2026-11-08 13:28:27 +0000 |
| commit | 875d680a9c3cfe7f3b5a92f86ce211b5a5dfcbb4 (patch) | |
| tree | 4b1790d6eeb321c0e5ca2fdb0acfe0a68b508865 /src | |
| parent | e5c2aa0ab1e568fe91c63cd5257aee7ff08ebed9 (diff) | |
| download | genenetwork4-875d680a9c3cfe7f3b5a92f86ce211b5a5dfcbb4.tar.gz | |
compute: one gemma job at a time, per caller, via #[compute_gate]
`POST /compute/gemma` now takes a single slot: one job runs at a time across all callers, a second submission from the same token is refused immediately with `409 compute_busy` (whatever it would have waited for, its own job is already in the way), and a submission from another token waits for the running job and then runs. Nothing else is gated -- ravanan and /compute/rqtl are untouched. The gate is one `tokio::sync::Semaphore(1)` (fair: waiters take the permit in arrival order) plus a `Mutex<HashSet<String>>` of subjects holding the slot or queued for it, in a per-process static. The claim is moved into the spawned job task, so a client disconnect -- which does not kill the child process -- cannot hand the slot to somebody else while the CPU is still busy. The endpoint carries none of that: `#[compute_gate(gemma, run_id = params.id.as_deref())]` expands to a claim and a run, and requires the handler to take `Ctx` and `AuthToken` (every compute endpoint is token-only, and the token's sub is the key), so an endpoint that is not token-gated cannot be decorated. The policy -- the 409's status, code, wording and next link -- stays in plain Rust (`Gate::claim_or_busy`), so the gate's own tests cover it; the attribute is decoration over two calls. * src/compute_gate.rs: new. Gate, Claim (released on drop, including a panic), Busy, claim/claim_or_busy/run, and four tests, two of them on a four-thread runtime (the earlier ones ran single-threaded and so never exercised the cross-thread hand-off; one now asserts at most one task inside the slot across eight tasks). * gnauth-macros/src/lib.rs: the `compute_gate` attribute, alongside the `gnauth` markers it shares `syn`/`quote` with; arguments documented (gate, endpoint, status, run_id) for the endpoints still to be gated. * src/routes/gemma.rs: the decorator replaces 57 lines of claim and spawn plumbing in compute_gemma. * docs/www/ai-instruction.md (4a, 5): one job at a time per caller, and that a `compute_busy` 409 means poll, not resubmit.
Diffstat (limited to 'src')
| -rw-r--r-- | src/compute_gate.rs | 334 | ||||
| -rw-r--r-- | src/lib.rs | 1 | ||||
| -rw-r--r-- | src/routes/gemma.rs | 5 |
3 files changed, 338 insertions, 2 deletions
diff --git a/src/compute_gate.rs b/src/compute_gate.rs new file mode 100644 index 0000000..538d293 --- /dev/null +++ b/src/compute_gate.rs @@ -0,0 +1,334 @@ +//! One compute job at a time, per endpoint. +//! +//! The gate is a single global slot plus a per-subject claim, which gives +//! the policy the deployment wants for GEMMA: +//! +//! - **one job runs at a time**, across users: the slot is a +//! [`tokio::sync::Semaphore`] with one permit, and a waiter takes the +//! permit in arrival order, so another user's request *waits* for the +//! running job to finish rather than being refused; +//! - **one job per subject**: a second submission from somebody who +//! already holds the slot *or is queued for it* is refused immediately +//! (`409 compute_busy`), because queueing it would only make them wait +//! twice; +//! - the refusal names the job that is in the way and how long it has +//! been running, so a caller knows whether to poll or to give up. +//! +//! The claim outlives the request deliberately: `/compute/gemma` runs a +//! child process that a client disconnect does not kill (see +//! [`crate::streaming`]), so the caller moves the [`Claim`] into the task +//! that awaits the job (`compute_gemma`'s own comment), and the slot is +//! held for exactly as long as the work is. +//! +//! What this is *not*: it is process-local, so it stops being the whole +//! truth the moment more than one server process serves the API (there, +//! the scheduler or a database row is the right home for the state), and +//! it only guards the endpoints that ask it to -- today `/compute/gemma` +//! alone. A second endpoint gets its own [`Gate`], which is what keeps +//! "one at a time" honest per endpoint rather than per server. + +use std::collections::HashSet; +use std::sync::{Mutex, OnceLock}; +use std::time::Instant; + +use tokio::sync::{Semaphore, SemaphorePermit}; + +/// A claim on the slot. +pub struct Claim<'a> { + gate: &'a Gate, + subject: String, + // Held for the claim's lifetime; dropping it lets the next waiter in. + _permit: SemaphorePermit<'a>, +} + +impl std::fmt::Debug for Claim<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Claim") + .field("subject", &self.subject) + .finish_non_exhaustive() + } +} + +impl Drop for Claim<'_> { + fn drop(&mut self) { + if let Ok(mut claimants) = self.gate.claimants.lock() { + claimants.remove(&self.subject); + } + if let Ok(mut running) = self.gate.running.lock() { + *running = None; + } + } +} + +/// Why a claim was refused: somebody else's job is running and this +/// subject already has one of their own in the way. +#[derive(Debug)] +pub struct Busy { + /// The run id of the job in the way, when the request that started it + /// named one (`?id=`). + pub run_id: Option<String>, + /// How long that job has been running. + pub elapsed: std::time::Duration, +} + +impl Busy { + /// The sentence that goes in the error's `detail`. + pub fn describe(&self, endpoint: &str) -> String { + let seconds = self.elapsed.as_secs(); + let waited = if seconds >= 60 { + format!("{}m{:02}s", seconds / 60, seconds % 60) + } else { + format!("{seconds}s") + }; + match &self.run_id { + Some(run_id) => format!( + "your {endpoint} job (run '{run_id}', started {waited} ago) is still running; \ + wait for it, or poll /compute/{endpoint}/status/{run_id}" + ), + None => format!( + "your {endpoint} job (started {waited} ago) is still running; wait for it to \ + finish -- one job per caller at a time -- and pass ?id= next time so you can \ + poll its status" + ), + } + } +} + +/// What is running, for the error message. +struct Active { + run_id: Option<String>, + started: Instant, +} + +/// One endpoint's gate. +pub struct Gate { + slot: Semaphore, + /// Subjects holding the slot *or waiting for it*. + claimants: Mutex<HashSet<String>>, + running: Mutex<Option<Active>>, +} + +impl Gate { + /// A gate with `permit` slots (one, for "one job at a time"). + fn new(permits: usize) -> Self { + Gate { + slot: Semaphore::new(permits), + claimants: Mutex::new(HashSet::new()), + running: Mutex::new(None), + } + } + + /// Claim the slot for `subject`, waiting for another user's job to + /// finish if need be. + /// + /// `None` when `subject` already holds a claim (running or queued) -- + /// the caller turns that into a `409`. + pub async fn claim(&self, subject: &str, run_id: Option<&str>) -> Result<Claim<'_>, Busy> { + { + let Ok(mut claimants) = self.claimants.lock() else { + // A poisoned lock means another thread panicked while + // holding it; refusing is safer than letting two jobs in. + return Err(Busy { + run_id: None, + elapsed: std::time::Duration::ZERO, + }); + }; + if !claimants.insert(subject.to_string()) { + let elapsed = self + .running + .lock() + .ok() + .and_then(|running| running.as_ref().map(|active| active.started.elapsed())) + .unwrap_or_default(); + let run_id = self + .running + .lock() + .ok() + .and_then(|running| running.as_ref().and_then(|active| active.run_id.clone())); + return Err(Busy { run_id, elapsed }); + } + } + + // Another subject's job may be running: this is where a different + // user waits (indefinitely -- the request timeout is the bound). + let permit = match self.slot.acquire().await { + Ok(permit) => permit, + Err(_) => { + // The semaphore is never closed in this code; be safe. + if let Ok(mut claimants) = self.claimants.lock() { + claimants.remove(subject); + } + return Err(Busy { + run_id: None, + elapsed: std::time::Duration::ZERO, + }); + } + }; + + if let Ok(mut running) = self.running.lock() { + *running = Some(Active { + run_id: run_id.map(str::to_string), + started: Instant::now(), + }); + } + Ok(Claim { + gate: self, + subject: subject.to_string(), + _permit: permit, + }) + } +} + +impl Gate { + /// Claim, or the `409` a handler should return: the policy (status, + /// code, wording, and the `next` link to the job in the way) lives + /// here rather than in each endpoint, so it is covered by this + /// module's tests. + /// + /// `#[compute_gate(...)]` (see `gnauth-macros`) expands to a call to + /// this and [`Gate::run`], which is why an endpoint carries no gate + /// code of its own. + pub async fn claim_or_busy( + &self, + subject: &str, + run_id: Option<&str>, + rid: crate::response::RequestInfo, + endpoint: &str, + status_prefix: &str, + ) -> Result<Claim<'_>, crate::error::ApiError> { + self.claim(subject, run_id).await.map_err(|busy| { + let mut error = crate::error::ApiError::new( + axum::http::StatusCode::CONFLICT, + "compute_busy", + "A compute job is already running", + busy.describe(endpoint), + endpoint, + ) + .with_request_id(rid); + if let Some(run_id) = busy.run_id.as_deref() { + error = error.with_next(&format!("{status_prefix}/{run_id}")); + } + error + }) + } + + /// Run `job` in a task that owns `claim`, so the slot is held for as + /// long as the work is -- not as long as the *request* is (a client + /// disconnect does not kill the child process, see + /// [`crate::streaming`]). + pub async fn run<T, F>(&self, claim: Claim<'static>, job: F) -> Result<T, crate::error::ApiError> + where + T: Send + 'static, + F: std::future::Future<Output = Result<T, crate::error::ApiError>> + Send + 'static, + { + let task = tokio::spawn(async move { + let _claim = claim; + job.await + }); + task.await.map_err(|e| { + crate::error::ApiError::new( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "internal_error", + "Internal error", + format!("the compute job task ended unexpectedly: {e}"), + "compute_gate", + ) + })? + } +} + +/// The GEMMA gate (`POST /compute/gemma`). +pub fn gemma() -> &'static Gate { + static GEMMA: OnceLock<Gate> = OnceLock::new(); + GEMMA.get_or_init(|| Gate::new(1)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + fn leaked_gate() -> &'static Gate { + Box::leak(Box::new(Gate::new(1))) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_second_claim_from_the_same_subject_is_refused() { + let gate = leaked_gate(); + let first = gate.claim("alice", Some("a-1")).await.expect("first claim"); + let busy = gate.claim("alice", None).await.expect_err("second claim"); + assert_eq!(busy.run_id.as_deref(), Some("a-1")); + assert!(busy.describe("gemma").contains("run 'a-1'")); + drop(first); + // ... and once released, the same subject can claim again. + assert!(gate.claim("alice", None).await.is_ok()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn another_subject_waits_instead_of_being_refused() { + let gate = leaked_gate(); + let first = gate.claim("alice", None).await.expect("alice's claim"); + let waiter = tokio::spawn(async move { + let started = Instant::now(); + let claim = gate.claim("bob", None).await.expect("bob's claim"); + (started.elapsed(), claim) + }); + // Bob is parked on the slot, not refusing. + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!waiter.is_finished(), "bob should still be waiting"); + drop(first); + let (waited, _bob) = waiter.await.expect("bob's task"); + assert!(waited >= Duration::from_millis(50), "bob waited: {waited:?}"); + } + + /// The server is a multi-threaded runtime, so the gate has to be one + /// shared object across worker threads, not a copy per thread: eight + /// tasks on four threads must never be inside the slot together. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_subjects_on_several_threads_still_run_one_at_a_time() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let gate = leaked_gate(); + let inside = std::sync::Arc::new(AtomicUsize::new(0)); + let peak = std::sync::Arc::new(AtomicUsize::new(0)); + let mut tasks = Vec::new(); + for index in 0..8 { + let inside = inside.clone(); + let peak = peak.clone(); + tasks.push(tokio::spawn(async move { + let _claim = gate + .claim(&format!("subject-{index}"), None) + .await + .expect("each subject claims at most once"); + let now = inside.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(now, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(5)).await; + inside.fetch_sub(1, Ordering::SeqCst); + })); + } + for task in tasks { + task.await.expect("task"); + } + assert_eq!( + peak.load(Ordering::SeqCst), + 1, + "only one job inside the slot at a time, across worker threads" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_queued_subject_cannot_queue_a_second_job() { + let gate = leaked_gate(); + let first = gate.claim("alice", None).await.expect("alice's claim"); + let waiting = tokio::spawn(async move { + let claim = gate.claim("bob", None).await.expect("bob's claim"); + drop(claim); + }); + tokio::time::sleep(Duration::from_millis(20)).await; + // Bob is waiting, so a second submission from bob is refused now + // rather than queued behind the first. + let busy = gate.claim("bob", None).await.expect_err("bob again"); + assert!(busy.describe("gemma").contains("one job per caller")); + drop(first); + waiting.await.expect("bob's task"); + } +} diff --git a/src/lib.rs b/src/lib.rs index 3ccc12f..c8e13b4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,7 @@ use utoipa_axum::router::OpenApiRouter; use utoipa_axum::routes; pub mod auth; +pub mod compute_gate; pub mod config; pub mod db; pub mod embedded; diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs index 4842b58..a0c470d 100644 --- a/src/routes/gemma.rs +++ b/src/routes/gemma.rs @@ -14,7 +14,7 @@ use axum::extract::{Path, Query}; use axum::http::StatusCode; use serde::Deserialize; -use gnauth_macros::gnauth; +use gnauth_macros::{compute_gate, gnauth}; use crate::auth::AuthToken; use crate::config::Config; @@ -97,9 +97,10 @@ trait values, and returns the resulting rows of GEMMA's own \ (status = 502, description = "gemma-wrapper or GEMMA failed", body = ApiError) ))] #[gnauth(token_only)] +#[compute_gate(gemma, run_id = params.id.as_deref())] pub async fn compute_gemma( ctx: Ctx, - _token: AuthToken, + token: AuthToken, Query(params): Query<ComputeGemmaParams>, Json(request): Json<GemmaRequest>, ) -> Result<Json<Envelope<serde_json::Value>>, ApiError> { |
