about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-11-08 13:28:27 +0000
committerPjotr <wrk@bosa.lan>2026-11-08 13:28:27 +0000
commit875d680a9c3cfe7f3b5a92f86ce211b5a5dfcbb4 (patch)
tree4b1790d6eeb321c0e5ca2fdb0acfe0a68b508865 /src
parente5c2aa0ab1e568fe91c63cd5257aee7ff08ebed9 (diff)
downloadgenenetwork4-875d680a9c3cfe7f3b5a92f86ce211b5a5dfcbb4.tar.gz
compute: one gemma job at a time, per caller, via #[compute_gate]
`POST /compute/gemma` now takes a single slot: one job runs at a time
across all callers, a second submission from the same token is refused
immediately with `409 compute_busy` (whatever it would have waited for,
its own job is already in the way), and a submission from another token
waits for the running job and then runs. Nothing else is gated -- ravanan
and /compute/rqtl are untouched.

The gate is one `tokio::sync::Semaphore(1)` (fair: waiters take the
permit in arrival order) plus a `Mutex<HashSet<String>>` of subjects
holding the slot or queued for it, in a per-process static. The claim is
moved into the spawned job task, so a client disconnect -- which does not
kill the child process -- cannot hand the slot to somebody else while the
CPU is still busy.

The endpoint carries none of that: `#[compute_gate(gemma, run_id =
params.id.as_deref())]` expands to a claim and a run, and requires the
handler to take `Ctx` and `AuthToken` (every compute endpoint is
token-only, and the token's sub is the key), so an endpoint that is not
token-gated cannot be decorated. The policy -- the 409's status, code,
wording and next link -- stays in plain Rust (`Gate::claim_or_busy`), so
the gate's own tests cover it; the attribute is decoration over two calls.

* src/compute_gate.rs: new. Gate, Claim (released on drop, including a
panic), Busy, claim/claim_or_busy/run, and four tests, two of them on a
four-thread runtime (the earlier ones ran single-threaded and so never
exercised the cross-thread hand-off; one now asserts at most one task
inside the slot across eight tasks).
* gnauth-macros/src/lib.rs: the `compute_gate` attribute, alongside the
`gnauth` markers it shares `syn`/`quote` with; arguments documented
(gate, endpoint, status, run_id) for the endpoints still to be gated.
* src/routes/gemma.rs: the decorator replaces 57 lines of claim and
spawn plumbing in compute_gemma.
* docs/www/ai-instruction.md (4a, 5): one job at a time per caller, and
that a `compute_busy` 409 means poll, not resubmit.
Diffstat (limited to 'src')
-rw-r--r--src/compute_gate.rs334
-rw-r--r--src/lib.rs1
-rw-r--r--src/routes/gemma.rs5
3 files changed, 338 insertions, 2 deletions
diff --git a/src/compute_gate.rs b/src/compute_gate.rs
new file mode 100644
index 0000000..538d293
--- /dev/null
+++ b/src/compute_gate.rs
@@ -0,0 +1,334 @@
+//! One compute job at a time, per endpoint.
+//!
+//! The gate is a single global slot plus a per-subject claim, which gives
+//! the policy the deployment wants for GEMMA:
+//!
+//! - **one job runs at a time**, across users: the slot is a
+//!   [`tokio::sync::Semaphore`] with one permit, and a waiter takes the
+//!   permit in arrival order, so another user's request *waits* for the
+//!   running job to finish rather than being refused;
+//! - **one job per subject**: a second submission from somebody who
+//!   already holds the slot *or is queued for it* is refused immediately
+//!   (`409 compute_busy`), because queueing it would only make them wait
+//!   twice;
+//! - the refusal names the job that is in the way and how long it has
+//!   been running, so a caller knows whether to poll or to give up.
+//!
+//! The claim outlives the request deliberately: `/compute/gemma` runs a
+//! child process that a client disconnect does not kill (see
+//! [`crate::streaming`]), so the caller moves the [`Claim`] into the task
+//! that awaits the job (`compute_gemma`'s own comment), and the slot is
+//! held for exactly as long as the work is.
+//!
+//! What this is *not*: it is process-local, so it stops being the whole
+//! truth the moment more than one server process serves the API (there,
+//! the scheduler or a database row is the right home for the state), and
+//! it only guards the endpoints that ask it to -- today `/compute/gemma`
+//! alone. A second endpoint gets its own [`Gate`], which is what keeps
+//! "one at a time" honest per endpoint rather than per server.
+
+use std::collections::HashSet;
+use std::sync::{Mutex, OnceLock};
+use std::time::Instant;
+
+use tokio::sync::{Semaphore, SemaphorePermit};
+
+/// A claim on the slot.
+pub struct Claim<'a> {
+    gate: &'a Gate,
+    subject: String,
+    // Held for the claim's lifetime; dropping it lets the next waiter in.
+    _permit: SemaphorePermit<'a>,
+}
+
+impl std::fmt::Debug for Claim<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Claim")
+            .field("subject", &self.subject)
+            .finish_non_exhaustive()
+    }
+}
+
+impl Drop for Claim<'_> {
+    fn drop(&mut self) {
+        if let Ok(mut claimants) = self.gate.claimants.lock() {
+            claimants.remove(&self.subject);
+        }
+        if let Ok(mut running) = self.gate.running.lock() {
+            *running = None;
+        }
+    }
+}
+
+/// Why a claim was refused: somebody else's job is running and this
+/// subject already has one of their own in the way.
+#[derive(Debug)]
+pub struct Busy {
+    /// The run id of the job in the way, when the request that started it
+    /// named one (`?id=`).
+    pub run_id: Option<String>,
+    /// How long that job has been running.
+    pub elapsed: std::time::Duration,
+}
+
+impl Busy {
+    /// The sentence that goes in the error's `detail`.
+    pub fn describe(&self, endpoint: &str) -> String {
+        let seconds = self.elapsed.as_secs();
+        let waited = if seconds >= 60 {
+            format!("{}m{:02}s", seconds / 60, seconds % 60)
+        } else {
+            format!("{seconds}s")
+        };
+        match &self.run_id {
+            Some(run_id) => format!(
+                "your {endpoint} job (run '{run_id}', started {waited} ago) is still running; \
+                 wait for it, or poll /compute/{endpoint}/status/{run_id}"
+            ),
+            None => format!(
+                "your {endpoint} job (started {waited} ago) is still running; wait for it to \
+                 finish -- one job per caller at a time -- and pass ?id= next time so you can \
+                 poll its status"
+            ),
+        }
+    }
+}
+
+/// What is running, for the error message.
+struct Active {
+    run_id: Option<String>,
+    started: Instant,
+}
+
+/// One endpoint's gate.
+pub struct Gate {
+    slot: Semaphore,
+    /// Subjects holding the slot *or waiting for it*.
+    claimants: Mutex<HashSet<String>>,
+    running: Mutex<Option<Active>>,
+}
+
+impl Gate {
+    /// A gate with `permit` slots (one, for "one job at a time").
+    fn new(permits: usize) -> Self {
+        Gate {
+            slot: Semaphore::new(permits),
+            claimants: Mutex::new(HashSet::new()),
+            running: Mutex::new(None),
+        }
+    }
+
+    /// Claim the slot for `subject`, waiting for another user's job to
+    /// finish if need be.
+    ///
+    /// `None` when `subject` already holds a claim (running or queued) --
+    /// the caller turns that into a `409`.
+    pub async fn claim(&self, subject: &str, run_id: Option<&str>) -> Result<Claim<'_>, Busy> {
+        {
+            let Ok(mut claimants) = self.claimants.lock() else {
+                // A poisoned lock means another thread panicked while
+                // holding it; refusing is safer than letting two jobs in.
+                return Err(Busy {
+                    run_id: None,
+                    elapsed: std::time::Duration::ZERO,
+                });
+            };
+            if !claimants.insert(subject.to_string()) {
+                let elapsed = self
+                    .running
+                    .lock()
+                    .ok()
+                    .and_then(|running| running.as_ref().map(|active| active.started.elapsed()))
+                    .unwrap_or_default();
+                let run_id = self
+                    .running
+                    .lock()
+                    .ok()
+                    .and_then(|running| running.as_ref().and_then(|active| active.run_id.clone()));
+                return Err(Busy { run_id, elapsed });
+            }
+        }
+
+        // Another subject's job may be running: this is where a different
+        // user waits (indefinitely -- the request timeout is the bound).
+        let permit = match self.slot.acquire().await {
+            Ok(permit) => permit,
+            Err(_) => {
+                // The semaphore is never closed in this code; be safe.
+                if let Ok(mut claimants) = self.claimants.lock() {
+                    claimants.remove(subject);
+                }
+                return Err(Busy {
+                    run_id: None,
+                    elapsed: std::time::Duration::ZERO,
+                });
+            }
+        };
+
+        if let Ok(mut running) = self.running.lock() {
+            *running = Some(Active {
+                run_id: run_id.map(str::to_string),
+                started: Instant::now(),
+            });
+        }
+        Ok(Claim {
+            gate: self,
+            subject: subject.to_string(),
+            _permit: permit,
+        })
+    }
+}
+
+impl Gate {
+    /// Claim, or the `409` a handler should return: the policy (status,
+    /// code, wording, and the `next` link to the job in the way) lives
+    /// here rather than in each endpoint, so it is covered by this
+    /// module's tests.
+    ///
+    /// `#[compute_gate(...)]` (see `gnauth-macros`) expands to a call to
+    /// this and [`Gate::run`], which is why an endpoint carries no gate
+    /// code of its own.
+    pub async fn claim_or_busy(
+        &self,
+        subject: &str,
+        run_id: Option<&str>,
+        rid: crate::response::RequestInfo,
+        endpoint: &str,
+        status_prefix: &str,
+    ) -> Result<Claim<'_>, crate::error::ApiError> {
+        self.claim(subject, run_id).await.map_err(|busy| {
+            let mut error = crate::error::ApiError::new(
+                axum::http::StatusCode::CONFLICT,
+                "compute_busy",
+                "A compute job is already running",
+                busy.describe(endpoint),
+                endpoint,
+            )
+            .with_request_id(rid);
+            if let Some(run_id) = busy.run_id.as_deref() {
+                error = error.with_next(&format!("{status_prefix}/{run_id}"));
+            }
+            error
+        })
+    }
+
+    /// Run `job` in a task that owns `claim`, so the slot is held for as
+    /// long as the work is -- not as long as the *request* is (a client
+    /// disconnect does not kill the child process, see
+    /// [`crate::streaming`]).
+    pub async fn run<T, F>(&self, claim: Claim<'static>, job: F) -> Result<T, crate::error::ApiError>
+    where
+        T: Send + 'static,
+        F: std::future::Future<Output = Result<T, crate::error::ApiError>> + Send + 'static,
+    {
+        let task = tokio::spawn(async move {
+            let _claim = claim;
+            job.await
+        });
+        task.await.map_err(|e| {
+            crate::error::ApiError::new(
+                axum::http::StatusCode::INTERNAL_SERVER_ERROR,
+                "internal_error",
+                "Internal error",
+                format!("the compute job task ended unexpectedly: {e}"),
+                "compute_gate",
+            )
+        })?
+    }
+}
+
+/// The GEMMA gate (`POST /compute/gemma`).
+pub fn gemma() -> &'static Gate {
+    static GEMMA: OnceLock<Gate> = OnceLock::new();
+    GEMMA.get_or_init(|| Gate::new(1))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::time::Duration;
+
+    fn leaked_gate() -> &'static Gate {
+        Box::leak(Box::new(Gate::new(1)))
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn a_second_claim_from_the_same_subject_is_refused() {
+        let gate = leaked_gate();
+        let first = gate.claim("alice", Some("a-1")).await.expect("first claim");
+        let busy = gate.claim("alice", None).await.expect_err("second claim");
+        assert_eq!(busy.run_id.as_deref(), Some("a-1"));
+        assert!(busy.describe("gemma").contains("run 'a-1'"));
+        drop(first);
+        // ... and once released, the same subject can claim again.
+        assert!(gate.claim("alice", None).await.is_ok());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn another_subject_waits_instead_of_being_refused() {
+        let gate = leaked_gate();
+        let first = gate.claim("alice", None).await.expect("alice's claim");
+        let waiter = tokio::spawn(async move {
+            let started = Instant::now();
+            let claim = gate.claim("bob", None).await.expect("bob's claim");
+            (started.elapsed(), claim)
+        });
+        // Bob is parked on the slot, not refusing.
+        tokio::time::sleep(Duration::from_millis(50)).await;
+        assert!(!waiter.is_finished(), "bob should still be waiting");
+        drop(first);
+        let (waited, _bob) = waiter.await.expect("bob's task");
+        assert!(waited >= Duration::from_millis(50), "bob waited: {waited:?}");
+    }
+
+    /// The server is a multi-threaded runtime, so the gate has to be one
+    /// shared object across worker threads, not a copy per thread: eight
+    /// tasks on four threads must never be inside the slot together.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn concurrent_subjects_on_several_threads_still_run_one_at_a_time() {
+        use std::sync::atomic::{AtomicUsize, Ordering};
+        let gate = leaked_gate();
+        let inside = std::sync::Arc::new(AtomicUsize::new(0));
+        let peak = std::sync::Arc::new(AtomicUsize::new(0));
+        let mut tasks = Vec::new();
+        for index in 0..8 {
+            let inside = inside.clone();
+            let peak = peak.clone();
+            tasks.push(tokio::spawn(async move {
+                let _claim = gate
+                    .claim(&format!("subject-{index}"), None)
+                    .await
+                    .expect("each subject claims at most once");
+                let now = inside.fetch_add(1, Ordering::SeqCst) + 1;
+                peak.fetch_max(now, Ordering::SeqCst);
+                tokio::time::sleep(Duration::from_millis(5)).await;
+                inside.fetch_sub(1, Ordering::SeqCst);
+            }));
+        }
+        for task in tasks {
+            task.await.expect("task");
+        }
+        assert_eq!(
+            peak.load(Ordering::SeqCst),
+            1,
+            "only one job inside the slot at a time, across worker threads"
+        );
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn a_queued_subject_cannot_queue_a_second_job() {
+        let gate = leaked_gate();
+        let first = gate.claim("alice", None).await.expect("alice's claim");
+        let waiting = tokio::spawn(async move {
+            let claim = gate.claim("bob", None).await.expect("bob's claim");
+            drop(claim);
+        });
+        tokio::time::sleep(Duration::from_millis(20)).await;
+        // Bob is waiting, so a second submission from bob is refused now
+        // rather than queued behind the first.
+        let busy = gate.claim("bob", None).await.expect_err("bob again");
+        assert!(busy.describe("gemma").contains("one job per caller"));
+        drop(first);
+        waiting.await.expect("bob's task");
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 3ccc12f..c8e13b4 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -7,6 +7,7 @@ use utoipa_axum::router::OpenApiRouter;
 use utoipa_axum::routes;
 
 pub mod auth;
+pub mod compute_gate;
 pub mod config;
 pub mod db;
 pub mod embedded;
diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs
index 4842b58..a0c470d 100644
--- a/src/routes/gemma.rs
+++ b/src/routes/gemma.rs
@@ -14,7 +14,7 @@ use axum::extract::{Path, Query};
 use axum::http::StatusCode;
 use serde::Deserialize;
 
-use gnauth_macros::gnauth;
+use gnauth_macros::{compute_gate, gnauth};
 
 use crate::auth::AuthToken;
 use crate::config::Config;
@@ -97,9 +97,10 @@ trait values, and returns the resulting rows of GEMMA's own \
         (status = 502, description = "gemma-wrapper or GEMMA failed", body = ApiError)
     ))]
 #[gnauth(token_only)]
+#[compute_gate(gemma, run_id = params.id.as_deref())]
 pub async fn compute_gemma(
     ctx: Ctx,
-    _token: AuthToken,
+    token: AuthToken,
     Query(params): Query<ComputeGemmaParams>,
     Json(request): Json<GemmaRequest>,
 ) -> Result<Json<Envelope<serde_json::Value>>, ApiError> {