about summary refs log tree commit diff
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-11-09 00:48:27 +0000
committerPjotr <wrk@bosa.lan>2026-11-09 00:48:27 +0000
commit731e085c42e22104f4cba9975a33f5d22b058caf (patch)
tree34419f60994c54eebba4c246419fe7dac1b7d885
parentab5fef05237309667dd2ca7f835a5fbf9c71a12a (diff)
downloadgenenetwork4-731e085c42e22104f4cba9975a33f5d22b058caf.tar.gz
errors: log every 5xx, and remember the last database failure
A user reported that the token was fine and the service was up, but
"everything backed by the relational database is failing" -- while
`/search` (Xapian) and `/version` answered. By the time I looked, every
data endpoint answered 200 and `/version` said `database: reachable`, and
there was **no record anywhere** of what they had hit: the service logged
nothing at all, so a caller's report could not be matched to a cause, and
`/version`'s `database` is a *now* probe that says nothing about the
minute before.

Two things now make that diagnosable:

* every 5xx is written to stderr as one JSON line -- status, code,
  operation_id, **request_id**, took_ms and a truncated detail -- so
  "500 at 12:31, request id ..." can be looked up. 4xx stay unlogged: they
  are the caller's business, and a 401/422 stream would drown this;
* the last database failure is remembered (`db::record_failure`, called
  from `ApiError::from_db`, the one place every database error passes
  through) and `/version` reports it in `notes` for ten minutes, with its
  kind (`unavailable`, `connection_lost`, `query`) and the driver's
  message. That is the gap that misled the user: a healthy probe beside
  failing endpoints looks like an auth problem from outside.

`is_connection_loss` gained a `_detail` form so the caller that already
has the rendered message does not render it twice.

* src/error.rs: the log line, and recording failures by kind;
* src/db.rs: `FailureKind`, `Failure`, `record_failure`, `recent_failure`,
  and two unit tests (a failure is remembered and ages out; a driver
  message is classified correctly);
* src/lib.rs: the `/version` note, absent unless something failed recently.

Checked against a deliberately broken `SQL_URI` on a scratch instance: the
data endpoint answers `503 db_unavailable`, the log carries the
request id, and `/version` shows the note while still answering 200.
-rw-r--r--src/db.rs108
-rw-r--r--src/error.rs59
-rw-r--r--src/lib.rs27
3 files changed, 179 insertions, 15 deletions
diff --git a/src/db.rs b/src/db.rs
index 283f4bf..975a821 100644
--- a/src/db.rs
+++ b/src/db.rs
@@ -113,7 +113,14 @@ pub fn is_connection_loss(error: &DbError) -> bool {
     let DbError::Query(error) = error else {
         return false;
     };
-    let text = error.to_string().to_lowercase();
+    is_connection_loss_detail(&error.to_string())
+}
+
+/// The same test against a driver message already rendered, so a caller
+/// that has to borrow the text anyway (for the error it returns) can ask
+/// the question without a second copy of the error.
+pub fn is_connection_loss_detail(message: &str) -> bool {
+    let text = message.to_lowercase();
     [
         "broken pipe",
         "connection reset",
@@ -141,6 +148,69 @@ pub fn forget_pool() {
     }
 }
 
+/// The last database failure this process saw, so a caller's report of a
+/// 500 can be matched to a cause even when nobody was watching the log.
+///
+/// Recorded in [`crate::error::ApiError::from_db`], i.e. at the one place
+/// every database error passes through, and read by `/version`: "the API
+/// answers but data endpoints fail" is otherwise indistinguishable from an
+/// authentication problem from the outside, which is exactly the confusion
+/// that sent a user to the wrong conclusion on 2026-11-08.
+static LAST_FAILURE: OnceLock<Mutex<Option<Failure>>> = OnceLock::new();
+
+/// How a database failure happened: what an operator needs to tell "the
+/// server is down" from "a connection died under us" from "the query was
+/// wrong".
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FailureKind {
+    /// Could not build a connection/pool at all: the database is down or
+    /// refuses this configuration.
+    Unavailable,
+    /// The server closed a connection we were using. The pool is rebuilt
+    /// when this is seen, so it is usually recovered-from by the next
+    /// request -- and it is the failure that *looks* like a server fault
+    /// while being a socket-level event.
+    ConnectionLost,
+    /// A query the database rejected.
+    Query,
+}
+
+impl FailureKind {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            FailureKind::Unavailable => "unavailable",
+            FailureKind::ConnectionLost => "connection_lost",
+            FailureKind::Query => "query",
+        }
+    }
+}
+
+/// One recorded failure: when, what kind, and the driver's message.
+#[derive(Debug, Clone)]
+pub struct Failure {
+    pub at: std::time::Instant,
+    pub kind: FailureKind,
+    pub detail: String,
+}
+
+/// Remember `failure` as the last database failure.
+pub fn record_failure(kind: FailureKind, detail: impl Into<String>) {
+    if let Ok(mut slot) = LAST_FAILURE.get_or_init(|| Mutex::new(None)).lock() {
+        *slot = Some(Failure {
+            at: std::time::Instant::now(),
+            kind,
+            detail: detail.into(),
+        });
+    }
+}
+
+/// The last recorded failure, if it was less than `max_age` ago.
+pub fn recent_failure(max_age: std::time::Duration) -> Option<Failure> {
+    let guard = LAST_FAILURE.get_or_init(|| Mutex::new(None)).lock().ok()?;
+    let failure = guard.as_ref()?;
+    (failure.at.elapsed() < max_age).then(|| failure.clone())
+}
+
 /// Database reachability, cached for a few seconds: what `/version`
 /// reports, so "the service answers but nothing behind it works" is
 /// visible without trying a data endpoint.
@@ -1500,3 +1570,39 @@ mod tests {
         )));
     }
 }
+
+#[cfg(test)]
+mod failure_tests {
+    use super::*;
+    use std::time::Duration;
+
+    /// The memory of the last failure is what makes `/version` able to say
+    /// "the data endpoints failed a minute ago" while its own probe is
+    /// healthy -- the exact confusion behind a user reading a database
+    /// fault as an authentication problem.
+    #[test]
+    fn a_recent_failure_is_remembered_and_ages_out() {
+        record_failure(FailureKind::ConnectionLost, "CodecError { IO error: Broken pipe }");
+        let failure = recent_failure(Duration::from_secs(60)).expect("just recorded");
+        assert_eq!(failure.kind, FailureKind::ConnectionLost);
+        assert!(failure.kind.as_str() == "connection_lost");
+        assert!(failure.detail.contains("Broken pipe"));
+        // Aged out: the note must not outlive its usefulness.
+        assert!(recent_failure(Duration::ZERO).is_none());
+    }
+
+    #[test]
+    fn a_driver_message_is_recognised_as_a_connection_loss() {
+        for message in [
+            "CodecError { IO error: Broken pipe }",
+            "server has gone away",
+            "Lost connection to MySQL server during query",
+            "connection reset by peer",
+        ] {
+            assert!(is_connection_loss_detail(message), "{message}");
+        }
+        assert!(!is_connection_loss_detail(
+            "Table 'db_webqtl.Nope' doesn't exist"
+        ));
+    }
+}
diff --git a/src/error.rs b/src/error.rs
index a9b01bd..b84cf9b 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -121,27 +121,40 @@ impl ApiError {
     /// the inline `map_err` sites agree.
     pub fn from_db(error: crate::db::DbError, operation_id: &str) -> ApiError {
         match error {
-            crate::db::DbError::Unavailable(detail) => ApiError::new(
-                StatusCode::SERVICE_UNAVAILABLE,
-                "db_unavailable",
-                "Database unavailable",
-                detail,
-                operation_id,
-            ),
+            crate::db::DbError::Unavailable(detail) => {
+                crate::db::record_failure(crate::db::FailureKind::Unavailable, detail.clone());
+                ApiError::new(
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    "db_unavailable",
+                    "Database unavailable",
+                    detail,
+                    operation_id,
+                )
+            }
             crate::db::DbError::Query(error) => {
                 // A connection that died under the query is not the
                 // query's fault. Forget the pool so the next request
                 // cannot draw the same dead sockets -- this is the one
                 // place every database error passes through, so the
                 // recovery does not depend on each caller remembering it.
+                let detail = error.to_string();
+                let lost = crate::db::is_connection_loss_detail(&detail);
+                crate::db::record_failure(
+                    if lost {
+                        crate::db::FailureKind::ConnectionLost
+                    } else {
+                        crate::db::FailureKind::Query
+                    },
+                    detail.clone(),
+                );
                 let api_error = ApiError::new(
                     StatusCode::INTERNAL_SERVER_ERROR,
                     "database_error",
                     "Database error",
-                    error.to_string(),
+                    detail,
                     operation_id,
                 );
-                if crate::db::is_connection_loss(&crate::db::DbError::Query(error)) {
+                if lost {
                     crate::db::forget_pool();
                 }
                 api_error
@@ -153,10 +166,38 @@ impl ApiError {
 impl IntoResponse for ApiError {
     fn into_response(self) -> Response {
         let status = StatusCode::from_u16(self.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
+        // Server-side failures get one line on stderr each, as JSON, so a
+        // caller's report ("500 at 12:31, request id ...") can be matched to
+        // what actually happened. 4xx are not logged: they are the caller's
+        // business (and a busy 401/422 stream would drown this out), but a
+        // 5xx is ours, and until this existed the service kept no record of
+        // one at all.
+        if status.is_server_error() {
+            eprintln!(
+                "{}",
+                serde_json::json!({
+                    "event": "error",
+                    "status": status.as_u16(),
+                    "code": self.code,
+                    "operation_id": self.operation_id,
+                    "request_id": self.request_id,
+                    "took_ms": self.took_ms,
+                    "detail": truncate(&self.detail, 500),
+                })
+            );
+        }
         (status, Json(self)).into_response()
     }
 }
 
+/// `text` cut to `limit` characters (not bytes), for one log line.
+fn truncate(text: &str, limit: usize) -> String {
+    if text.chars().count() <= limit {
+        return text.to_string();
+    }
+    text.chars().take(limit).collect::<String>() + "…"
+}
+
 /// The 401 every token-gated endpoint can return, as one shared
 /// `IntoResponses` type (an `ApiError` body under the hood) instead of
 /// writing out `(status = 401, description = "...", body = ApiError)`
diff --git a/src/lib.rs b/src/lib.rs
index 6edeed6..765660c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -185,10 +185,14 @@ struct VersionInfo {
 #[gnauth(disabled)]
 async fn version(ctx: Ctx) -> Json<Envelope<VersionInfo>> {
     let git = env!("GN4_GIT_HASH");
-    // No `notes` here: the fields say what they are (see VersionInfo's
-    // doc comments), and /version is the cheapest endpoint for the
-    // envelope test's "notes is an empty array" case.
-    Json(ctx.envelope(VersionInfo {
+    // No `notes` in the ordinary case: the fields say what they are (see
+    // VersionInfo's doc comments), and /version is the cheapest endpoint
+    // for the envelope test's "notes is an empty array" case. The
+    // exception is a database failure in the last few minutes, which is
+    // the one thing a caller cannot otherwise see from here -- `database`
+    // is a *now* probe, so an intermittent fault leaves every data
+    // endpoint failing with nothing on this endpoint to say so.
+    let mut envelope = ctx.envelope(VersionInfo {
         name: "GeneNetwork4",
         short_name: "GN4AI",
         version: 4.0,
@@ -203,5 +207,18 @@ async fn version(ctx: Ctx) -> Json<Envelope<VersionInfo>> {
         } else {
             "unreachable"
         },
-    }))
+    });
+    if let Some(failure) = crate::db::recent_failure(std::time::Duration::from_secs(600)) {
+        let seconds = failure.at.elapsed().as_secs();
+        envelope = envelope.with_note(&format!(
+            "A database request failed {}s ago ({}): {}. `database` is a probe of the \
+             connection pool as well, so `reachable` here does not mean the failure did not \
+             happen -- it means one is not happening now. The pool is rebuilt when a \
+             connection dies, so a single such line is usually recovered from.",
+            seconds,
+            failure.kind.as_str(),
+            failure.detail.chars().take(200).collect::<String>()
+        ));
+    }
+    Json(envelope)
 }