about summary refs log tree commit diff
path: root/src/lib.rs
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-11-09 00:48:27 +0000
committerPjotr <wrk@bosa.lan>2026-11-09 00:48:27 +0000
commit731e085c42e22104f4cba9975a33f5d22b058caf (patch)
tree34419f60994c54eebba4c246419fe7dac1b7d885 /src/lib.rs
parentab5fef05237309667dd2ca7f835a5fbf9c71a12a (diff)
downloadgenenetwork4-731e085c42e22104f4cba9975a33f5d22b058caf.tar.gz
errors: log every 5xx, and remember the last database failure
A user reported that the token was fine and the service was up, but
"everything backed by the relational database is failing" -- while
`/search` (Xapian) and `/version` answered. By the time I looked, every
data endpoint answered 200 and `/version` said `database: reachable`, and
there was **no record anywhere** of what they had hit: the service logged
nothing at all, so a caller's report could not be matched to a cause, and
`/version`'s `database` is a *now* probe that says nothing about the
minute before.

Two things now make that diagnosable:

* every 5xx is written to stderr as one JSON line -- status, code,
  operation_id, **request_id**, took_ms and a truncated detail -- so
  "500 at 12:31, request id ..." can be looked up. 4xx stay unlogged: they
  are the caller's business, and a 401/422 stream would drown this;
* the last database failure is remembered (`db::record_failure`, called
  from `ApiError::from_db`, the one place every database error passes
  through) and `/version` reports it in `notes` for ten minutes, with its
  kind (`unavailable`, `connection_lost`, `query`) and the driver's
  message. That is the gap that misled the user: a healthy probe beside
  failing endpoints looks like an auth problem from outside.

`is_connection_loss` gained a `_detail` form so the caller that already
has the rendered message does not render it twice.

* src/error.rs: the log line, and recording failures by kind;
* src/db.rs: `FailureKind`, `Failure`, `record_failure`, `recent_failure`,
  and two unit tests (a failure is remembered and ages out; a driver
  message is classified correctly);
* src/lib.rs: the `/version` note, absent unless something failed recently.

Checked against a deliberately broken `SQL_URI` on a scratch instance: the
data endpoint answers `503 db_unavailable`, the log carries the
request id, and `/version` shows the note while still answering 200.
Diffstat (limited to 'src/lib.rs')
-rw-r--r--src/lib.rs27
1 files changed, 22 insertions, 5 deletions
diff --git a/src/lib.rs b/src/lib.rs
index 6edeed6..765660c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -185,10 +185,14 @@ struct VersionInfo {
 #[gnauth(disabled)]
 async fn version(ctx: Ctx) -> Json<Envelope<VersionInfo>> {
     let git = env!("GN4_GIT_HASH");
-    // No `notes` here: the fields say what they are (see VersionInfo's
-    // doc comments), and /version is the cheapest endpoint for the
-    // envelope test's "notes is an empty array" case.
-    Json(ctx.envelope(VersionInfo {
+    // No `notes` in the ordinary case: the fields say what they are (see
+    // VersionInfo's doc comments), and /version is the cheapest endpoint
+    // for the envelope test's "notes is an empty array" case. The
+    // exception is a database failure in the last few minutes, which is
+    // the one thing a caller cannot otherwise see from here -- `database`
+    // is a *now* probe, so an intermittent fault leaves every data
+    // endpoint failing with nothing on this endpoint to say so.
+    let mut envelope = ctx.envelope(VersionInfo {
         name: "GeneNetwork4",
         short_name: "GN4AI",
         version: 4.0,
@@ -203,5 +207,18 @@ async fn version(ctx: Ctx) -> Json<Envelope<VersionInfo>> {
         } else {
             "unreachable"
         },
-    }))
+    });
+    if let Some(failure) = crate::db::recent_failure(std::time::Duration::from_secs(600)) {
+        let seconds = failure.at.elapsed().as_secs();
+        envelope = envelope.with_note(&format!(
+            "A database request failed {}s ago ({}): {}. `database` is a probe of the \
+             connection pool as well, so `reachable` here does not mean the failure did not \
+             happen -- it means one is not happening now. The pool is rebuilt when a \
+             connection dies, so a single such line is usually recovered from.",
+            seconds,
+            failure.kind.as_str(),
+            failure.detail.chars().take(200).collect::<String>()
+        ));
+    }
+    Json(envelope)
 }