diff options
| author | Pjotr <wrk@bosa.lan> | 2026-11-09 00:48:27 +0000 |
|---|---|---|
| committer | Pjotr <wrk@bosa.lan> | 2026-11-09 00:48:27 +0000 |
| commit | 731e085c42e22104f4cba9975a33f5d22b058caf (patch) | |
| tree | 34419f60994c54eebba4c246419fe7dac1b7d885 /src | |
| parent | ab5fef05237309667dd2ca7f835a5fbf9c71a12a (diff) | |
| download | genenetwork4-731e085c42e22104f4cba9975a33f5d22b058caf.tar.gz | |
errors: log every 5xx, and remember the last database failure
A user reported that the token was fine and the service was up, but "everything backed by the relational database is failing" -- while `/search` (Xapian) and `/version` answered. By the time I looked, every data endpoint answered 200 and `/version` said `database: reachable`, and there was **no record anywhere** of what they had hit: the service logged nothing at all, so a caller's report could not be matched to a cause, and `/version`'s `database` is a *now* probe that says nothing about the minute before. Two things now make that diagnosable: * every 5xx is written to stderr as one JSON line -- status, code, operation_id, **request_id**, took_ms and a truncated detail -- so "500 at 12:31, request id ..." can be looked up. 4xx stay unlogged: they are the caller's business, and a 401/422 stream would drown this; * the last database failure is remembered (`db::record_failure`, called from `ApiError::from_db`, the one place every database error passes through) and `/version` reports it in `notes` for ten minutes, with its kind (`unavailable`, `connection_lost`, `query`) and the driver's message. That is the gap that misled the user: a healthy probe beside failing endpoints looks like an auth problem from outside. `is_connection_loss` gained a `_detail` form so the caller that already has the rendered message does not render it twice. * src/error.rs: the log line, and recording failures by kind; * src/db.rs: `FailureKind`, `Failure`, `record_failure`, `recent_failure`, and two unit tests (a failure is remembered and ages out; a driver message is classified correctly); * src/lib.rs: the `/version` note, absent unless something failed recently. Checked against a deliberately broken `SQL_URI` on a scratch instance: the data endpoint answers `503 db_unavailable`, the log carries the request id, and `/version` shows the note while still answering 200.
Diffstat (limited to 'src')
| -rw-r--r-- | src/db.rs | 108 | ||||
| -rw-r--r-- | src/error.rs | 59 | ||||
| -rw-r--r-- | src/lib.rs | 27 |
3 files changed, 179 insertions, 15 deletions
diff --git a/src/db.rs b/src/db.rs index 283f4bf..975a821 100644 --- a/src/db.rs +++ b/src/db.rs @@ -113,7 +113,14 @@ pub fn is_connection_loss(error: &DbError) -> bool { let DbError::Query(error) = error else { return false; }; - let text = error.to_string().to_lowercase(); + is_connection_loss_detail(&error.to_string()) +} + +/// The same test against a driver message already rendered, so a caller +/// that has to borrow the text anyway (for the error it returns) can ask +/// the question without a second copy of the error. +pub fn is_connection_loss_detail(message: &str) -> bool { + let text = message.to_lowercase(); [ "broken pipe", "connection reset", @@ -141,6 +148,69 @@ pub fn forget_pool() { } } +/// The last database failure this process saw, so a caller's report of a +/// 500 can be matched to a cause even when nobody was watching the log. +/// +/// Recorded in [`crate::error::ApiError::from_db`], i.e. at the one place +/// every database error passes through, and read by `/version`: "the API +/// answers but data endpoints fail" is otherwise indistinguishable from an +/// authentication problem from the outside, which is exactly the confusion +/// that sent a user to the wrong conclusion on 2026-11-08. +static LAST_FAILURE: OnceLock<Mutex<Option<Failure>>> = OnceLock::new(); + +/// How a database failure happened: what an operator needs to tell "the +/// server is down" from "a connection died under us" from "the query was +/// wrong". +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FailureKind { + /// Could not build a connection/pool at all: the database is down or + /// refuses this configuration. + Unavailable, + /// The server closed a connection we were using. The pool is rebuilt + /// when this is seen, so it is usually recovered-from by the next + /// request -- and it is the failure that *looks* like a server fault + /// while being a socket-level event. + ConnectionLost, + /// A query the database rejected. + Query, +} + +impl FailureKind { + pub fn as_str(self) -> &'static str { + match self { + FailureKind::Unavailable => "unavailable", + FailureKind::ConnectionLost => "connection_lost", + FailureKind::Query => "query", + } + } +} + +/// One recorded failure: when, what kind, and the driver's message. +#[derive(Debug, Clone)] +pub struct Failure { + pub at: std::time::Instant, + pub kind: FailureKind, + pub detail: String, +} + +/// Remember `failure` as the last database failure. +pub fn record_failure(kind: FailureKind, detail: impl Into<String>) { + if let Ok(mut slot) = LAST_FAILURE.get_or_init(|| Mutex::new(None)).lock() { + *slot = Some(Failure { + at: std::time::Instant::now(), + kind, + detail: detail.into(), + }); + } +} + +/// The last recorded failure, if it was less than `max_age` ago. +pub fn recent_failure(max_age: std::time::Duration) -> Option<Failure> { + let guard = LAST_FAILURE.get_or_init(|| Mutex::new(None)).lock().ok()?; + let failure = guard.as_ref()?; + (failure.at.elapsed() < max_age).then(|| failure.clone()) +} + /// Database reachability, cached for a few seconds: what `/version` /// reports, so "the service answers but nothing behind it works" is /// visible without trying a data endpoint. @@ -1500,3 +1570,39 @@ mod tests { ))); } } + +#[cfg(test)] +mod failure_tests { + use super::*; + use std::time::Duration; + + /// The memory of the last failure is what makes `/version` able to say + /// "the data endpoints failed a minute ago" while its own probe is + /// healthy -- the exact confusion behind a user reading a database + /// fault as an authentication problem. + #[test] + fn a_recent_failure_is_remembered_and_ages_out() { + record_failure(FailureKind::ConnectionLost, "CodecError { IO error: Broken pipe }"); + let failure = recent_failure(Duration::from_secs(60)).expect("just recorded"); + assert_eq!(failure.kind, FailureKind::ConnectionLost); + assert!(failure.kind.as_str() == "connection_lost"); + assert!(failure.detail.contains("Broken pipe")); + // Aged out: the note must not outlive its usefulness. + assert!(recent_failure(Duration::ZERO).is_none()); + } + + #[test] + fn a_driver_message_is_recognised_as_a_connection_loss() { + for message in [ + "CodecError { IO error: Broken pipe }", + "server has gone away", + "Lost connection to MySQL server during query", + "connection reset by peer", + ] { + assert!(is_connection_loss_detail(message), "{message}"); + } + assert!(!is_connection_loss_detail( + "Table 'db_webqtl.Nope' doesn't exist" + )); + } +} diff --git a/src/error.rs b/src/error.rs index a9b01bd..b84cf9b 100644 --- a/src/error.rs +++ b/src/error.rs @@ -121,27 +121,40 @@ impl ApiError { /// the inline `map_err` sites agree. pub fn from_db(error: crate::db::DbError, operation_id: &str) -> ApiError { match error { - crate::db::DbError::Unavailable(detail) => ApiError::new( - StatusCode::SERVICE_UNAVAILABLE, - "db_unavailable", - "Database unavailable", - detail, - operation_id, - ), + crate::db::DbError::Unavailable(detail) => { + crate::db::record_failure(crate::db::FailureKind::Unavailable, detail.clone()); + ApiError::new( + StatusCode::SERVICE_UNAVAILABLE, + "db_unavailable", + "Database unavailable", + detail, + operation_id, + ) + } crate::db::DbError::Query(error) => { // A connection that died under the query is not the // query's fault. Forget the pool so the next request // cannot draw the same dead sockets -- this is the one // place every database error passes through, so the // recovery does not depend on each caller remembering it. + let detail = error.to_string(); + let lost = crate::db::is_connection_loss_detail(&detail); + crate::db::record_failure( + if lost { + crate::db::FailureKind::ConnectionLost + } else { + crate::db::FailureKind::Query + }, + detail.clone(), + ); let api_error = ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "database_error", "Database error", - error.to_string(), + detail, operation_id, ); - if crate::db::is_connection_loss(&crate::db::DbError::Query(error)) { + if lost { crate::db::forget_pool(); } api_error @@ -153,10 +166,38 @@ impl ApiError { impl IntoResponse for ApiError { fn into_response(self) -> Response { let status = StatusCode::from_u16(self.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + // Server-side failures get one line on stderr each, as JSON, so a + // caller's report ("500 at 12:31, request id ...") can be matched to + // what actually happened. 4xx are not logged: they are the caller's + // business (and a busy 401/422 stream would drown this out), but a + // 5xx is ours, and until this existed the service kept no record of + // one at all. + if status.is_server_error() { + eprintln!( + "{}", + serde_json::json!({ + "event": "error", + "status": status.as_u16(), + "code": self.code, + "operation_id": self.operation_id, + "request_id": self.request_id, + "took_ms": self.took_ms, + "detail": truncate(&self.detail, 500), + }) + ); + } (status, Json(self)).into_response() } } +/// `text` cut to `limit` characters (not bytes), for one log line. +fn truncate(text: &str, limit: usize) -> String { + if text.chars().count() <= limit { + return text.to_string(); + } + text.chars().take(limit).collect::<String>() + "…" +} + /// The 401 every token-gated endpoint can return, as one shared /// `IntoResponses` type (an `ApiError` body under the hood) instead of /// writing out `(status = 401, description = "...", body = ApiError)` diff --git a/src/lib.rs b/src/lib.rs index 6edeed6..765660c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -185,10 +185,14 @@ struct VersionInfo { #[gnauth(disabled)] async fn version(ctx: Ctx) -> Json<Envelope<VersionInfo>> { let git = env!("GN4_GIT_HASH"); - // No `notes` here: the fields say what they are (see VersionInfo's - // doc comments), and /version is the cheapest endpoint for the - // envelope test's "notes is an empty array" case. - Json(ctx.envelope(VersionInfo { + // No `notes` in the ordinary case: the fields say what they are (see + // VersionInfo's doc comments), and /version is the cheapest endpoint + // for the envelope test's "notes is an empty array" case. The + // exception is a database failure in the last few minutes, which is + // the one thing a caller cannot otherwise see from here -- `database` + // is a *now* probe, so an intermittent fault leaves every data + // endpoint failing with nothing on this endpoint to say so. + let mut envelope = ctx.envelope(VersionInfo { name: "GeneNetwork4", short_name: "GN4AI", version: 4.0, @@ -203,5 +207,18 @@ async fn version(ctx: Ctx) -> Json<Envelope<VersionInfo>> { } else { "unreachable" }, - })) + }); + if let Some(failure) = crate::db::recent_failure(std::time::Duration::from_secs(600)) { + let seconds = failure.at.elapsed().as_secs(); + envelope = envelope.with_note(&format!( + "A database request failed {}s ago ({}): {}. `database` is a probe of the \ + connection pool as well, so `reachable` here does not mean the failure did not \ + happen -- it means one is not happening now. The pool is rebuilt when a \ + connection dies, so a single such line is usually recovered from.", + seconds, + failure.kind.as_str(), + failure.detail.chars().take(200).collect::<String>() + )); + } + Json(envelope) } |
