about summary refs log tree commit diff
path: root/tests/db_recovery.rs
blob: 7a24a0e20431ed70380edac04bac23b975d4a027 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
//! The database pool survives a connection that goes away under it.
//!
//! Requires a live MariaDB reachable with the configured `SQL_URI` (and a
//! token for the route); skips without them. The test asks the database to
//! `KILL` every session this process holds, which is the same situation as
//! a server-side close of an idle pooled connection -- the one that made
//! every database-backed request 500 with `CodecError { IO error: Broken
//! pipe }` until the process was restarted.

use axum::http::StatusCode;
use http_body_util::BodyExt;
use mysql::prelude::Queryable;
use serde_json::Value;

mod common;

use common::{auth_token, db_available, get_with_token};

#[tokio::test]
async fn a_killed_pooled_connection_does_not_take_the_service_with_it() {
    if !db_available() {
        eprintln!("skipping a_killed_pooled_connection_does_not_take_the_service_with_it: no usable database (check SQL_URI credentials)");
        return;
    }
    let Some(token) = auth_token().await else {
        eprintln!("skipping a_killed_pooled_connection_does_not_take_the_service_with_it: set GN4_TEST_AUTH_TOKEN (see docs/gn-auth.md)");
        return;
    };

    // Warm the pool and prove the route works.
    let response = get_with_token("GET /groups/BXD", "/groups/BXD", &token).await;
    assert_eq!(response.status(), StatusCode::OK);

    // Kill this process's sessions from a separate connection, the way a
    // server-side timeout or an administrator would.
    let config = genenetwork4::config::Config::load();
    let opts = mysql::Opts::from_url(&config.sql_uri).expect("SQL_URI parses");
    let mut killer = mysql::Conn::new(opts).expect("a second connection");
    let ids: Vec<u64> = killer
        .query("SELECT Id FROM information_schema.processlist \
                WHERE User = SUBSTRING_INDEX(SUBSTRING_INDEX(USER(), '@', 1), '@', 1) \
                  AND Id <> CONNECTION_ID()")
        .expect("reading the process list");
    assert!(!ids.is_empty(), "the pool should hold connections by now");
    for id in ids {
        // Ignore failures: a session may already be gone.
        let _ = killer.query_drop(format!("KILL {id}"));
    }

    // The next requests must recover: `check_health` replaces a dead
    // pooled connection at checkout, and if one slips through, the pool is
    // forgotten rather than drawn from again.
    let mut failures = Vec::new();
    for attempt in 1..=12 {
        let response = get_with_token(
            &format!("GET /groups/BXD (after a kill, attempt {attempt})"),
            "/groups/BXD",
            &token,
        )
        .await;
        let status = response.status();
        let body = response.into_body().collect().await.unwrap().to_bytes();
        if status != StatusCode::OK {
            let error: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);
            failures.push(format!(
                "attempt {attempt}: {status} {}",
                error["detail"].as_str().unwrap_or("")
            ));
        }
    }
    assert!(
        failures.is_empty(),
        "every request after the kill must succeed, got: {failures:?}"
    );
}