about summary refs log tree commit diff
path: root/tests
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-10-07 13:48:21 +0000
committerPjotr <wrk@bosa.lan>2026-10-07 13:48:21 +0000
commitdfd4bf042cce290a103f83ce19df5a4781e51c17 (patch)
treee0b5e3c425892dacfd6175048b8a81fd28b75ba4 /tests
parente912c87deaf220fd5fce1299d669e8fa992b5338 (diff)
downloadgenenetwork4-dfd4bf042cce290a103f83ce19df5a4781e51c17.tar.gz
* tests/benchmark.rs: add the answerable small_benchmark.csv questions.
Checks the 30 of the 39 questions in aisearch/data/small_benchmark.csv
that the API can answer today against the benchmark's own answers:
species and group metadata, group datasets, trait metadata, and
search-based trait questions (marker, additive effect, PubMed id, mean,
LOD within 0.5%). The benchmark's LOD is LRS / 4.604, ours LRS / 4.61.
The uncovered questions are listed in the file's header comment.

* test/data/benchmark_search.json: recorded production gn3 search
responses for those questions (2026-10-07, trimmed to the relevant
hits), so the test needs no network. Database-backed checks skip when
no usable database is reachable.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Diffstat (limited to 'tests')
-rw-r--r--tests/benchmark.rs460
1 files changed, 460 insertions, 0 deletions
diff --git a/tests/benchmark.rs b/tests/benchmark.rs
new file mode 100644
index 0000000..fbb61e1
--- /dev/null
+++ b/tests/benchmark.rs
@@ -0,0 +1,460 @@
+//! The questions of aisearch/data/small_benchmark.csv that this API can
+//! answer today, each checked against the benchmark's own answer. Row
+//! numbers below are 0-based data rows of that file.
+//!
+//! Not covered, on purpose (see the discussion of missing endpoints):
+//! rows 11, 25, 30 (a trait named by its phenotype abbreviation, which
+//! the API neither exposes nor looks up), 16 and 37 (a GeneNetwork trait
+//! page URL), 18 (the benchmark lists datasets our database does not
+//! link to the group), 19 (no mammal/taxonomy class), 34 (needs
+//! /search pagination) and 38 (the benchmark's answer is an arbitrary
+//! sample of many matching traits).
+//!
+//! Search-based questions run against recorded production gn3 responses
+//! (test/data/benchmark_search.json, test/data/search_serotonin_midbrain_bxd.json;
+//! recorded 2026-10-07 and trimmed to the question's relevant hits), so no
+//! network is needed. Tokens are signed locally and checked against a stub
+//! JWKS, as in tests/search.rs. The questions that read the database
+//! (species, groups, datasets, trait metadata, and the peak-marker `locus`
+//! of search hits) are skipped, not failed, when no usable database is
+//! reachable. Like tests/search.rs, everything runs inside one test
+//! because it sets process-global environment variables.
+//!
+//! LOD: the benchmark's LOD values are LRS / 4.604, ours LRS / 4.61 (the
+//! GeneNetwork2 convention), a 0.13% difference, so LOD comparisons allow
+//! 0.5%. The marker names and additive effects are compared exactly.
+
+use axum::Router;
+use axum::body::Body;
+use axum::extract::Query;
+use axum::http::{Request, StatusCode};
+use axum::routing::get;
+use http_body_util::BodyExt;
+use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
+use serde::Serialize;
+use serde_json::Value;
+use std::collections::{BTreeSet, HashMap};
+use tower::ServiceExt;
+
+#[allow(dead_code)]
+mod common;
+
+const BENCHMARK_SEARCH: &str = include_str!("../test/data/benchmark_search.json");
+const SEROTONIN_MIDBRAIN: &str = include_str!("../test/data/search_serotonin_midbrain_bxd.json");
+const TEST_JWKS: &str = include_str!("../test/data/test_jwks.json");
+const TEST_PRIVATE_KEY: &str = include_str!("../test/data/test_rsa_private.pem");
+
+/// Stands in for gn3's /api/search: serves the recorded response for the
+/// exact query string, and a 400 with gn3's error shape for any query
+/// that was not recorded (so a typo fails loudly, as a 422).
+async fn serve_gn3(Query(params): Query<HashMap<String, String>>) -> (StatusCode, String) {
+    let query = params.get("query").map(String::as_str).unwrap_or_default();
+    if query == "serotonin AND midbrain AND group:BXD" {
+        return (StatusCode::OK, SEROTONIN_MIDBRAIN.to_string());
+    }
+    let recorded: HashMap<String, Value> = serde_json::from_str(BENCHMARK_SEARCH).unwrap();
+    match recorded.get(query) {
+        Some(hits) => (StatusCode::OK, hits.to_string()),
+        None => (
+            StatusCode::BAD_REQUEST,
+            format!(r#"{{"error": "no recorded fixture for query: {query}"}}"#),
+        ),
+    }
+}
+
+async fn serve_jwks() -> (StatusCode, &'static str) {
+    (StatusCode::OK, TEST_JWKS)
+}
+
+async fn spawn(app: Router) -> String {
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let port = listener.local_addr().unwrap().port();
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+    format!("http://127.0.0.1:{port}")
+}
+
+/// Signs a token locally with the committed test private key; `issuer`
+/// must equal the configured AUTH_SERVER_URL (src/auth.rs checks `iss`).
+fn sign_test_token(issuer: &str) -> String {
+    #[derive(Serialize)]
+    struct Claims {
+        sub: String,
+        scope: String,
+        iss: String,
+        iat: u64,
+        exp: u64,
+    }
+    let now = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_secs();
+    let claims = Claims {
+        sub: "benchmark-test-user".to_string(),
+        scope: "profile".to_string(),
+        iss: issuer.to_string(),
+        iat: now,
+        exp: now + 3600,
+    };
+    let key = EncodingKey::from_rsa_pem(TEST_PRIVATE_KEY.as_bytes()).unwrap();
+    encode(&Header::new(Algorithm::RS256), &claims, &key).unwrap()
+}
+
+fn set(key: &str, value: &str) {
+    // SAFETY: this file's single test is the only thing touching these
+    // keys, in one sequence (see the module doc comment).
+    unsafe { std::env::set_var(key, value) };
+}
+
+/// GETs `uri` with the test token and returns the JSON body, which must
+/// come with a 200.
+async fn get_json(uri: &str, token: &str) -> Value {
+    let request = Request::builder()
+        .uri(uri.replace(' ', "%20"))
+        .header("Authorization", format!("Bearer {token}"))
+        .body(Body::empty())
+        .unwrap();
+    let response = genenetwork4::app().oneshot(request).await.unwrap();
+    let status = response.status();
+    let body = response.into_body().collect().await.unwrap().to_bytes();
+    let json: Value = serde_json::from_slice(&body).unwrap();
+    assert_eq!(status, StatusCode::OK, "GET {uri}: {json}");
+    json
+}
+
+/// `/search` for a recorded query, ranked by LOD, as the hit list.
+async fn search(q: &str, token: &str) -> Vec<Value> {
+    let uri = format!(
+        "/search?q={}&sort=lod&limit=100",
+        q.replace(' ', "%20")
+    );
+    get_json(&uri, token).await["data"]
+        .as_array()
+        .unwrap()
+        .clone()
+}
+
+/// The highest-LOD hit whose description contains every `needles` and
+/// none of the `excluded` strings.
+fn best<'a>(hits: &'a [Value], needles: &[&str], excluded: &[&str]) -> &'a Value {
+    hits.iter()
+        .find(|hit| {
+            let d = hit["description"].as_str().unwrap();
+            needles.iter().all(|n| d.contains(n)) && !excluded.iter().any(|x| d.contains(x))
+        })
+        .unwrap_or_else(|| panic!("no hit with {needles:?} (and none of {excluded:?})"))
+}
+
+/// The benchmark's LOD is LRS / 4.604; allow our LRS / 4.61 to be 0.5% off.
+fn assert_lod(hit: &Value, benchmark_lod: f64) {
+    let lod = hit["lod"].as_f64().unwrap();
+    assert!(
+        (lod - benchmark_lod).abs() / benchmark_lod < 0.005,
+        "{}: lod {lod} vs benchmark {benchmark_lod}",
+        hit["name"]
+    );
+}
+
+fn names(groups: &[&Value]) -> BTreeSet<String> {
+    groups
+        .iter()
+        .map(|g| g["name"].as_str().unwrap().to_string())
+        .collect()
+}
+
+fn set_of(items: &[&str]) -> BTreeSet<String> {
+    items.iter().map(|s| s.to_string()).collect()
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn benchmark_questions() {
+    let gn3 = spawn(Router::new().route("/api/search/", get(serve_gn3))).await;
+    let gn_auth = spawn(Router::new().route("/auth/public-jwks", get(serve_jwks))).await;
+    set("GN3_URL", &format!("{gn3}/api/"));
+    set("AUTH_SERVER_URL", &gn_auth);
+    let token = sign_test_token(&gn_auth);
+    let token = token.as_str();
+
+    let db = common::db_available();
+    if !db {
+        eprintln!("skipping the database-backed questions: no usable database (check SQL_URI)");
+    }
+
+    // ---- Search-based questions (recorded gn3) -------------------------
+
+    // Row 8: "What is the highest LOD score, locus marker, and additive
+    // effect for serotonin in BXD male midbrain?" Benchmark: LOD
+    // 2.6063138915106485172, rsm10000012461, -1.3041285214285716076.
+    let hits = search("serotonin AND midbrain AND group:BXD", token).await;
+    let hit = best(
+        &hits,
+        &["Midbrain 5-hydroxytryptamine (5-HT, serotonin) level in adult males ["],
+        &[],
+    );
+    assert_eq!(hit["name"], "17985");
+    assert_lod(hit, 2.6063138915106485);
+    assert!((hit["additive"].as_f64().unwrap() - -1.3041285214285716).abs() < 1e-9);
+    if db {
+        assert_eq!(hit["locus"], "rsm10000012461");
+    }
+
+    // Rows 21-24, 27: the MPTP / Parkinson's model in BXD striatum.
+    let mptp = search("MPTP AND group:BXD", token).await;
+    // Row 21: highest LOD, serotonin, male striatum, after MPTP treatment.
+    let hit = best(
+        &mptp,
+        &["serotonin (5-HT)", "males 48 h after injection (saline-MPTP group)"],
+        &["females"],
+    );
+    assert_eq!(hit["name"], "15154");
+    assert_lod(hit, 3.9673994445052552);
+    if db {
+        assert_eq!(hit["locus"], "rsm10000002155");
+    }
+    // Row 23: the dopamine metabolite 3-MT in male striatum 48 h after MPTP.
+    let hit = best(
+        &mptp,
+        &["3-methoxytyramine", "males 48 h after injection (MPTP group)"],
+        &["females"],
+    );
+    assert_eq!(hit["name"], "15144");
+    assert_lod(hit, 2.0516721070584274);
+    if db {
+        assert_eq!(hit["locus"], "rsm10000001111");
+    }
+    // Row 24: the saline-vs-MPTP difference in 3-MT, male striatum.
+    let hit = best(
+        &mptp,
+        &["3-methoxytyramine", "males 48 h after injection (saline-MPTP group)"],
+        &["females"],
+    );
+    assert_eq!(hit["name"], "15145");
+    assert_lod(hit, 2.5855803408910196);
+    if db {
+        assert_eq!(hit["locus"], "rsm10000002558");
+    }
+    // Row 27: neurotransmitters measured in the MPTP model: serotonin,
+    // its metabolite 5-HIAA and the dopamine metabolite 3-MT, in the
+    // caudate-putamen (striatum).
+    for needle in ["serotonin (5-HT)", "5-HIAA", "3-methoxytyramine"] {
+        best(&mptp, &[needle, "caudate-putamen (striatum)"], &[]);
+    }
+
+    // Row 26: a trait measured in the olfactory bulb of BXD mice one hour
+    // after saline injection: THDOC.
+    let hits = search("olfactory AND group:BXD", token).await;
+    best(&hits, &["THDOC", "olfactory bulb", "1 hour after saline"], &[]);
+
+    // Row 28: serotonin in male vs female BXD diencephalon; the benchmark
+    // gives the two traits' means.
+    let hits = search("diencephalon AND serotonin AND group:BXD", token).await;
+    let male = best(&hits, &["Diencephalon 5-hydroxytryptamine", "adult males ["], &[]);
+    let female = best(&hits, &["Diencephalon 5-hydroxytryptamine", "adult females ["], &[]);
+    assert_eq!(male["name"], "17976");
+    assert_eq!(female["name"], "17979");
+    assert!((male["mean"].as_f64().unwrap() - 13.318529409520766).abs() < 1e-9);
+    assert!((female["mean"].as_f64().unwrap() - 14.784413830987338).abs() < 1e-9);
+
+    // Row 29: highest association for acetaminophen hepatocyte viability
+    // in male BXD liver.
+    let hits = search("APAP AND hepatocyte AND group:BXD", token).await;
+    assert_lod(&hits[0], 4.7056254566002762);
+    if db {
+        assert_eq!(hits[0]["locus"], "rsm10000003730");
+    }
+
+    // Row 31: the PubMed study on acoustic startle response in BXD mice.
+    // Hits carry gn3's pubmed_id through untouched.
+    let hits = search("startle AND group:BXD", token).await;
+    assert!(
+        hits.iter()
+            .any(|h| h["pubmed_id"] == 10371755
+                && h["description"].as_str().unwrap().contains("acoustic startle")),
+        "a startle trait from PubMed 10371755"
+    );
+
+    // Row 32: highest LOD for raw striatum volume in BXD mice.
+    let hits = search("striatum AND volume AND group:BXD", token).await;
+    let hit = best(&hits, &["dorsal striatum of young adults", "shrinkage corrected"], &[]);
+    assert_lod(hit, 2.8594045664833709);
+    if db {
+        assert_eq!(hit["locus"], "rsm10000003425");
+    }
+
+    // Row 33: the marker with the highest LOD for sleep need (slow wave
+    // sleep delta EEG power after 6 hours of deprivation).
+    let hits = search("sleep AND deprivation AND group:BXD", token).await;
+    assert_lod(&hits[0], 3.6721742203463301);
+    if db {
+        assert_eq!(hits[0]["locus"], "rsm10000010045");
+    }
+
+    // Row 35: the microbiome genera reported in PubMed 31634382:
+    // Streptococcus, Staphylococcus and Stenotrophomonas.
+    for genus in ["Streptococcus", "Staphylococcus", "Stenotrophomonas"] {
+        let hits = search(&format!("{genus} AND group:BXD"), token).await;
+        assert!(
+            hits.iter().any(|h| h["pubmed_id"] == 31634382
+                && h["description"].as_str().unwrap().contains(&format!("{genus} (genus)"))),
+            "{genus} from PubMed 31634382"
+        );
+    }
+
+    // Row 36: the autoimmune thyroid disease model behind the TBI and T4
+    // traits: Graves' disease, induced by TSHR A-subunit adenovirus.
+    let hits = search("Graves AND group:BXD", token).await;
+    best(&hits, &["TSHR A-subunit adenovirus"], &[]);
+
+    if !db {
+        return;
+    }
+
+    // ---- Database-backed questions ------------------------------------
+
+    // Rows 9, 10, 15: trait BXDPublish_17985 (serotonin in BXD male
+    // midbrain): its PubMed id, its mean, and what it is.
+    let info = get_json("/groups/BXD/traits/17985", token).await;
+    assert_eq!(info["pubmed_id"], 24102824);
+    assert!((info["mean"].as_f64().unwrap() - 12.037411717807545).abs() < 1e-12);
+    let description = info["post_publication_description"].as_str().unwrap();
+    assert!(description.contains("Midbrain") && description.contains("serotonin"));
+    assert!(description.contains("adult males"));
+
+    // Row 22: the PubMed study on MPTP-induced serotonin changes in BXD
+    // male striatum.
+    let info = get_json("/groups/BXD/traits/15154", token).await;
+    assert_eq!(info["pubmed_id"], 23558233);
+
+    // Row 0: metazoa other than vertebrates: medaka, fly, nematode.
+    let species = get_json("/species", token).await;
+    let species = species.as_array().unwrap();
+    let of_family = |family: &str| -> Vec<Value> {
+        species.iter().filter(|s| s["family"] == family).cloned().collect()
+    };
+    let mut full_names = Vec::new();
+    for s in of_family("Other Metazoa") {
+        let detail = get_json(&format!("/species/{}", s["name"].as_str().unwrap()), token).await;
+        full_names.push(detail["full_name"].as_str().unwrap().to_string());
+    }
+    for latin in ["Oryzias latipes", "Drosophila melanogaster", "Caenorhabditis elegans"] {
+        assert!(full_names.iter().any(|n| n.contains(latin)), "{latin} in {full_names:?}");
+    }
+    // Row 20: the plants.
+    let mut plants = Vec::new();
+    for s in of_family("Plants") {
+        let detail = get_json(&format!("/species/{}", s["name"].as_str().unwrap()), token).await;
+        plants.push(detail["full_name"].as_str().unwrap().to_string());
+    }
+    assert_eq!(plants.len(), 5);
+    for latin in [
+        "Arabidopsis thaliana",
+        "Glycine max",
+        "Solanum lycopersicum",
+        "Hordeum vulgare",
+        "Populus trichocarpa",
+    ] {
+        assert!(plants.iter().any(|n| n.contains(latin)), "{latin} in {plants:?}");
+    }
+
+    // Group-level questions. /groups has no `family`, so read each group.
+    let list = get_json("/groups", token).await;
+    let mut groups = Vec::new();
+    for g in list.as_array().unwrap() {
+        groups.push(get_json(&format!("/groups/{}", g["name"].as_str().unwrap()), token).await);
+    }
+    let family = |prefix: &'static str| -> Vec<&Value> {
+        groups
+            .iter()
+            .filter(|g| g["family"].as_str().is_some_and(|f| f.starts_with(prefix)))
+            .collect()
+    };
+    let group = |name: &str| -> &Value {
+        groups.iter().find(|g| g["name"] == name).unwrap_or_else(|| panic!("group {name}"))
+    };
+
+    // Row 1: the reference populations.
+    assert_eq!(
+        names(&family("Reference Populations")),
+        set_of(&["AKXD", "AXBXA", "BXD", "BXH", "CC", "CXB", "HXBBXH", "LXS", "MDP"])
+    );
+    // Row 2: the sets of the Heterogeneous Stock population (the
+    // benchmark spells the names with underscores).
+    assert_eq!(
+        names(&family("Crosses and Heterogeneous Stock")),
+        set_of(&[
+            "HSNIH-Palmer",
+            "HSNIH-Palmer_r4",
+            "HSNIH-RGSMC",
+            "NWU_WKYxF344_F2",
+            "SRxSHRSPF2"
+        ])
+    );
+    // Row 4: the BXD sets used to study Alzheimer's disease.
+    let ad: Vec<&Value> = groups
+        .iter()
+        .filter(|g| g["name"].as_str().unwrap().starts_with("BXD"))
+        .filter(|g| {
+            let full = g["full_name"].as_str().unwrap_or_default();
+            full.split_whitespace().any(|w| w == "AD") || full.contains("Alzheimer")
+        })
+        .collect();
+    assert_eq!(names(&ad), set_of(&["BXD-JAX-AD", "BXD-NIA-AD"]));
+    // Row 6: retinal transcriptomics sets (the search also finds a
+    // glaucoma and aged retina set).
+    let retina = names(
+        &groups
+            .iter()
+            .filter(|g| g["full_name"].as_str().unwrap_or_default().to_lowercase().contains("retina"))
+            .collect::<Vec<_>>(),
+    );
+    assert!(retina.contains("TIGEM-Retina-RNA-Seq") && retina.contains("Retina-RGC-Rheaume"));
+
+    // Rows 5, 7, 12: which species a set belongs to, via the species id
+    // (a group only has the numeric id; /species/{name} has id and
+    // full_name).
+    let mut species_full_name = HashMap::new();
+    for s in species {
+        let detail = get_json(&format!("/species/{}", s["name"].as_str().unwrap()), token).await;
+        species_full_name.insert(
+            detail["id"].as_u64().unwrap(),
+            detail["full_name"].as_str().unwrap().to_string(),
+        );
+    }
+    let species_of = |name: &str| species_full_name[&group(name)["species_id"].as_u64().unwrap()].clone();
+    assert!(species_of("HRDP_HXB-BXH-BP").contains("Rattus norvegicus"));
+    assert!(group("MAGIC_Lines")["full_name"].as_str().unwrap().contains("MAGIC"));
+    assert!(species_of("MAGIC_Lines").contains("Caenorhabditis elegans"));
+    assert!(species_of("Oregon-R_x_2b3").contains("Drosophila melanogaster"));
+
+    // Rows 3, 13: parental strains, only in the group description text.
+    let bxh = group("BXH")["description"].as_str().unwrap();
+    assert!(bxh.contains("C57BL/6J") && bxh.contains("C3H/HeJ"));
+    let lxs = group("LXS")["description"].as_str().unwrap();
+    assert!(lxs.contains("Inbred Long-Sleep (ILS) and Inbred Short-Sleep (ISS)"));
+
+    // Row 14: brain regions with transcriptomic datasets for HIV-1Tg.
+    let datasets = get_json("/groups/HIV-1Tg/datasets", token).await;
+    let full: Vec<&str> = datasets
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|d| d["full_name"].as_str().unwrap())
+        .collect();
+    for region in ["Hippocampus", "PFC", "Striatum"] {
+        assert!(full.iter().any(|n| n.contains(region)), "{region} in {full:?}");
+    }
+    // Row 17: the BXD colon transcriptomics datasets under different diets.
+    let datasets = get_json("/groups/BXD/datasets", token).await;
+    let colon: BTreeSet<String> = datasets
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|d| d["name"].as_str().unwrap().to_string())
+        .filter(|n| n.contains("_Col-"))
+        .collect();
+    assert_eq!(
+        colon,
+        set_of(&["EPFL-LISP_BXD_Col-CD+HFD0917", "EPFL-LISP_BXD_Col-CD0917", "EPFL-LISP_BXD_Col-HFD0917"])
+    );
+}