diff options
| author | Pjotr <wrk@bosa.lan> | 2026-10-07 13:48:21 +0000 |
|---|---|---|
| committer | Pjotr <wrk@bosa.lan> | 2026-10-07 13:48:21 +0000 |
| commit | dfd4bf042cce290a103f83ce19df5a4781e51c17 (patch) | |
| tree | e0b5e3c425892dacfd6175048b8a81fd28b75ba4 /tests | |
| parent | e912c87deaf220fd5fce1299d669e8fa992b5338 (diff) | |
| download | genenetwork4-dfd4bf042cce290a103f83ce19df5a4781e51c17.tar.gz | |
* tests/benchmark.rs: add the answerable small_benchmark.csv questions.
Checks the 30 of the 39 questions in aisearch/data/small_benchmark.csv that the API can answer today against the benchmark's own answers: species and group metadata, group datasets, trait metadata, and search-based trait questions (marker, additive effect, PubMed id, mean, LOD within 0.5%). The benchmark's LOD is LRS / 4.604, ours LRS / 4.61. The uncovered questions are listed in the file's header comment. * test/data/benchmark_search.json: recorded production gn3 search responses for those questions (2026-10-07, trimmed to the relevant hits), so the test needs no network. Database-backed checks skip when no usable database is reachable. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/benchmark.rs | 460 |
1 files changed, 460 insertions, 0 deletions
diff --git a/tests/benchmark.rs b/tests/benchmark.rs new file mode 100644 index 0000000..fbb61e1 --- /dev/null +++ b/tests/benchmark.rs @@ -0,0 +1,460 @@ +//! The questions of aisearch/data/small_benchmark.csv that this API can +//! answer today, each checked against the benchmark's own answer. Row +//! numbers below are 0-based data rows of that file. +//! +//! Not covered, on purpose (see the discussion of missing endpoints): +//! rows 11, 25, 30 (a trait named by its phenotype abbreviation, which +//! the API neither exposes nor looks up), 16 and 37 (a GeneNetwork trait +//! page URL), 18 (the benchmark lists datasets our database does not +//! link to the group), 19 (no mammal/taxonomy class), 34 (needs +//! /search pagination) and 38 (the benchmark's answer is an arbitrary +//! sample of many matching traits). +//! +//! Search-based questions run against recorded production gn3 responses +//! (test/data/benchmark_search.json, test/data/search_serotonin_midbrain_bxd.json; +//! recorded 2026-10-07 and trimmed to the question's relevant hits), so no +//! network is needed. Tokens are signed locally and checked against a stub +//! JWKS, as in tests/search.rs. The questions that read the database +//! (species, groups, datasets, trait metadata, and the peak-marker `locus` +//! of search hits) are skipped, not failed, when no usable database is +//! reachable. Like tests/search.rs, everything runs inside one test +//! because it sets process-global environment variables. +//! +//! LOD: the benchmark's LOD values are LRS / 4.604, ours LRS / 4.61 (the +//! GeneNetwork2 convention), a 0.13% difference, so LOD comparisons allow +//! 0.5%. The marker names and additive effects are compared exactly. + +use axum::Router; +use axum::body::Body; +use axum::extract::Query; +use axum::http::{Request, StatusCode}; +use axum::routing::get; +use http_body_util::BodyExt; +use jsonwebtoken::{Algorithm, EncodingKey, Header, encode}; +use serde::Serialize; +use serde_json::Value; +use std::collections::{BTreeSet, HashMap}; +use tower::ServiceExt; + +#[allow(dead_code)] +mod common; + +const BENCHMARK_SEARCH: &str = include_str!("../test/data/benchmark_search.json"); +const SEROTONIN_MIDBRAIN: &str = include_str!("../test/data/search_serotonin_midbrain_bxd.json"); +const TEST_JWKS: &str = include_str!("../test/data/test_jwks.json"); +const TEST_PRIVATE_KEY: &str = include_str!("../test/data/test_rsa_private.pem"); + +/// Stands in for gn3's /api/search: serves the recorded response for the +/// exact query string, and a 400 with gn3's error shape for any query +/// that was not recorded (so a typo fails loudly, as a 422). +async fn serve_gn3(Query(params): Query<HashMap<String, String>>) -> (StatusCode, String) { + let query = params.get("query").map(String::as_str).unwrap_or_default(); + if query == "serotonin AND midbrain AND group:BXD" { + return (StatusCode::OK, SEROTONIN_MIDBRAIN.to_string()); + } + let recorded: HashMap<String, Value> = serde_json::from_str(BENCHMARK_SEARCH).unwrap(); + match recorded.get(query) { + Some(hits) => (StatusCode::OK, hits.to_string()), + None => ( + StatusCode::BAD_REQUEST, + format!(r#"{{"error": "no recorded fixture for query: {query}"}}"#), + ), + } +} + +async fn serve_jwks() -> (StatusCode, &'static str) { + (StatusCode::OK, TEST_JWKS) +} + +async fn spawn(app: Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://127.0.0.1:{port}") +} + +/// Signs a token locally with the committed test private key; `issuer` +/// must equal the configured AUTH_SERVER_URL (src/auth.rs checks `iss`). +fn sign_test_token(issuer: &str) -> String { + #[derive(Serialize)] + struct Claims { + sub: String, + scope: String, + iss: String, + iat: u64, + exp: u64, + } + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let claims = Claims { + sub: "benchmark-test-user".to_string(), + scope: "profile".to_string(), + iss: issuer.to_string(), + iat: now, + exp: now + 3600, + }; + let key = EncodingKey::from_rsa_pem(TEST_PRIVATE_KEY.as_bytes()).unwrap(); + encode(&Header::new(Algorithm::RS256), &claims, &key).unwrap() +} + +fn set(key: &str, value: &str) { + // SAFETY: this file's single test is the only thing touching these + // keys, in one sequence (see the module doc comment). + unsafe { std::env::set_var(key, value) }; +} + +/// GETs `uri` with the test token and returns the JSON body, which must +/// come with a 200. +async fn get_json(uri: &str, token: &str) -> Value { + let request = Request::builder() + .uri(uri.replace(' ', "%20")) + .header("Authorization", format!("Bearer {token}")) + .body(Body::empty()) + .unwrap(); + let response = genenetwork4::app().oneshot(request).await.unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let json: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(status, StatusCode::OK, "GET {uri}: {json}"); + json +} + +/// `/search` for a recorded query, ranked by LOD, as the hit list. +async fn search(q: &str, token: &str) -> Vec<Value> { + let uri = format!( + "/search?q={}&sort=lod&limit=100", + q.replace(' ', "%20") + ); + get_json(&uri, token).await["data"] + .as_array() + .unwrap() + .clone() +} + +/// The highest-LOD hit whose description contains every `needles` and +/// none of the `excluded` strings. +fn best<'a>(hits: &'a [Value], needles: &[&str], excluded: &[&str]) -> &'a Value { + hits.iter() + .find(|hit| { + let d = hit["description"].as_str().unwrap(); + needles.iter().all(|n| d.contains(n)) && !excluded.iter().any(|x| d.contains(x)) + }) + .unwrap_or_else(|| panic!("no hit with {needles:?} (and none of {excluded:?})")) +} + +/// The benchmark's LOD is LRS / 4.604; allow our LRS / 4.61 to be 0.5% off. +fn assert_lod(hit: &Value, benchmark_lod: f64) { + let lod = hit["lod"].as_f64().unwrap(); + assert!( + (lod - benchmark_lod).abs() / benchmark_lod < 0.005, + "{}: lod {lod} vs benchmark {benchmark_lod}", + hit["name"] + ); +} + +fn names(groups: &[&Value]) -> BTreeSet<String> { + groups + .iter() + .map(|g| g["name"].as_str().unwrap().to_string()) + .collect() +} + +fn set_of(items: &[&str]) -> BTreeSet<String> { + items.iter().map(|s| s.to_string()).collect() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn benchmark_questions() { + let gn3 = spawn(Router::new().route("/api/search/", get(serve_gn3))).await; + let gn_auth = spawn(Router::new().route("/auth/public-jwks", get(serve_jwks))).await; + set("GN3_URL", &format!("{gn3}/api/")); + set("AUTH_SERVER_URL", &gn_auth); + let token = sign_test_token(&gn_auth); + let token = token.as_str(); + + let db = common::db_available(); + if !db { + eprintln!("skipping the database-backed questions: no usable database (check SQL_URI)"); + } + + // ---- Search-based questions (recorded gn3) ------------------------- + + // Row 8: "What is the highest LOD score, locus marker, and additive + // effect for serotonin in BXD male midbrain?" Benchmark: LOD + // 2.6063138915106485172, rsm10000012461, -1.3041285214285716076. + let hits = search("serotonin AND midbrain AND group:BXD", token).await; + let hit = best( + &hits, + &["Midbrain 5-hydroxytryptamine (5-HT, serotonin) level in adult males ["], + &[], + ); + assert_eq!(hit["name"], "17985"); + assert_lod(hit, 2.6063138915106485); + assert!((hit["additive"].as_f64().unwrap() - -1.3041285214285716).abs() < 1e-9); + if db { + assert_eq!(hit["locus"], "rsm10000012461"); + } + + // Rows 21-24, 27: the MPTP / Parkinson's model in BXD striatum. + let mptp = search("MPTP AND group:BXD", token).await; + // Row 21: highest LOD, serotonin, male striatum, after MPTP treatment. + let hit = best( + &mptp, + &["serotonin (5-HT)", "males 48 h after injection (saline-MPTP group)"], + &["females"], + ); + assert_eq!(hit["name"], "15154"); + assert_lod(hit, 3.9673994445052552); + if db { + assert_eq!(hit["locus"], "rsm10000002155"); + } + // Row 23: the dopamine metabolite 3-MT in male striatum 48 h after MPTP. + let hit = best( + &mptp, + &["3-methoxytyramine", "males 48 h after injection (MPTP group)"], + &["females"], + ); + assert_eq!(hit["name"], "15144"); + assert_lod(hit, 2.0516721070584274); + if db { + assert_eq!(hit["locus"], "rsm10000001111"); + } + // Row 24: the saline-vs-MPTP difference in 3-MT, male striatum. + let hit = best( + &mptp, + &["3-methoxytyramine", "males 48 h after injection (saline-MPTP group)"], + &["females"], + ); + assert_eq!(hit["name"], "15145"); + assert_lod(hit, 2.5855803408910196); + if db { + assert_eq!(hit["locus"], "rsm10000002558"); + } + // Row 27: neurotransmitters measured in the MPTP model: serotonin, + // its metabolite 5-HIAA and the dopamine metabolite 3-MT, in the + // caudate-putamen (striatum). + for needle in ["serotonin (5-HT)", "5-HIAA", "3-methoxytyramine"] { + best(&mptp, &[needle, "caudate-putamen (striatum)"], &[]); + } + + // Row 26: a trait measured in the olfactory bulb of BXD mice one hour + // after saline injection: THDOC. + let hits = search("olfactory AND group:BXD", token).await; + best(&hits, &["THDOC", "olfactory bulb", "1 hour after saline"], &[]); + + // Row 28: serotonin in male vs female BXD diencephalon; the benchmark + // gives the two traits' means. + let hits = search("diencephalon AND serotonin AND group:BXD", token).await; + let male = best(&hits, &["Diencephalon 5-hydroxytryptamine", "adult males ["], &[]); + let female = best(&hits, &["Diencephalon 5-hydroxytryptamine", "adult females ["], &[]); + assert_eq!(male["name"], "17976"); + assert_eq!(female["name"], "17979"); + assert!((male["mean"].as_f64().unwrap() - 13.318529409520766).abs() < 1e-9); + assert!((female["mean"].as_f64().unwrap() - 14.784413830987338).abs() < 1e-9); + + // Row 29: highest association for acetaminophen hepatocyte viability + // in male BXD liver. + let hits = search("APAP AND hepatocyte AND group:BXD", token).await; + assert_lod(&hits[0], 4.7056254566002762); + if db { + assert_eq!(hits[0]["locus"], "rsm10000003730"); + } + + // Row 31: the PubMed study on acoustic startle response in BXD mice. + // Hits carry gn3's pubmed_id through untouched. + let hits = search("startle AND group:BXD", token).await; + assert!( + hits.iter() + .any(|h| h["pubmed_id"] == 10371755 + && h["description"].as_str().unwrap().contains("acoustic startle")), + "a startle trait from PubMed 10371755" + ); + + // Row 32: highest LOD for raw striatum volume in BXD mice. + let hits = search("striatum AND volume AND group:BXD", token).await; + let hit = best(&hits, &["dorsal striatum of young adults", "shrinkage corrected"], &[]); + assert_lod(hit, 2.8594045664833709); + if db { + assert_eq!(hit["locus"], "rsm10000003425"); + } + + // Row 33: the marker with the highest LOD for sleep need (slow wave + // sleep delta EEG power after 6 hours of deprivation). + let hits = search("sleep AND deprivation AND group:BXD", token).await; + assert_lod(&hits[0], 3.6721742203463301); + if db { + assert_eq!(hits[0]["locus"], "rsm10000010045"); + } + + // Row 35: the microbiome genera reported in PubMed 31634382: + // Streptococcus, Staphylococcus and Stenotrophomonas. + for genus in ["Streptococcus", "Staphylococcus", "Stenotrophomonas"] { + let hits = search(&format!("{genus} AND group:BXD"), token).await; + assert!( + hits.iter().any(|h| h["pubmed_id"] == 31634382 + && h["description"].as_str().unwrap().contains(&format!("{genus} (genus)"))), + "{genus} from PubMed 31634382" + ); + } + + // Row 36: the autoimmune thyroid disease model behind the TBI and T4 + // traits: Graves' disease, induced by TSHR A-subunit adenovirus. + let hits = search("Graves AND group:BXD", token).await; + best(&hits, &["TSHR A-subunit adenovirus"], &[]); + + if !db { + return; + } + + // ---- Database-backed questions ------------------------------------ + + // Rows 9, 10, 15: trait BXDPublish_17985 (serotonin in BXD male + // midbrain): its PubMed id, its mean, and what it is. + let info = get_json("/groups/BXD/traits/17985", token).await; + assert_eq!(info["pubmed_id"], 24102824); + assert!((info["mean"].as_f64().unwrap() - 12.037411717807545).abs() < 1e-12); + let description = info["post_publication_description"].as_str().unwrap(); + assert!(description.contains("Midbrain") && description.contains("serotonin")); + assert!(description.contains("adult males")); + + // Row 22: the PubMed study on MPTP-induced serotonin changes in BXD + // male striatum. + let info = get_json("/groups/BXD/traits/15154", token).await; + assert_eq!(info["pubmed_id"], 23558233); + + // Row 0: metazoa other than vertebrates: medaka, fly, nematode. + let species = get_json("/species", token).await; + let species = species.as_array().unwrap(); + let of_family = |family: &str| -> Vec<Value> { + species.iter().filter(|s| s["family"] == family).cloned().collect() + }; + let mut full_names = Vec::new(); + for s in of_family("Other Metazoa") { + let detail = get_json(&format!("/species/{}", s["name"].as_str().unwrap()), token).await; + full_names.push(detail["full_name"].as_str().unwrap().to_string()); + } + for latin in ["Oryzias latipes", "Drosophila melanogaster", "Caenorhabditis elegans"] { + assert!(full_names.iter().any(|n| n.contains(latin)), "{latin} in {full_names:?}"); + } + // Row 20: the plants. + let mut plants = Vec::new(); + for s in of_family("Plants") { + let detail = get_json(&format!("/species/{}", s["name"].as_str().unwrap()), token).await; + plants.push(detail["full_name"].as_str().unwrap().to_string()); + } + assert_eq!(plants.len(), 5); + for latin in [ + "Arabidopsis thaliana", + "Glycine max", + "Solanum lycopersicum", + "Hordeum vulgare", + "Populus trichocarpa", + ] { + assert!(plants.iter().any(|n| n.contains(latin)), "{latin} in {plants:?}"); + } + + // Group-level questions. /groups has no `family`, so read each group. + let list = get_json("/groups", token).await; + let mut groups = Vec::new(); + for g in list.as_array().unwrap() { + groups.push(get_json(&format!("/groups/{}", g["name"].as_str().unwrap()), token).await); + } + let family = |prefix: &'static str| -> Vec<&Value> { + groups + .iter() + .filter(|g| g["family"].as_str().is_some_and(|f| f.starts_with(prefix))) + .collect() + }; + let group = |name: &str| -> &Value { + groups.iter().find(|g| g["name"] == name).unwrap_or_else(|| panic!("group {name}")) + }; + + // Row 1: the reference populations. + assert_eq!( + names(&family("Reference Populations")), + set_of(&["AKXD", "AXBXA", "BXD", "BXH", "CC", "CXB", "HXBBXH", "LXS", "MDP"]) + ); + // Row 2: the sets of the Heterogeneous Stock population (the + // benchmark spells the names with underscores). + assert_eq!( + names(&family("Crosses and Heterogeneous Stock")), + set_of(&[ + "HSNIH-Palmer", + "HSNIH-Palmer_r4", + "HSNIH-RGSMC", + "NWU_WKYxF344_F2", + "SRxSHRSPF2" + ]) + ); + // Row 4: the BXD sets used to study Alzheimer's disease. + let ad: Vec<&Value> = groups + .iter() + .filter(|g| g["name"].as_str().unwrap().starts_with("BXD")) + .filter(|g| { + let full = g["full_name"].as_str().unwrap_or_default(); + full.split_whitespace().any(|w| w == "AD") || full.contains("Alzheimer") + }) + .collect(); + assert_eq!(names(&ad), set_of(&["BXD-JAX-AD", "BXD-NIA-AD"])); + // Row 6: retinal transcriptomics sets (the search also finds a + // glaucoma and aged retina set). + let retina = names( + &groups + .iter() + .filter(|g| g["full_name"].as_str().unwrap_or_default().to_lowercase().contains("retina")) + .collect::<Vec<_>>(), + ); + assert!(retina.contains("TIGEM-Retina-RNA-Seq") && retina.contains("Retina-RGC-Rheaume")); + + // Rows 5, 7, 12: which species a set belongs to, via the species id + // (a group only has the numeric id; /species/{name} has id and + // full_name). + let mut species_full_name = HashMap::new(); + for s in species { + let detail = get_json(&format!("/species/{}", s["name"].as_str().unwrap()), token).await; + species_full_name.insert( + detail["id"].as_u64().unwrap(), + detail["full_name"].as_str().unwrap().to_string(), + ); + } + let species_of = |name: &str| species_full_name[&group(name)["species_id"].as_u64().unwrap()].clone(); + assert!(species_of("HRDP_HXB-BXH-BP").contains("Rattus norvegicus")); + assert!(group("MAGIC_Lines")["full_name"].as_str().unwrap().contains("MAGIC")); + assert!(species_of("MAGIC_Lines").contains("Caenorhabditis elegans")); + assert!(species_of("Oregon-R_x_2b3").contains("Drosophila melanogaster")); + + // Rows 3, 13: parental strains, only in the group description text. + let bxh = group("BXH")["description"].as_str().unwrap(); + assert!(bxh.contains("C57BL/6J") && bxh.contains("C3H/HeJ")); + let lxs = group("LXS")["description"].as_str().unwrap(); + assert!(lxs.contains("Inbred Long-Sleep (ILS) and Inbred Short-Sleep (ISS)")); + + // Row 14: brain regions with transcriptomic datasets for HIV-1Tg. + let datasets = get_json("/groups/HIV-1Tg/datasets", token).await; + let full: Vec<&str> = datasets + .as_array() + .unwrap() + .iter() + .map(|d| d["full_name"].as_str().unwrap()) + .collect(); + for region in ["Hippocampus", "PFC", "Striatum"] { + assert!(full.iter().any(|n| n.contains(region)), "{region} in {full:?}"); + } + // Row 17: the BXD colon transcriptomics datasets under different diets. + let datasets = get_json("/groups/BXD/datasets", token).await; + let colon: BTreeSet<String> = datasets + .as_array() + .unwrap() + .iter() + .map(|d| d["name"].as_str().unwrap().to_string()) + .filter(|n| n.contains("_Col-")) + .collect(); + assert_eq!( + colon, + set_of(&["EPFL-LISP_BXD_Col-CD+HFD0917", "EPFL-LISP_BXD_Col-CD0917", "EPFL-LISP_BXD_Col-HFD0917"]) + ); +} |
