//! The questions of aisearch/data/small_benchmark.csv and //! full_benchmark.csv that this API can answer today, each checked //! against the benchmark's own answer. "Row N" is a 0-based data row of //! the small file, "Full row N" of the full file (whose 115 rows include //! 37 of the small file's, which are tested once, as small rows). //! //! Not covered, small file: rows 18 (the benchmark lists datasets our //! database does not link to the group), 19 (no mammal/taxonomy class) //! and 38 (an arbitrary sample of many matching traits). //! Not covered, full file (the 14 new rows the API cannot answer, or //! answers only in part): 1, 4, 5, 8, 32, 42, 43, 52, 55 (the tissue //! reaches every benchmark dataset, but with 36 more), 61, 62, 63, 87 //! (the benchmark's LOD belongs to the males' trait, though the //! question says female) and 112 (several traits match). //! Full rows 71, 78, 79, 80, 83, 84, 88, 101, 103 and 104 name traits by the //! benchmark's sanitised short names (every non-alphanumeric character //! turned into `_`), which the database stores unsanitised //! (`AAP/Lys_CD`, `TAG(56:7)_2_CD`), so they are matched against the //! group's full trait listing here. //! //! Search-based questions run against recorded production gn3 responses //! (test/data/benchmark_search.json, test/data/search_serotonin_midbrain_bxd.json; //! recorded 2026-10-07 and trimmed to the question's relevant hits), so no //! network is needed. Tokens are signed locally and checked against a stub //! JWKS, as in tests/search.rs. The questions that read the database //! (species, groups, datasets, trait metadata, and the peak-marker `locus` //! of search hits) are skipped, not failed, when no usable database is //! reachable. Like tests/search.rs, everything runs inside one test //! because it sets process-global environment variables. //! //! LOD: the benchmark's LOD values are LRS / 4.604, as ours are, so they //! are compared to rounding error. Marker names and additive effects are //! compared exactly. use axum::Router; use axum::body::Body; use axum::extract::Query; use axum::http::{Request, StatusCode}; use axum::routing::get; use http_body_util::BodyExt; use jsonwebtoken::{Algorithm, EncodingKey, Header, encode}; use serde::Serialize; use serde_json::Value; use std::collections::{BTreeSet, HashMap}; use tower::ServiceExt; #[allow(dead_code)] mod common; const BENCHMARK_SEARCH: &str = include_str!("../test/data/benchmark_search.json"); const SEROTONIN_MIDBRAIN: &str = include_str!("../test/data/search_serotonin_midbrain_bxd.json"); const TEST_JWKS: &str = include_str!("../test/data/test_jwks.json"); const TEST_PRIVATE_KEY: &str = include_str!("../test/data/test_rsa_private.pem"); /// Stands in for gn3's /api/search: serves the recorded response for the /// exact query string, and a 400 with gn3's error shape for any query /// that was not recorded (so a typo fails loudly, as a 422). async fn serve_gn3(Query(params): Query>) -> (StatusCode, String) { let query = params.get("query").map(String::as_str).unwrap_or_default(); if query == "serotonin AND midbrain AND group:BXD" { return (StatusCode::OK, SEROTONIN_MIDBRAIN.to_string()); } let recorded: HashMap = serde_json::from_str(BENCHMARK_SEARCH).unwrap(); match recorded.get(query) { Some(hits) => (StatusCode::OK, hits.to_string()), None => ( StatusCode::BAD_REQUEST, format!(r#"{{"error": "no recorded fixture for query: {query}"}}"#), ), } } async fn serve_jwks() -> (StatusCode, &'static str) { (StatusCode::OK, TEST_JWKS) } async fn spawn(app: Router) -> String { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); format!("http://127.0.0.1:{port}") } /// Signs a token locally with the committed test private key; `issuer` /// must equal the configured AUTH_SERVER_URL (src/auth.rs checks `iss`). fn sign_test_token(issuer: &str) -> String { #[derive(Serialize)] struct Claims { sub: String, scope: String, iss: String, iat: u64, exp: u64, } let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_secs(); let claims = Claims { sub: "benchmark-test-user".to_string(), scope: "profile".to_string(), iss: issuer.to_string(), iat: now, exp: now + 3600, }; let key = EncodingKey::from_rsa_pem(TEST_PRIVATE_KEY.as_bytes()).unwrap(); encode(&Header::new(Algorithm::RS256), &claims, &key).unwrap() } fn set(key: &str, value: &str) { // SAFETY: this file's single test is the only thing touching these // keys, in one sequence (see the module doc comment). unsafe { std::env::set_var(key, value) }; } /// GETs `uri` with the test token and returns the `data` of the response /// envelope, which must come with a 200. async fn get_json(uri: &str, token: &str) -> Value { let request = Request::builder() .uri(uri.replace(' ', "%20")) .header("Authorization", format!("Bearer {token}")) .body(Body::empty()) .unwrap(); let response = genenetwork4::app().oneshot(request).await.unwrap(); let status = response.status(); let body = response.into_body().collect().await.unwrap().to_bytes(); let json: Value = serde_json::from_slice(&body).unwrap(); assert_eq!(status, StatusCode::OK, "GET {uri}: {json}"); assert!( json["meta"]["state"].is_string(), "GET {uri}: no envelope in {json}" ); json["data"].clone() } /// `/search` for a recorded query, ranked by LOD, as the hit list. async fn search(q: &str, token: &str) -> Vec { // Three pages of a hundred: a benchmark row can name a trait that ranks // outside the first page of its own query (the ASR row ranks ~150th of // `startle AND group:BXD`), and pinning page 1 made that look like the // trait had vanished from the index. let mut hits = Vec::new(); for page in 1..=3 { let page_hits = search_page(q, page, token).await; let short = page_hits.len() < 100; hits.extend(page_hits); if short { break; } } hits } /// Like `search`, for page `page` of 100 ranked hits. async fn search_page(q: &str, page: u32, token: &str) -> Vec { let uri = format!( "/search?q={}&sort=lod&limit=100&page={page}", q.replace(' ', "%20") ); get_json(&uri, token).await.as_array().unwrap().clone() } /// The highest-LOD hit whose description contains every `needles` and /// none of the `excluded` strings. fn best<'a>(hits: &'a [Value], needles: &[&str], excluded: &[&str]) -> &'a Value { hits.iter() .find(|hit| { let d = hit["description"].as_str().unwrap(); needles.iter().all(|n| d.contains(n)) && !excluded.iter().any(|x| d.contains(x)) }) .unwrap_or_else(|| panic!("no hit with {needles:?} (and none of {excluded:?})")) } /// The benchmark's LOD is LRS / 4.604, the same as ours. fn assert_lod(hit: &Value, benchmark_lod: f64) { let lod = hit["lod"].as_f64().unwrap(); assert!( (lod - benchmark_lod).abs() / benchmark_lod < 1e-9, "{}: lod {lod} vs benchmark {benchmark_lod}", hit["name"] ); } /// The names of the groups `/groups?{query}` returns. async fn group_names(query: &str, token: &str) -> BTreeSet { get_json(&format!("/groups?{query}"), token) .await .as_array() .unwrap() .iter() .map(|g| g["name"].as_str().unwrap().to_string()) .collect() } fn names(groups: &[&Value]) -> BTreeSet { groups .iter() .map(|g| g["name"].as_str().unwrap().to_string()) .collect() } fn set_of(items: &[&str]) -> BTreeSet { items.iter().map(|s| s.to_string()).collect() } /// `text` lowercased with everything but letters and digits removed, to /// compare descriptions and names across the benchmark's punctuation. fn norm(text: &str) -> String { text.chars() .filter(|c| c.is_ascii_alphanumeric()) .collect::() .to_lowercase() } /// A set of names compared by `norm`. fn nset<'a>(items: impl IntoIterator) -> BTreeSet { items.into_iter().map(norm).collect() } fn assert_close(actual: f64, expected: f64, what: &str) { assert!( (actual - expected).abs() <= 1e-9 * expected.abs().max(1.0), "{what}: {actual} vs benchmark {expected}" ); } /// The trait of `listing` (`/groups/{g}/traits`) whose short name is the /// benchmark's `sanitised` name, i.e. equal once every non-alphanumeric /// character is `_`. fn find_sanitised<'a>(listing: &'a [Value], sanitised: &str) -> &'a Value { let sanitise = |s: &str| { s.chars() .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) .collect::() .trim_matches('_') .to_string() }; let wanted = sanitise(sanitised); let found: Vec<&Value> = listing .iter() .filter(|t| { t["abbreviation"] .as_str() .is_some_and(|a| sanitise(a) == wanted) }) .collect(); assert_eq!( found.len(), 1, "short name {sanitised}: {} matches", found.len() ); found[0] } fn description(trait_info: &Value) -> &str { trait_info["post_publication_description"] .as_str() .or_else(|| trait_info["description"].as_str()) .unwrap() } /// Full-benchmark rows answered from `/search` (recorded gn3 responses). async fn full_benchmark_search_rows(token: &str, db: bool) { // Full rows 66-68: the trait for age at onset of working memory deficit // in Alzheimer's BXD mice, its description and its PubMed id. let hits = search("working AND memory AND onset AND group:BXD", token).await; let aao = best( &hits, &[ "Age at onset of working memory deficit", "male and females average of AD-BXD cases positive for 5XFAD", ], &[], ); assert_lod(aao, 2.4026782929885950324); assert!(norm(aao["description"].as_str().unwrap()).ends_with(&norm( "Central nervous system, behavior, learning and memory: Age at onset of working memory deficit defined as first age at which performance dropped to chance 50%, male and females average of AD-BXD cases positive for 5XFAD transgene [months]" ))); assert_eq!(aao["pubmed_id"], 30595332); if db { assert_eq!(aao["locus"], "rsm10000005323"); } // Full row 74: highest mapping score for 3NP neurodegeneration. let nitro = search("nitropropionic", token).await; let hit = best(&nitro, &["3-nitropropionic acid neurodegeneration"], &[]); assert_lod(hit, 6.1340526876326206818); if db { assert_eq!(hit["locus"], "rsm10000005366"); } // Full row 77: the PubMed study on THDOC in the olfactory bulb. let olfactory = search("olfactory AND group:BXD", token).await; let thdoc = best( &olfactory, &["THDOC", "olfactory bulb", "1 hour after saline"], &[], ); assert_eq!(thdoc["pubmed_id"], 21494628); // Full rows 85-86: the liver enzyme measured in 14-week old females. let alt = search("alanine AND transaminase AND group:BXD", token).await; let hit = best(&alt, &["Alanine transaminase", "14-week old females"], &[]); assert_lod(hit, 2.1671810310648718101); assert_eq!(hit["pubmed_id"], 22939713); if db { assert_eq!(hit["locus"], "rsm10000012410"); } // Full row 91: additive effect of the acoustic startle response to // 110 dB at 10 kHz (the trait without prepulse, not a PPI trait). let startle = search("startle AND group:BXD", token).await; let asr = best( &startle, &["Acoustic startle response to a 110 dB SPL, 10 kHz tone without prepul"], &[], ); assert_close( asr["additive"].as_f64().unwrap(), -5.7583334166666668708, "ASR additive", ); // Full row 94: the PubMed article on striatum volume. let volume = search("striatum AND volume AND group:BXD", token).await; let hit = best( &volume, &["dorsal striatum of young adults", "shrinkage corrected"], &[], ); assert_eq!(hit["pubmed_id"], 19191878); // Full rows 97-100: T cell populations estimated in the breast cancer // tumors of BXD x C3-1TAg F1 females, found past the first 100 hits. // Three pages: the gamma-delta rows rank past 200 of this query. let timer = search("TIMER AND group:BXD", token).await; let ending = |tail: &str| -> &Value { timer .iter() .find(|h| h["description"].as_str().unwrap().ends_with(tail)) .unwrap_or_else(|| panic!("no TIMER trait ending {tail}")) }; ending("T.cell.gamma.delta_CIBERSORT"); // row 97 ending("T.cell.CD4..naive_XCELL"); // row 98 ... let cd8_naive = ending("T.cell.CD8..naive_XCELL"); // ... and 98-99 assert_lod(cd8_naive, 2.6767753665261841967); if db { assert_eq!(cd8_naive["locus"], "rsm10000001634"); } let cd8 = ending("T.cell.CD8._TIMER"); // row 100: the model named in the description let model = cd8["description"].as_str().unwrap(); assert!(model.contains("C3 (1)-TAg") && model.contains("JAX Stock 013591")); // Full rows 106-107: serum TSH binding inhibition after 2 and 3 // immunizations with TSHR A-subunit adenovirus, and its PubMed id. let tsh = search("TSH AND binding AND group:BXD", token).await; let two = best(&tsh, &["1 week after 2 immunizations"], &[]); let three = best(&tsh, &["4 weeks after 3 immunizations"], &[]); assert_lod(two, 7.8087209638496242192); assert_lod(three, 11.292110851469207233); assert_eq!(three["pubmed_id"], 25051451); if db { assert_eq!(two["locus"], "rs3672987"); assert_eq!(three["locus"], "rs3672987"); } // Full row 114: mean of 3-MT in male BXD striatum after saline. let mptp = search("MPTP AND group:BXD", token).await; let saline = best( &mptp, &[ "3-methoxytyramine", "2-4 month-old males 48 h after saline injection", ], &[], ); assert_close( saline["mean"].as_f64().unwrap(), 0.39399999976158139869, "3-MT mean", ); } /// Full-benchmark rows answered from the database-backed endpoints. async fn full_benchmark_database_rows(token: &str) { // Full row 0: three taxonomic families. let species = get_json("/species", token).await; let families: BTreeSet<&str> = species .as_array() .unwrap() .iter() .map(|s| s["family"].as_str().unwrap()) .collect(); assert_eq!( families, ["Other Metazoa", "Plants", "Vertebrates"] .into_iter() .collect() ); // Full row 3: Drosophila's reference assembly. let fly = get_json("/species/drosophila", token).await; assert!(fly["menu_name"].as_str().unwrap().contains("dm6")); // Group questions: the list (family, species, genetic type) plus every // group's detail (description, set code). let list = get_json("/groups", token).await; let list = list.as_array().unwrap(); let bxd = list.iter().find(|g| g["name"] == "BXD").unwrap(); assert_eq!(bxd["genetic_type"], "riset"); // full row 12 let mut details = Vec::new(); for g in list { details.push(get_json(&format!("/groups/{}", g["name"].as_str().unwrap()), token).await); } let detail = |name: &str| details.iter().find(|d| d["name"] == name).unwrap(); // Full rows 10, 13, 15: genotypes, the full name of DOL, a category. assert_eq!(detail("HIV-1Tg")["family"], "Groups Without Genotypes"); // no genotypes assert!( detail("DOL")["full_name"] .as_str() .unwrap() .contains("Boon Lung Infection") ); assert_eq!(detail("B6D2F2")["family"], "Crosses, AIL, HS"); // Full row 11: eight population categories. let categories: BTreeSet<&str> = details .iter() .filter_map(|d| d["family"].as_str()) .collect(); assert_eq!(categories.len(), 8, "{categories:?}"); // Full row 20: the reference-population RI panel from BALB/cBy and C57BL/6By. let panel: Vec<&str> = details .iter() .filter(|d| { d["family"] .as_str() .is_some_and(|f| f.starts_with("Reference")) }) .filter(|d| { d["description"] .as_str() .unwrap_or_default() .contains("BALB/cBy") }) .map(|d| d["name"].as_str().unwrap()) .collect(); assert_eq!(panel, ["CXB"]); // Full row 21: the HS rat is outbred. assert!( detail("HSNIH-Palmer")["description"] .as_str() .unwrap() .to_lowercase() .contains("outbred") ); // Full row 23: the set coded RHV. let coded: Vec<&str> = details .iter() .filter(|d| d["group_code"] == "RHV") .map(|d| d["name"].as_str().unwrap()) .collect(); assert_eq!(coded, ["HIV-1Tg"]); // Full row 24: the human set on brain connectivity. let human = group_names("species=human", token).await; let connectivity: Vec<&str> = details .iter() .filter(|d| human.contains(d["name"].as_str().unwrap())) .filter(|d| { let text = format!("{} {}", d["full_name"], d["description"]).to_lowercase(); text.contains("connectom") || text.contains("connectivity") }) .map(|d| d["name"].as_str().unwrap()) .collect(); assert_eq!(connectivity, ["HCP"]); // Group lookups by family, species or text. let hrdp = group_names("family=HRDP", token).await; assert_eq!( nset(hrdp.iter().map(String::as_str)), nset(["HRDP_HXB_BXH_BP", "HRDP_HXB_BXH_Metb"]) ); // 7 assert_eq!(group_names("q=gulf", token).await, set_of(&["DOD-BXD-GWI"])); // 17 assert_eq!( group_names("q=B6MRLF2", token).await, set_of(&["B6MRLF2-D2MRLF2"]) ); // 18 assert_eq!(group_names("q=lens", token).await, set_of(&["B6-Lens"])); // 19 assert!( group_names("q=UCI", token) .await .contains("Aging-Brain-UCI") ); // 25 assert_eq!( group_names("q=islet", token).await, set_of(&["Islets-Gerling"]) ); // 26 assert_eq!( group_names("species=human&q=child", token).await, set_of(&["CANDLE"]) ); // 28 assert_eq!( group_names("q=CEPH", token).await, set_of(&["CEPH-2004", "CEPH-2009"]) ); // 29 assert_eq!( group_names("species=medaka", token).await, set_of(&["MIKK"]) ); // 31 assert_eq!( group_names("q=EBV", token).await, set_of(&["EBV_T_Cells_PERKINS"]) ); // 49 // Full row 41: the human sets (no genotype field: all of them). assert_eq!( nset(human.iter().map(String::as_str)), nset([ "GTEx", "GTEx_v5", "GTEx_v8", "HCP", "EBV_T_Cells_PERKINS", "HB", "HLC", "HLT", "HSB", "TIGEM_Retina_RNA_Seq", "Islets_Gerling", "Brain_Normal_NIH_Gibbs", "Aging_Brain_UCI", "AD_cases_controls", "AD_cases_controls_Myers", "CANDLE", "CEPH_2004", "CEPH_2009", ]) ); // Full row 48: CANDLE's species. let candle = list.iter().find(|g| g["name"] == "CANDLE").unwrap(); let candle_species = get_json( &format!("/species/{}", candle["species"].as_str().unwrap()), token, ) .await; assert!( candle_species["full_name"] .as_str() .unwrap() .contains("Homo sapiens") ); // Full row 50: the human transcriptome resource with the most datasets. let mut counts = Vec::new(); for name in &human { let datasets = get_json(&format!("/groups/{name}/datasets"), token).await; counts.push((datasets.as_array().unwrap().len(), name.clone())); } counts.sort(); assert!(counts.last().unwrap().1.starts_with("GTEx"), "{counts:?}"); // Full row 44: the human dataset with both log2-transformed and // TPM-normalised expression across tissues: only GTEx v8 has TPM // datasets, and they are all on the log2 scale. let mut tpm_groups = BTreeSet::new(); for name in &human { let datasets = get_json(&format!("/groups/{name}/datasets"), token).await; for dataset in datasets.as_array().unwrap() { if dataset["full_name"].as_str().unwrap().contains("TPM") { tpm_groups.insert(name.clone()); assert_eq!(dataset["data_scale"], "log2", "{}", dataset["name"]); } } } assert_eq!(tpm_groups, set_of(&["GTEx_v8"])); // Full row 33: the mapping tools in use, including those no group uses. let methods = get_json("/mapping-methods", token).await; let tools: BTreeSet<&str> = methods .as_array() .unwrap() .iter() .map(|m| m["name"].as_str().unwrap()) .collect(); assert_eq!( tools, ["FastMap", "HappyR", "PLINK", "Rqtl", "qtlreaper"] .into_iter() .collect() ); // Full row 34: the preferred tool for heterogeneous stock and outbred // populations is Rqtl (HSNIH-RGSMC maps with qtlreaper, so it is a // majority, not a rule). for name in ["HSNIH-Palmer", "DO", "CC"] { let group = list.iter().find(|g| g["name"] == name).unwrap(); assert_eq!(group["mapping_method"], "Rqtl", "{name}"); } // Full row 35: GN602 is the BXD phenotype dataset. let gn602 = get_json("/accessions/GN602", token).await; assert_eq!(gn602["category"], "publish"); assert_eq!(gn602["group"], "BXD"); assert_eq!(gn602["title"], "BXD Published Phenotypes"); // Full rows 56-58: BXD datasets by topic. let datasets = get_json("/groups/BXD/datasets", token).await; let named = |word: &str| -> BTreeSet { datasets .as_array() .unwrap() .iter() .map(|d| d["name"].as_str().unwrap()) .filter(|n| n.to_lowercase().contains(word)) .map(norm) .collect() }; assert_eq!( named("gut"), nset(["UTHSC_GutExL_0414", "UTHSC_GutGL_0414"]) ); assert_eq!( named("neut"), nset(["UTHSC_Neut_EL_1014", "UTHSC_Neut_1014"]) ); assert_eq!( named("germ"), nset([ "JAX_BXD_Germ_Cells_edgeR_0820", "JAX_BXD_Germ_Cells_edgeR_log2_0820", "JAX_BXD_Germ_Cells_edgeR_log2z_0820", ]) ); // Trait questions. Short names the database stores verbatim: let by_abbreviation = |abbreviation: &'static str| { let token = token.to_string(); async move { let traits = get_json( &format!("/groups/BXD/traits?abbreviation={abbreviation}"), &token, ) .await; let traits = traits.as_array().unwrap(); assert_eq!(traits.len(), 1, "{abbreviation}"); traits[0].clone() } }; let otu = by_abbreviation("OTU_310_CD_norm_log10").await; assert_eq!(otu["dataset"], "BXDPublish"); // full row 40: set BXD assert!(description(&otu).contains("EPFL LISP3 Cohort")); // 105 assert_close( otu["mean"].as_f64().unwrap(), 7.8246294298479632801, "OTU mean", ); // 109 let serotonin = by_abbreviation("5HT_MB_M").await; assert_eq!( serotonin["url"], "https://genenetwork.org/show_trait?trait_id=17985&dataset=BXDPublish" ); // 53 assert!(description(&serotonin).to_lowercase().contains("serotonin")); // 54 // BXD's full trait listing, for short names the database does not store // verbatim (see the module comment). let listing = get_json("/groups/BXD/traits", token).await; let listing = listing.as_array().unwrap(); let mptp = find_sanitised(listing, "5HT_CPU_Sal_MPTP_M"); assert!(norm(description(mptp)).contains(&norm( "MPTP 1-methyl-4-phenyl-1,2,3,6-tetrahydropyridine response 12.5 mg/kg ip, serotonin 5-HT concentration in caudate-putamen striatum in males 48 h after injection saline-MPTP group [ug/mg total protein]" ))); // 71 let amph = find_sanitised(listing, "AMPHLOC5mg"); assert_close( amph["mean"].as_f64().unwrap(), 7220.8214285714284415, "AMPHLOC5mg mean", ); assert_lod(amph, 3.4041048067742516459); assert_eq!(amph["locus"], "D4Mit178"); assert!(norm(description(amph)).contains(&norm( "Amphetamine response 5 mg/kg ip, locomotor activity from 0-60 min after injection [cm]" ))); // 83-84 let trait_23391 = get_json("/groups/BXD/traits/23391", token).await; assert_eq!(trait_23391["abbreviation"], "OTU_310_CD_norm_log10"); // 110 let trait_15959 = get_json("/groups/BXD/traits/15959", token).await; assert_eq!( trait_15959["url"], "https://genenetwork.org/show_trait?trait_id=15959&dataset=BXDPublish" ); // 111 // Short names the benchmark sanitises (see the module comment). let ratio = find_sanitised(listing, "AAP_Lys_CD"); assert_close( ratio["mean"].as_f64().unwrap(), 0.0045021891883398230949, "AAP/Lys mean", ); // 78 assert_eq!(ratio["pubmed_id"], 25215496); // 79 let alcohol = find_sanitised(listing, "20__Alcohol_intake_W4D4_all_Females"); // 80 assert!(norm(description(alcohol)).contains(&norm( "Ethanol response, consumption of 20% v/v EtOH using the drinking in the dark DID method, 4 h access on day 4 week 4 normally housed 4-6 month old females 5 week baseline [g/kg]" ))); let alt = find_sanitised(listing, "ALT_d0__mean_F"); // 88 assert!( description(alt) .contains("Liver injury day 0 in adult females before carbon tetrachloride challenge") ); let strep = find_sanitised(listing, "Streptococcus_genus_CD_norm_log10"); // 101 assert_lod(strep, 2.6777190182706016586); assert_eq!(strep["locus"], "rsm10000014670"); let chow = find_sanitised(listing, "TAG_56_7__2_CD"); // 103-104 let high = find_sanitised(listing, "TAG_56_7__2_HFD"); assert_lod(chow, 1.9755325276024489067); assert_eq!(chow["locus"], "rs27728896"); assert_lod(high, 3.5849473664517539362); assert_eq!(high["locus"], "rsm10000014270"); assert!(description(chow).contains("TG(16:0_18:1_22:6)")); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn benchmark_questions() { let gn3 = spawn(Router::new().route("/api/search/", get(serve_gn3))).await; let gn_auth = spawn(Router::new().route("/auth/public-jwks", get(serve_jwks))).await; set("GN3_URL", &format!("{gn3}/api/")); set("AUTH_SERVER_URL", &gn_auth); let token = sign_test_token(&gn_auth); let token = token.as_str(); let db = common::db_available(); if !db { eprintln!("skipping the database-backed questions: no usable database (check SQL_URI)"); } // ---- Search-based questions (recorded gn3) ------------------------- // Row 8: "What is the highest LOD score, locus marker, and additive // effect for serotonin in BXD male midbrain?" Benchmark: LOD // 2.6063138915106485172, rsm10000012461, -1.3041285214285716076. let hits = search("serotonin AND midbrain AND group:BXD", token).await; let hit = best( &hits, &["Midbrain 5-hydroxytryptamine (5-HT, serotonin) level in adult males ["], &[], ); assert_eq!(hit["name"], "17985"); assert_lod(hit, 2.6063138915106485); assert!((hit["additive"].as_f64().unwrap() - -1.3041285214285716).abs() < 1e-9); if db { assert_eq!(hit["locus"], "rsm10000012461"); } // Rows 21-24, 27: the MPTP / Parkinson's model in BXD striatum. let mptp = search("MPTP AND group:BXD", token).await; // Row 21: highest LOD, serotonin, male striatum, after MPTP treatment. let hit = best( &mptp, &[ "serotonin (5-HT)", "males 48 h after injection (saline-MPTP group)", ], &["females"], ); assert_eq!(hit["name"], "15154"); assert_lod(hit, 3.9673994445052552); if db { assert_eq!(hit["locus"], "rsm10000002155"); } // Row 23: the dopamine metabolite 3-MT in male striatum 48 h after MPTP. let hit = best( &mptp, &[ "3-methoxytyramine", "males 48 h after injection (MPTP group)", ], &["females"], ); assert_eq!(hit["name"], "15144"); assert_lod(hit, 2.0516721070584274); if db { assert_eq!(hit["locus"], "rsm10000001111"); } // Row 24: the saline-vs-MPTP difference in 3-MT, male striatum. let hit = best( &mptp, &[ "3-methoxytyramine", "males 48 h after injection (saline-MPTP group)", ], &["females"], ); assert_eq!(hit["name"], "15145"); assert_lod(hit, 2.5855803408910196); if db { assert_eq!(hit["locus"], "rsm10000002558"); } // Row 27: neurotransmitters measured in the MPTP model: serotonin, // its metabolite 5-HIAA and the dopamine metabolite 3-MT, in the // caudate-putamen (striatum). for needle in ["serotonin (5-HT)", "5-HIAA", "3-methoxytyramine"] { best(&mptp, &[needle, "caudate-putamen (striatum)"], &[]); } // Row 26: a trait measured in the olfactory bulb of BXD mice one hour // after saline injection: THDOC. let hits = search("olfactory AND group:BXD", token).await; best( &hits, &["THDOC", "olfactory bulb", "1 hour after saline"], &[], ); // Row 28: serotonin in male vs female BXD diencephalon; the benchmark // gives the two traits' means. let hits = search("diencephalon AND serotonin AND group:BXD", token).await; let male = best( &hits, &["Diencephalon 5-hydroxytryptamine", "adult males ["], &[], ); let female = best( &hits, &["Diencephalon 5-hydroxytryptamine", "adult females ["], &[], ); assert_eq!(male["name"], "17976"); assert_eq!(female["name"], "17979"); assert!((male["mean"].as_f64().unwrap() - 13.318529409520766).abs() < 1e-9); assert!((female["mean"].as_f64().unwrap() - 14.784413830987338).abs() < 1e-9); // Row 29: highest association for acetaminophen hepatocyte viability // in male BXD liver. let hits = search("APAP AND hepatocyte AND group:BXD", token).await; assert_lod(&hits[0], 4.7056254566002762); if db { assert_eq!(hits[0]["locus"], "rsm10000003730"); } // Row 31: the PubMed study on acoustic startle response in BXD mice. // Hits carry gn3's pubmed_id through untouched. let hits = search("startle AND group:BXD", token).await; assert!( hits.iter().any(|h| h["pubmed_id"] == 10371755 && h["description"] .as_str() .unwrap() .contains("acoustic startle")), "a startle trait from PubMed 10371755" ); // Row 32: highest LOD for raw striatum volume in BXD mice. let hits = search("striatum AND volume AND group:BXD", token).await; let hit = best( &hits, &["dorsal striatum of young adults", "shrinkage corrected"], &[], ); assert_lod(hit, 2.8594045664833709); if db { assert_eq!(hit["locus"], "rsm10000003425"); } // Row 33: the marker with the highest LOD for sleep need (slow wave // sleep delta EEG power after 6 hours of deprivation). let hits = search("sleep AND deprivation AND group:BXD", token).await; assert_lod(&hits[0], 3.6721742203463301); if db { assert_eq!(hits[0]["locus"], "rsm10000010045"); } // Row 35: the microbiome genera reported in PubMed 31634382: // Streptococcus, Staphylococcus and Stenotrophomonas. for genus in ["Streptococcus", "Staphylococcus", "Stenotrophomonas"] { let hits = search(&format!("{genus} AND group:BXD"), token).await; assert!( hits.iter().any(|h| h["pubmed_id"] == 31634382 && h["description"] .as_str() .unwrap() .contains(&format!("{genus} (genus)"))), "{genus} from PubMed 31634382" ); } // Row 36: the autoimmune thyroid disease model behind the TBI and T4 // traits: Graves' disease, induced by TSHR A-subunit adenovirus. let hits = search("Graves AND group:BXD", token).await; best(&hits, &["TSHR A-subunit adenovirus"], &[]); // Row 34: the T cell populations estimated by TIMER 2.0 in breast // cancer tumors from BXD x C3-1TAg F1 females, and their loci. The // two traits have low LOD (they rank 189th and 220th of 247), so they // are on page 2 of the LOD-ranked hits, not page 1. let page1 = search_page("TIMER AND group:BXD", 1, token).await; // The two traits rank near the *end* of the second page (the comment // above puts them 189th and 220th of ~247), so "on page 2" is a // boundary that one re-index can push over: they were found on page 3 // after the index was rebuilt. What the test is really about is that // neither is on the first page and both are reachable with their LODs, // so search forward instead of pinning a page. let mut later = Vec::new(); for page in 2..=3 { later.extend(search_page("TIMER AND group:BXD", page, token).await); } let ends_with = |hit: &Value, tail: &str| hit["description"].as_str().unwrap().ends_with(tail); assert!( !page1.iter().any(|h| ends_with(h, "T.cell.CD8._TIMER")), "the CD8 TIMER trait is not a page-1 hit" ); let cd8 = later .iter() .find(|h| ends_with(h, "T.cell.CD8._TIMER")) .expect("CD8 TIMER beyond page 1"); let cd4 = later .iter() .find(|h| ends_with(h, "T.cell.CD4..effector.memory_XCELL")) .expect("CD4 effector memory XCELL beyond page 1"); assert_lod(cd4, 1.4524872961762274); assert_lod(cd8, 1.8031565920123300); if db { assert_eq!(cd4["locus"], "rsm10000014666"); assert_eq!(cd8["locus"], "rs29338660"); } full_benchmark_search_rows(token, db).await; if !db { return; } // ---- Database-backed questions ------------------------------------ // Rows 9, 10, 15: trait BXDPublish_17985 (serotonin in BXD male // midbrain): its PubMed id, its mean, and what it is. let info = get_json("/groups/BXD/traits/17985", token).await; assert_eq!(info["pubmed_id"], 24102824); assert!((info["mean"].as_f64().unwrap() - 12.037411717807545).abs() < 1e-12); let description = info["post_publication_description"].as_str().unwrap(); assert!(description.contains("Midbrain") && description.contains("serotonin")); assert!(description.contains("adult males")); // Row 22: the PubMed study on MPTP-induced serotonin changes in BXD // male striatum. let info = get_json("/groups/BXD/traits/15154", token).await; assert_eq!(info["pubmed_id"], 23558233); // Row 0: metazoa other than vertebrates: medaka, fly, nematode. let species = get_json("/species", token).await; let species = species.as_array().unwrap(); let of_family = |family: &str| -> Vec { species .iter() .filter(|s| s["family"] == family) .cloned() .collect() }; let mut full_names = Vec::new(); for s in of_family("Other Metazoa") { let detail = get_json(&format!("/species/{}", s["name"].as_str().unwrap()), token).await; full_names.push(detail["full_name"].as_str().unwrap().to_string()); } for latin in [ "Oryzias latipes", "Drosophila melanogaster", "Caenorhabditis elegans", ] { assert!( full_names.iter().any(|n| n.contains(latin)), "{latin} in {full_names:?}" ); } // Row 20: the plants. let mut plants = Vec::new(); for s in of_family("Plants") { let detail = get_json(&format!("/species/{}", s["name"].as_str().unwrap()), token).await; plants.push(detail["full_name"].as_str().unwrap().to_string()); } assert_eq!(plants.len(), 5); for latin in [ "Arabidopsis thaliana", "Glycine max", "Solanum lycopersicum", "Hordeum vulgare", "Populus trichocarpa", ] { assert!( plants.iter().any(|n| n.contains(latin)), "{latin} in {plants:?}" ); } // Group-level questions: /groups carries each group's family and // species, and filters on them. // Row 1: the reference populations. assert_eq!( group_names("family=Reference%20Populations", token).await, set_of(&[ "AKXD", "AXBXA", "BXD", "BXH", "CC", "CXB", "HXBBXH", "LXS", "MDP" ]) ); // Row 2: the sets of the Heterogeneous Stock population (the // benchmark spells the names with underscores). assert_eq!( group_names("family=Heterogeneous%20Stock", token).await, set_of(&[ "HSNIH-Palmer", "HSNIH-Palmer_r4", "HSNIH-RGSMC", "NWU_WKYxF344_F2", "SRxSHRSPF2" ]) ); // Row 4: the BXD sets used to study Alzheimer's disease ("AD" is a // word of the full name in one, "Alzheimer" in the other). let bxd_sets = get_json("/groups?species=mouse&q=BXD", token).await; let ad: Vec<&Value> = bxd_sets .as_array() .unwrap() .iter() .filter(|g| { let full = g["full_name"].as_str().unwrap_or_default(); full.split_whitespace().any(|w| w == "AD") || full.contains("Alzheimer") }) .collect(); assert_eq!(names(&ad), set_of(&["BXD-JAX-AD", "BXD-NIA-AD"])); // Row 6: retinal transcriptomics sets (the search also finds a // glaucoma and aged retina set). let retina = group_names("q=retina", token).await; assert!(retina.contains("TIGEM-Retina-RNA-Seq") && retina.contains("Retina-RGC-Rheaume")); // Rows 5, 7, 12: which species a set belongs to: the group names its // species, /species/{name} gives the Latin name. let all_groups = get_json("/groups", token).await; let species_of = |name: &str| -> String { let group = all_groups .as_array() .unwrap() .iter() .find(|g| g["name"] == name) .unwrap(); group["species"].as_str().unwrap().to_string() }; let latin = |short: String| { let token = token.to_string(); async move { get_json(&format!("/species/{short}"), &token).await["full_name"] .as_str() .unwrap() .to_string() } }; assert!( latin(species_of("HRDP_HXB-BXH-BP")) .await .contains("Rattus norvegicus") ); assert!( latin(species_of("MAGIC_Lines")) .await .contains("Caenorhabditis elegans") ); assert!( latin(species_of("Oregon-R_x_2b3")) .await .contains("Drosophila melanogaster") ); let magic = all_groups .as_array() .unwrap() .iter() .find(|g| g["name"] == "MAGIC_Lines") .unwrap(); assert!(magic["full_name"].as_str().unwrap().contains("MAGIC")); // Rows 3, 13: parental strains, only in the group description text. let bxh = get_json("/groups/BXH", token).await; let bxh = bxh["description"].as_str().unwrap(); assert!(bxh.contains("C57BL/6J") && bxh.contains("C3H/HeJ")); let lxs = get_json("/groups/LXS", token).await; let lxs = lxs["description"].as_str().unwrap(); assert!(lxs.contains("Inbred Long-Sleep (ILS) and Inbred Short-Sleep (ISS)")); // Rows 11, 25, 30: a trait named by its short name (abbreviation). // The benchmark's descriptions leave out the parentheses and, for the // first and third, the leading subject area ("Liver, metabolism, ...:"). let without_parens = |text: &str| text.replace(['(', ')'], ""); let by_abbreviation = |abbreviation: &'static str| { let token = token.to_string(); async move { let traits = get_json( &format!("/groups/BXD/traits?abbreviation={abbreviation}"), &token, ) .await; let traits = traits.as_array().unwrap(); assert_eq!(traits.len(), 1, "{abbreviation}"); traits[0].clone() } }; let otu = by_abbreviation("OTU_310_CD_norm_log10").await; assert_eq!( without_parens(otu["post_publication_description"].as_str().unwrap()), "Metagenome, microbiota, gastrointestinal system, metabolism: OTU310 Firmicutes;Clostridiales;Lachnospiraceae proportion assessed by 16S rRNA sequencing of cecum content from young adult males 3-6 months on a standard chow diet CD, EPFL LISP3 Cohort, 2014-2018 [normalised log fraction]" ); let nitropropionic = by_abbreviation("3NP_Tox").await; assert!( without_parens(nitropropionic["post_publication_description"].as_str().unwrap()).ends_with( "3-nitropropionic acid neurodegeneration in striatum caudate putamen 50 mg/kg ip in young adults, presence or absence of lesions [-1 = lesions, +1 = no lesions]" ) ); let apap = by_abbreviation("APAP_log2").await; assert!( without_parens(apap["post_publication_description"].as_str().unwrap()).ends_with( "Acetaminophen APAP hepatocyte viability response 9 hr after treatment 0.5 mM in vehicle, lactate dehydrogenase LDH release assay adjusted for DNA content, young adult male liver [log2 LDH/DNA ratio treated/control]" ) ); // Rows 16, 37: the GeneNetwork trait page of a trait named by its // short name. let serotonin = by_abbreviation("5HT_MB_M").await; assert_eq!( serotonin["url"], "https://genenetwork.org/show_trait?trait_id=17985&dataset=BXDPublish" ); assert_eq!( nitropropionic["url"], "https://genenetwork.org/show_trait?trait_id=15959&dataset=BXDPublish" ); // Rows 31, 35 again, by publication rather than by search: the // traits PubMed 10371755 (acoustic startle) and 31634382 (BXD // microbiome) report. let startle = get_json("/publications/10371755/traits", token).await; assert!(startle.as_array().unwrap().iter().any(|t| { t["post_publication_description"] .as_str() .unwrap() .contains("acoustic startle") })); let microbiome = get_json("/publications/31634382/traits", token).await; for genus in ["Streptococcus", "Staphylococcus", "Stenotrophomonas"] { assert!( microbiome.as_array().unwrap().iter().any(|t| { t["post_publication_description"] .as_str() .unwrap() .contains(&format!("{genus} (genus)")) }), "{genus} in PubMed 31634382's traits" ); } // Row 14: brain regions with transcriptomic datasets for HIV-1Tg. let datasets = get_json("/groups/HIV-1Tg/datasets", token).await; let full: Vec<&str> = datasets .as_array() .unwrap() .iter() .map(|d| d["full_name"].as_str().unwrap()) .collect(); for region in ["Hippocampus", "PFC", "Striatum"] { assert!( full.iter().any(|n| n.contains(region)), "{region} in {full:?}" ); } // Row 17: the BXD colon transcriptomics datasets under different diets. let datasets = get_json("/groups/BXD/datasets", token).await; let colon: BTreeSet = datasets .as_array() .unwrap() .iter() .map(|d| d["name"].as_str().unwrap().to_string()) .filter(|n| n.contains("_Col-")) .collect(); assert_eq!( colon, set_of(&[ "EPFL-LISP_BXD_Col-CD+HFD0917", "EPFL-LISP_BXD_Col-CD0917", "EPFL-LISP_BXD_Col-HFD0917" ]) ); full_benchmark_database_rows(token).await; }