diff options
| author | Pjotr <wrk@bosa.lan> | 2026-11-06 13:36:09 +0000 |
|---|---|---|
| committer | Pjotr <wrk@bosa.lan> | 2026-11-06 13:36:09 +0000 |
| commit | 3cfacfc429451d39db96fe57904db9bb04ed18d1 (patch) | |
| tree | e7e8e8fc1ed6b95c5a68eacebd8ff492dc498c0e | |
| parent | 4ea099490114ec845285642280d4def715fa0c20 (diff) | |
| download | genenetwork4-3cfacfc429451d39db96fe57904db9bb04ed18d1.tar.gz | |
search: cover the native engine with a live database
The suite covered each half of /search separately: tests/search.rs pins the MariaDB side (locus, abbreviation, complete/partial) but through the gn3 stub engine, and the search_native* tests pin the native engine against a dev index and never need a database. Native engine + live db_webqtl -- the combination that actually runs in production -- was only ever checked by hand. tests/search_native_db.rs closes that: route level, dataset:bxdpublish must come back 200 with meta.state complete and annotated hits; at the engine seam, dataset:BXDPublish must match the same documents as dataset:bxdpublish; and the match set's ids must be the database's PublishXRef ids for the BXD group. The strict id comparison runs only against a full index, selected with GN4_FULL_INDEX (skipped by default, like GN4_PARITY_FIXTURES). Against the dev index the ids are printed as reference data instead: it is a different snapshot, missing 1 772 of the database's traits and holding 349 the database no longer has, which is exactly what made the strict form worth having.
| -rw-r--r-- | tests/search_native_db.rs | 236 |
1 files changed, 236 insertions, 0 deletions
diff --git a/tests/search_native_db.rs b/tests/search_native_db.rs new file mode 100644 index 0000000..0b4fb48 --- /dev/null +++ b/tests/search_native_db.rs @@ -0,0 +1,236 @@ +//! Native engine against the real database: the combination the rest of +//! the suite leaves open. +//! +//! `tests/search.rs` covers the MariaDB side of `/search` — `locus` and +//! `abbreviation` enrichment and the `complete`/`partial` state — but +//! through the gn3 *stub* engine. `tests/search_native*.rs` cover the +//! native engine against a dev index and never need a database. So the +//! pair that actually matters operationally, native engine + live +//! `db_webqtl`, was only ever checked by hand: `dataset:bxdpublish` +//! returning `state: complete` with peak markers filled in, and its hit +//! count agreeing with `/groups/{group}/traits`. +//! +//! This test closes that gap, and doubles as the opt-in production-index +//! check: +//! +//! * `XAPIAN_DB_PATH` — a dev index. Runs the route and case checks; +//! counts are printed as reference data only (AMENDMENT B: a dev +//! index is a different snapshot, never a count oracle). +//! * `GN4_FULL_INDEX` — a full/production index. Adds the strict check +//! that the engine's `dataset:bxdpublish` match set IS the database's +//! `PublishXRef` set for the BXD group, id for id, which is the +//! invariant the dev index famously fails (it is missing ~1.7k traits +//! and holds ~350 ids the database no longer has). +//! * `SQL_URI` must point at a usable database (skips otherwise, like +//! `tests/search.rs`). +//! +//! One sequential test function per binary: `SEARCH_ENGINE`, +//! `XAPIAN_DB_PATH` and `AUTH_SERVER_URL` are process-global. + +use std::collections::BTreeSet; + +use axum::Router; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use axum::routing::get; +use genenetwork4::search::SearchHit; +use http_body_util::BodyExt; +use jsonwebtoken::{Algorithm, EncodingKey, Header, encode}; +use serde::Serialize; +use serde_json::Value; +use tower::ServiceExt; + +mod common; + +const TEST_JWKS: &str = include_str!("../test/data/test_jwks.json"); +const TEST_PRIVATE_KEY: &str = include_str!("../test/data/test_rsa_private.pem"); + +async fn serve_jwks() -> (StatusCode, &'static str) { + (StatusCode::OK, TEST_JWKS) +} + +fn set(key: &str, value: &str) { + // SAFETY: this test binary has a single test function and no other + // thread reads these keys concurrently (see the module doc). + unsafe { std::env::set_var(key, value) }; +} + +fn index_dir(path: &str) -> bool { + std::fs::metadata(path).map(|m| m.is_dir()).unwrap_or(false) +} + +async fn get_json(uri: &str, token: &str) -> (StatusCode, Value) { + let request = Request::builder() + .uri(uri) + .header("Authorization", format!("Bearer {token}")) + .body(Body::empty()) + .unwrap(); + let response = genenetwork4::app().oneshot(request).await.unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + (status, serde_json::from_slice(&body).unwrap_or(Value::Null)) +} + +/// Every hit's `name`, i.e. the PublishXRef `Id`s in the match set. +fn hit_ids(hits: &[SearchHit]) -> BTreeSet<u32> { + hits.iter() + .filter_map(|hit| hit.name.as_deref()?.parse::<u32>().ok()) + .collect() +} + +/// The engine's full match set for `dataset:bxdpublish` (limit beyond any +/// real match count: the engine retrieves the whole set regardless, this +/// is only to keep the per-page invariant out of the way). +async fn bxd_publish_hits() -> Vec<SearchHit> { + genenetwork4::search::search("dataset:bxdpublish", "phenotype", 1_000_000, 1) + .await + .expect("dataset:bxdpublish must be served by the configured engine") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_engine_agrees_with_the_database() { + let dev_index = std::env::var("XAPIAN_DB_PATH").unwrap_or_default(); + let full_index = std::env::var("GN4_FULL_INDEX").unwrap_or_default(); + + // A full index wins when both are set: it is the one that can be + // compared to the database. + let (index, is_full) = if index_dir(&full_index) { + (full_index, true) + } else if index_dir(&dev_index) { + (dev_index, false) + } else { + eprintln!( + "SKIPPED native_engine_agrees_with_the_database: neither GN4_FULL_INDEX nor \ + XAPIAN_DB_PATH points at a xapian index directory" + ); + return; + }; + if !common::db_available() { + eprintln!( + "SKIPPED native_engine_agrees_with_the_database: no usable database \ + (check SQL_URI credentials)" + ); + return; + } + + // The gn-auth stand-in, so token verification needs no network. + let jwks_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let jwks_port = jwks_listener.local_addr().unwrap().port(); + let jwks_app = Router::new().route("/auth/public-jwks", get(serve_jwks)); + tokio::spawn(async move { + axum::serve(jwks_listener, jwks_app).await.unwrap(); + }); + let auth_url = format!("http://127.0.0.1:{jwks_port}"); + set("AUTH_SERVER_URL", &auth_url); + set("SEARCH_ENGINE", "native"); + set("XAPIAN_DB_PATH", &index); + + #[derive(Serialize)] + struct Claims { + sub: String, + scope: String, + iss: String, + iat: u64, + exp: u64, + } + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let claims = Claims { + sub: "native-db-test-user".to_string(), + scope: "profile".to_string(), + iss: auth_url.clone(), + iat: now, + exp: now + 3600, + }; + let key = EncodingKey::from_rsa_pem(TEST_PRIVATE_KEY.as_bytes()).unwrap(); + let token = encode(&Header::new(Algorithm::RS256), &claims, &key).unwrap(); + + eprintln!( + "native+DB: {} index at {index}", + if is_full { "full" } else { "dev" } + ); + + // 1. Route level: the annotation path runs, so the response is + // `complete` and at least one hit carries its peak marker and + // abbreviation. (A native-engine run without a database would + // report `partial` here — the state `tests/search.rs` pins for + // the gn3 engine.) + let (status, body) = get_json("/search?q=dataset%3Abxdpublish&limit=10", &token).await; + assert_eq!(status, StatusCode::OK, "dataset:bxdpublish must serve"); + assert_eq!( + body["meta"]["state"], "complete", + "with a usable database the annotation lookup must succeed: {body}" + ); + let annotated = body["data"] + .as_array() + .map(|hits| { + hits.iter() + .filter(|hit| !hit["locus"].is_null() && !hit["abbreviation"].is_null()) + .count() + }) + .unwrap_or(0); + assert!( + annotated > 0, + "hits must carry locus and abbreviation from PublishXRef: {body}" + ); + eprintln!(" dataset:bxdpublish -> 200 complete, {annotated}/10 hits annotated"); + + // 2. The boolean-prefix value is case-insensitive: the index holds no + // mixed-case terms (`XDSBXDPublish` does not exist), so writing + // the dataset name as it appears on the site must not come back + // empty. Pinned here as well as in the parser unit test, because + // this is the shape a caller actually sends. + let lower = bxd_publish_hits().await; + let upper = genenetwork4::search::search("dataset:BXDPublish", "phenotype", 1_000_000, 1) + .await + .expect("dataset:BXDPublish must parse and serve"); + assert!( + !lower.is_empty(), + "dataset:bxdpublish matched nothing in the index at {index}" + ); + assert_eq!( + upper.len(), + lower.len(), + "dataset:BXDPublish must match the same documents as dataset:bxdpublish" + ); + + // 3. Against the database. The engine's phenotype match set is the + // PublishXRef population, so for the BXD group it must be exactly + // `/groups/{group}/traits`'s row set — same ids, both ways. + let bxd = genenetwork4::db::get_group_by_name("BXD") + .expect("BXD group lookup must not fail") + .expect("the BXD group must exist"); + let db_ids: BTreeSet<u32> = genenetwork4::db::get_group_traits(bxd.id) + .expect("BXD traits must be readable") + .into_iter() + .map(|trait_info| trait_info.id) + .collect(); + let index_ids = hit_ids(&lower); + eprintln!( + " index {} ids vs database {} ids", + index_ids.len(), + db_ids.len() + ); + if is_full { + assert_eq!( + index_ids, db_ids, + "the full index must hold exactly the database's PublishXRef ids for BXD \ + (only in index: {:?}, only in database: {:?})", + index_ids.difference(&db_ids).take(10).collect::<Vec<_>>(), + db_ids.difference(&index_ids).take(10).collect::<Vec<_>>() + ); + } else { + assert!( + !index_ids.is_disjoint(&db_ids), + "the dev index must at least overlap the database's BXD ids" + ); + eprintln!( + " dev index is a different snapshot: {} ids only in index, {} only in database \ + (reference only, never asserted)", + index_ids.difference(&db_ids).count(), + db_ids.difference(&index_ids).count() + ); + } +} |
