//! Native engine against the real database: the combination the rest of //! the suite leaves open. //! //! `tests/search.rs` covers the MariaDB side of `/search` — `locus` and //! `abbreviation` enrichment and the `complete`/`partial` state — but //! through the gn3 *stub* engine. `tests/search_native*.rs` cover the //! native engine against a dev index and never need a database. So the //! pair that actually matters operationally, native engine + live //! `db_webqtl`, was only ever checked by hand: `dataset:bxdpublish` //! returning `state: complete` with peak markers filled in, and its hit //! count agreeing with `/groups/{group}/traits`. //! //! This test closes that gap, and doubles as the opt-in production-index //! check: //! //! * `XAPIAN_DB_PATH` — a dev index. Runs the route and case checks; //! counts are printed as reference data only (AMENDMENT B: a dev //! index is a different snapshot, never a count oracle). //! * `GN4_FULL_INDEX` — a full/production index. Adds the strict check //! that the engine's `dataset:bxdpublish` match set IS the database's //! `PublishXRef` set for the BXD group, id for id, which is the //! invariant the dev index famously fails (it is missing ~1.7k traits //! and holds ~350 ids the database no longer has). //! * `SQL_URI` must point at a usable database (skips otherwise, like //! `tests/search.rs`). //! //! One sequential test function per binary: `SEARCH_ENGINE`, //! `XAPIAN_DB_PATH` and `AUTH_SERVER_URL` are process-global. use std::collections::BTreeSet; use axum::Router; use axum::body::Body; use axum::http::{Request, StatusCode}; use axum::routing::get; use genenetwork4::search::SearchHit; use http_body_util::BodyExt; use jsonwebtoken::{Algorithm, EncodingKey, Header, encode}; use serde::Serialize; use serde_json::Value; use tower::ServiceExt; mod common; const TEST_JWKS: &str = include_str!("../test/data/test_jwks.json"); const TEST_PRIVATE_KEY: &str = include_str!("../test/data/test_rsa_private.pem"); async fn serve_jwks() -> (StatusCode, &'static str) { (StatusCode::OK, TEST_JWKS) } fn set(key: &str, value: &str) { // SAFETY: this test binary has a single test function and no other // thread reads these keys concurrently (see the module doc). unsafe { std::env::set_var(key, value) }; } fn index_dir(path: &str) -> bool { std::fs::metadata(path).map(|m| m.is_dir()).unwrap_or(false) } async fn get_json(uri: &str, token: &str) -> (StatusCode, Value) { let request = Request::builder() .uri(uri) .header("Authorization", format!("Bearer {token}")) .body(Body::empty()) .unwrap(); let response = genenetwork4::app().oneshot(request).await.unwrap(); let status = response.status(); let body = response.into_body().collect().await.unwrap().to_bytes(); (status, serde_json::from_slice(&body).unwrap_or(Value::Null)) } /// Every hit's `name`, i.e. the PublishXRef `Id`s in the match set. fn hit_ids(hits: &[SearchHit]) -> BTreeSet { hits.iter() .filter_map(|hit| hit.name.as_deref()?.parse::().ok()) .collect() } /// The engine's full match set for `dataset:bxdpublish` (limit beyond any /// real match count: the engine retrieves the whole set regardless, this /// is only to keep the per-page invariant out of the way). async fn bxd_publish_hits() -> Vec { genenetwork4::search::search("dataset:bxdpublish", "phenotype", 1_000_000, 1) .await .expect("dataset:bxdpublish must be served by the configured engine") } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn native_engine_agrees_with_the_database() { let dev_index = std::env::var("XAPIAN_DB_PATH").unwrap_or_default(); let full_index = std::env::var("GN4_FULL_INDEX").unwrap_or_default(); // A full index wins when both are set: it is the one that can be // compared to the database. let (index, is_full) = if index_dir(&full_index) { (full_index, true) } else if index_dir(&dev_index) { (dev_index, false) } else { eprintln!( "SKIPPED native_engine_agrees_with_the_database: neither GN4_FULL_INDEX nor \ XAPIAN_DB_PATH points at a xapian index directory" ); return; }; if !common::db_available() { eprintln!( "SKIPPED native_engine_agrees_with_the_database: no usable database \ (check SQL_URI credentials)" ); return; } // The gn-auth stand-in, so token verification needs no network. let jwks_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let jwks_port = jwks_listener.local_addr().unwrap().port(); let jwks_app = Router::new().route("/auth/public-jwks", get(serve_jwks)); tokio::spawn(async move { axum::serve(jwks_listener, jwks_app).await.unwrap(); }); let auth_url = format!("http://127.0.0.1:{jwks_port}"); set("AUTH_SERVER_URL", &auth_url); set("SEARCH_ENGINE", "native"); set("XAPIAN_DB_PATH", &index); #[derive(Serialize)] struct Claims { sub: String, scope: String, iss: String, iat: u64, exp: u64, } let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_secs(); let claims = Claims { sub: "native-db-test-user".to_string(), scope: "profile".to_string(), iss: auth_url.clone(), iat: now, exp: now + 3600, }; let key = EncodingKey::from_rsa_pem(TEST_PRIVATE_KEY.as_bytes()).unwrap(); let token = encode(&Header::new(Algorithm::RS256), &claims, &key).unwrap(); eprintln!( "native+DB: {} index at {index}", if is_full { "full" } else { "dev" } ); // 1. Route level: the annotation path runs, so the response is // `complete` and at least one hit carries its peak marker and // abbreviation. (A native-engine run without a database would // report `partial` here — the state `tests/search.rs` pins for // the gn3 engine.) let (status, body) = get_json("/search?q=dataset%3Abxdpublish&limit=10", &token).await; assert_eq!(status, StatusCode::OK, "dataset:bxdpublish must serve"); assert_eq!( body["meta"]["state"], "complete", "with a usable database the annotation lookup must succeed: {body}" ); let annotated = body["data"] .as_array() .map(|hits| { hits.iter() .filter(|hit| !hit["locus"].is_null() && !hit["abbreviation"].is_null()) .count() }) .unwrap_or(0); assert!( annotated > 0, "hits must carry locus and abbreviation from PublishXRef: {body}" ); eprintln!(" dataset:bxdpublish -> 200 complete, {annotated}/10 hits annotated"); // 2. The boolean-prefix value is case-insensitive: the index holds no // mixed-case terms (`XDSBXDPublish` does not exist), so writing // the dataset name as it appears on the site must not come back // empty. Pinned here as well as in the parser unit test, because // this is the shape a caller actually sends. let lower = bxd_publish_hits().await; let upper = genenetwork4::search::search("dataset:BXDPublish", "phenotype", 1_000_000, 1) .await .expect("dataset:BXDPublish must parse and serve"); assert!( !lower.is_empty(), "dataset:bxdpublish matched nothing in the index at {index}" ); assert_eq!( upper.len(), lower.len(), "dataset:BXDPublish must match the same documents as dataset:bxdpublish" ); // 3. Against the database. The engine's phenotype match set is the // PublishXRef population, so for the BXD group it must be exactly // `/groups/{group}/traits`'s row set — same ids, both ways. let bxd = genenetwork4::db::get_group_by_name("BXD") .expect("BXD group lookup must not fail") .expect("the BXD group must exist"); let db_ids: BTreeSet = genenetwork4::db::get_group_traits(bxd.id) .expect("BXD traits must be readable") .into_iter() .map(|trait_info| trait_info.id) .collect(); let index_ids = hit_ids(&lower); eprintln!( " index {} ids vs database {} ids", index_ids.len(), db_ids.len() ); if is_full { assert_eq!( index_ids, db_ids, "the full index must hold exactly the database's PublishXRef ids for BXD \ (only in index: {:?}, only in database: {:?})", index_ids.difference(&db_ids).take(10).collect::>(), db_ids.difference(&index_ids).take(10).collect::>() ); } else { assert!( !index_ids.is_disjoint(&db_ids), "the dev index must at least overlap the database's BXD ids" ); eprintln!( " dev index is a different snapshot: {} ids only in index, {} only in database \ (reference only, never asserted)", index_ids.difference(&db_ids).count(), db_ids.difference(&index_ids).count() ); } }