about summary refs log tree commit diff
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-11-06 13:36:09 +0000
committerPjotr <wrk@bosa.lan>2026-11-06 13:36:09 +0000
commit3cfacfc429451d39db96fe57904db9bb04ed18d1 (patch)
treee7e8e8fc1ed6b95c5a68eacebd8ff492dc498c0e
parent4ea099490114ec845285642280d4def715fa0c20 (diff)
downloadgenenetwork4-3cfacfc429451d39db96fe57904db9bb04ed18d1.tar.gz
search: cover the native engine with a live database
The suite covered each half of /search separately: tests/search.rs pins
the MariaDB side (locus, abbreviation, complete/partial) but through the
gn3 stub engine, and the search_native* tests pin the native engine
against a dev index and never need a database. Native engine + live
db_webqtl -- the combination that actually runs in production -- was
only ever checked by hand.

tests/search_native_db.rs closes that: route level, dataset:bxdpublish
must come back 200 with meta.state complete and annotated hits; at the
engine seam, dataset:BXDPublish must match the same documents as
dataset:bxdpublish; and the match set's ids must be the database's
PublishXRef ids for the BXD group.

The strict id comparison runs only against a full index, selected with
GN4_FULL_INDEX (skipped by default, like GN4_PARITY_FIXTURES). Against
the dev index the ids are printed as reference data instead: it is a
different snapshot, missing 1 772 of the database's traits and holding
349 the database no longer has, which is exactly what made the strict
form worth having.
-rw-r--r--tests/search_native_db.rs236
1 files changed, 236 insertions, 0 deletions
diff --git a/tests/search_native_db.rs b/tests/search_native_db.rs
new file mode 100644
index 0000000..0b4fb48
--- /dev/null
+++ b/tests/search_native_db.rs
@@ -0,0 +1,236 @@
+//! Native engine against the real database: the combination the rest of
+//! the suite leaves open.
+//!
+//! `tests/search.rs` covers the MariaDB side of `/search` — `locus` and
+//! `abbreviation` enrichment and the `complete`/`partial` state — but
+//! through the gn3 *stub* engine. `tests/search_native*.rs` cover the
+//! native engine against a dev index and never need a database. So the
+//! pair that actually matters operationally, native engine + live
+//! `db_webqtl`, was only ever checked by hand: `dataset:bxdpublish`
+//! returning `state: complete` with peak markers filled in, and its hit
+//! count agreeing with `/groups/{group}/traits`.
+//!
+//! This test closes that gap, and doubles as the opt-in production-index
+//! check:
+//!
+//! * `XAPIAN_DB_PATH` — a dev index. Runs the route and case checks;
+//!   counts are printed as reference data only (AMENDMENT B: a dev
+//!   index is a different snapshot, never a count oracle).
+//! * `GN4_FULL_INDEX` — a full/production index. Adds the strict check
+//!   that the engine's `dataset:bxdpublish` match set IS the database's
+//!   `PublishXRef` set for the BXD group, id for id, which is the
+//!   invariant the dev index famously fails (it is missing ~1.7k traits
+//!   and holds ~350 ids the database no longer has).
+//! * `SQL_URI` must point at a usable database (skips otherwise, like
+//!   `tests/search.rs`).
+//!
+//! One sequential test function per binary: `SEARCH_ENGINE`,
+//! `XAPIAN_DB_PATH` and `AUTH_SERVER_URL` are process-global.
+
+use std::collections::BTreeSet;
+
+use axum::Router;
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use axum::routing::get;
+use genenetwork4::search::SearchHit;
+use http_body_util::BodyExt;
+use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
+use serde::Serialize;
+use serde_json::Value;
+use tower::ServiceExt;
+
+mod common;
+
+const TEST_JWKS: &str = include_str!("../test/data/test_jwks.json");
+const TEST_PRIVATE_KEY: &str = include_str!("../test/data/test_rsa_private.pem");
+
+async fn serve_jwks() -> (StatusCode, &'static str) {
+    (StatusCode::OK, TEST_JWKS)
+}
+
+fn set(key: &str, value: &str) {
+    // SAFETY: this test binary has a single test function and no other
+    // thread reads these keys concurrently (see the module doc).
+    unsafe { std::env::set_var(key, value) };
+}
+
+fn index_dir(path: &str) -> bool {
+    std::fs::metadata(path).map(|m| m.is_dir()).unwrap_or(false)
+}
+
+async fn get_json(uri: &str, token: &str) -> (StatusCode, Value) {
+    let request = Request::builder()
+        .uri(uri)
+        .header("Authorization", format!("Bearer {token}"))
+        .body(Body::empty())
+        .unwrap();
+    let response = genenetwork4::app().oneshot(request).await.unwrap();
+    let status = response.status();
+    let body = response.into_body().collect().await.unwrap().to_bytes();
+    (status, serde_json::from_slice(&body).unwrap_or(Value::Null))
+}
+
+/// Every hit's `name`, i.e. the PublishXRef `Id`s in the match set.
+fn hit_ids(hits: &[SearchHit]) -> BTreeSet<u32> {
+    hits.iter()
+        .filter_map(|hit| hit.name.as_deref()?.parse::<u32>().ok())
+        .collect()
+}
+
+/// The engine's full match set for `dataset:bxdpublish` (limit beyond any
+/// real match count: the engine retrieves the whole set regardless, this
+/// is only to keep the per-page invariant out of the way).
+async fn bxd_publish_hits() -> Vec<SearchHit> {
+    genenetwork4::search::search("dataset:bxdpublish", "phenotype", 1_000_000, 1)
+        .await
+        .expect("dataset:bxdpublish must be served by the configured engine")
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn native_engine_agrees_with_the_database() {
+    let dev_index = std::env::var("XAPIAN_DB_PATH").unwrap_or_default();
+    let full_index = std::env::var("GN4_FULL_INDEX").unwrap_or_default();
+
+    // A full index wins when both are set: it is the one that can be
+    // compared to the database.
+    let (index, is_full) = if index_dir(&full_index) {
+        (full_index, true)
+    } else if index_dir(&dev_index) {
+        (dev_index, false)
+    } else {
+        eprintln!(
+            "SKIPPED native_engine_agrees_with_the_database: neither GN4_FULL_INDEX nor \
+             XAPIAN_DB_PATH points at a xapian index directory"
+        );
+        return;
+    };
+    if !common::db_available() {
+        eprintln!(
+            "SKIPPED native_engine_agrees_with_the_database: no usable database \
+             (check SQL_URI credentials)"
+        );
+        return;
+    }
+
+    // The gn-auth stand-in, so token verification needs no network.
+    let jwks_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let jwks_port = jwks_listener.local_addr().unwrap().port();
+    let jwks_app = Router::new().route("/auth/public-jwks", get(serve_jwks));
+    tokio::spawn(async move {
+        axum::serve(jwks_listener, jwks_app).await.unwrap();
+    });
+    let auth_url = format!("http://127.0.0.1:{jwks_port}");
+    set("AUTH_SERVER_URL", &auth_url);
+    set("SEARCH_ENGINE", "native");
+    set("XAPIAN_DB_PATH", &index);
+
+    #[derive(Serialize)]
+    struct Claims {
+        sub: String,
+        scope: String,
+        iss: String,
+        iat: u64,
+        exp: u64,
+    }
+    let now = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_secs();
+    let claims = Claims {
+        sub: "native-db-test-user".to_string(),
+        scope: "profile".to_string(),
+        iss: auth_url.clone(),
+        iat: now,
+        exp: now + 3600,
+    };
+    let key = EncodingKey::from_rsa_pem(TEST_PRIVATE_KEY.as_bytes()).unwrap();
+    let token = encode(&Header::new(Algorithm::RS256), &claims, &key).unwrap();
+
+    eprintln!(
+        "native+DB: {} index at {index}",
+        if is_full { "full" } else { "dev" }
+    );
+
+    // 1. Route level: the annotation path runs, so the response is
+    //    `complete` and at least one hit carries its peak marker and
+    //    abbreviation. (A native-engine run without a database would
+    //    report `partial` here — the state `tests/search.rs` pins for
+    //    the gn3 engine.)
+    let (status, body) = get_json("/search?q=dataset%3Abxdpublish&limit=10", &token).await;
+    assert_eq!(status, StatusCode::OK, "dataset:bxdpublish must serve");
+    assert_eq!(
+        body["meta"]["state"], "complete",
+        "with a usable database the annotation lookup must succeed: {body}"
+    );
+    let annotated = body["data"]
+        .as_array()
+        .map(|hits| {
+            hits.iter()
+                .filter(|hit| !hit["locus"].is_null() && !hit["abbreviation"].is_null())
+                .count()
+        })
+        .unwrap_or(0);
+    assert!(
+        annotated > 0,
+        "hits must carry locus and abbreviation from PublishXRef: {body}"
+    );
+    eprintln!("  dataset:bxdpublish -> 200 complete, {annotated}/10 hits annotated");
+
+    // 2. The boolean-prefix value is case-insensitive: the index holds no
+    //    mixed-case terms (`XDSBXDPublish` does not exist), so writing
+    //    the dataset name as it appears on the site must not come back
+    //    empty. Pinned here as well as in the parser unit test, because
+    //    this is the shape a caller actually sends.
+    let lower = bxd_publish_hits().await;
+    let upper = genenetwork4::search::search("dataset:BXDPublish", "phenotype", 1_000_000, 1)
+        .await
+        .expect("dataset:BXDPublish must parse and serve");
+    assert!(
+        !lower.is_empty(),
+        "dataset:bxdpublish matched nothing in the index at {index}"
+    );
+    assert_eq!(
+        upper.len(),
+        lower.len(),
+        "dataset:BXDPublish must match the same documents as dataset:bxdpublish"
+    );
+
+    // 3. Against the database. The engine's phenotype match set is the
+    //    PublishXRef population, so for the BXD group it must be exactly
+    //    `/groups/{group}/traits`'s row set — same ids, both ways.
+    let bxd = genenetwork4::db::get_group_by_name("BXD")
+        .expect("BXD group lookup must not fail")
+        .expect("the BXD group must exist");
+    let db_ids: BTreeSet<u32> = genenetwork4::db::get_group_traits(bxd.id)
+        .expect("BXD traits must be readable")
+        .into_iter()
+        .map(|trait_info| trait_info.id)
+        .collect();
+    let index_ids = hit_ids(&lower);
+    eprintln!(
+        "  index {} ids vs database {} ids",
+        index_ids.len(),
+        db_ids.len()
+    );
+    if is_full {
+        assert_eq!(
+            index_ids, db_ids,
+            "the full index must hold exactly the database's PublishXRef ids for BXD \
+             (only in index: {:?}, only in database: {:?})",
+            index_ids.difference(&db_ids).take(10).collect::<Vec<_>>(),
+            db_ids.difference(&index_ids).take(10).collect::<Vec<_>>()
+        );
+    } else {
+        assert!(
+            !index_ids.is_disjoint(&db_ids),
+            "the dev index must at least overlap the database's BXD ids"
+        );
+        eprintln!(
+            "  dev index is a different snapshot: {} ids only in index, {} only in database \
+             (reference only, never asserted)",
+            index_ids.difference(&db_ids).count(),
+            db_ids.difference(&index_ids).count()
+        );
+    }
+}