about summary refs log tree commit diff
path: root/tests/search_native_db.rs
blob: 0b4fb480f0efb9e097ef10f9cc7e25cb86d074e3 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
//! Native engine against the real database: the combination the rest of
//! the suite leaves open.
//!
//! `tests/search.rs` covers the MariaDB side of `/search` — `locus` and
//! `abbreviation` enrichment and the `complete`/`partial` state — but
//! through the gn3 *stub* engine. `tests/search_native*.rs` cover the
//! native engine against a dev index and never need a database. So the
//! pair that actually matters operationally, native engine + live
//! `db_webqtl`, was only ever checked by hand: `dataset:bxdpublish`
//! returning `state: complete` with peak markers filled in, and its hit
//! count agreeing with `/groups/{group}/traits`.
//!
//! This test closes that gap, and doubles as the opt-in production-index
//! check:
//!
//! * `XAPIAN_DB_PATH` — a dev index. Runs the route and case checks;
//!   counts are printed as reference data only (AMENDMENT B: a dev
//!   index is a different snapshot, never a count oracle).
//! * `GN4_FULL_INDEX` — a full/production index. Adds the strict check
//!   that the engine's `dataset:bxdpublish` match set IS the database's
//!   `PublishXRef` set for the BXD group, id for id, which is the
//!   invariant the dev index famously fails (it is missing ~1.7k traits
//!   and holds ~350 ids the database no longer has).
//! * `SQL_URI` must point at a usable database (skips otherwise, like
//!   `tests/search.rs`).
//!
//! One sequential test function per binary: `SEARCH_ENGINE`,
//! `XAPIAN_DB_PATH` and `AUTH_SERVER_URL` are process-global.

use std::collections::BTreeSet;

use axum::Router;
use axum::body::Body;
use axum::http::{Request, StatusCode};
use axum::routing::get;
use genenetwork4::search::SearchHit;
use http_body_util::BodyExt;
use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
use serde::Serialize;
use serde_json::Value;
use tower::ServiceExt;

mod common;

const TEST_JWKS: &str = include_str!("../test/data/test_jwks.json");
const TEST_PRIVATE_KEY: &str = include_str!("../test/data/test_rsa_private.pem");

async fn serve_jwks() -> (StatusCode, &'static str) {
    (StatusCode::OK, TEST_JWKS)
}

fn set(key: &str, value: &str) {
    // SAFETY: this test binary has a single test function and no other
    // thread reads these keys concurrently (see the module doc).
    unsafe { std::env::set_var(key, value) };
}

fn index_dir(path: &str) -> bool {
    std::fs::metadata(path).map(|m| m.is_dir()).unwrap_or(false)
}

async fn get_json(uri: &str, token: &str) -> (StatusCode, Value) {
    let request = Request::builder()
        .uri(uri)
        .header("Authorization", format!("Bearer {token}"))
        .body(Body::empty())
        .unwrap();
    let response = genenetwork4::app().oneshot(request).await.unwrap();
    let status = response.status();
    let body = response.into_body().collect().await.unwrap().to_bytes();
    (status, serde_json::from_slice(&body).unwrap_or(Value::Null))
}

/// Every hit's `name`, i.e. the PublishXRef `Id`s in the match set.
fn hit_ids(hits: &[SearchHit]) -> BTreeSet<u32> {
    hits.iter()
        .filter_map(|hit| hit.name.as_deref()?.parse::<u32>().ok())
        .collect()
}

/// The engine's full match set for `dataset:bxdpublish` (limit beyond any
/// real match count: the engine retrieves the whole set regardless, this
/// is only to keep the per-page invariant out of the way).
async fn bxd_publish_hits() -> Vec<SearchHit> {
    genenetwork4::search::search("dataset:bxdpublish", "phenotype", 1_000_000, 1)
        .await
        .expect("dataset:bxdpublish must be served by the configured engine")
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn native_engine_agrees_with_the_database() {
    let dev_index = std::env::var("XAPIAN_DB_PATH").unwrap_or_default();
    let full_index = std::env::var("GN4_FULL_INDEX").unwrap_or_default();

    // A full index wins when both are set: it is the one that can be
    // compared to the database.
    let (index, is_full) = if index_dir(&full_index) {
        (full_index, true)
    } else if index_dir(&dev_index) {
        (dev_index, false)
    } else {
        eprintln!(
            "SKIPPED native_engine_agrees_with_the_database: neither GN4_FULL_INDEX nor \
             XAPIAN_DB_PATH points at a xapian index directory"
        );
        return;
    };
    if !common::db_available() {
        eprintln!(
            "SKIPPED native_engine_agrees_with_the_database: no usable database \
             (check SQL_URI credentials)"
        );
        return;
    }

    // The gn-auth stand-in, so token verification needs no network.
    let jwks_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let jwks_port = jwks_listener.local_addr().unwrap().port();
    let jwks_app = Router::new().route("/auth/public-jwks", get(serve_jwks));
    tokio::spawn(async move {
        axum::serve(jwks_listener, jwks_app).await.unwrap();
    });
    let auth_url = format!("http://127.0.0.1:{jwks_port}");
    set("AUTH_SERVER_URL", &auth_url);
    set("SEARCH_ENGINE", "native");
    set("XAPIAN_DB_PATH", &index);

    #[derive(Serialize)]
    struct Claims {
        sub: String,
        scope: String,
        iss: String,
        iat: u64,
        exp: u64,
    }
    let now = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap()
        .as_secs();
    let claims = Claims {
        sub: "native-db-test-user".to_string(),
        scope: "profile".to_string(),
        iss: auth_url.clone(),
        iat: now,
        exp: now + 3600,
    };
    let key = EncodingKey::from_rsa_pem(TEST_PRIVATE_KEY.as_bytes()).unwrap();
    let token = encode(&Header::new(Algorithm::RS256), &claims, &key).unwrap();

    eprintln!(
        "native+DB: {} index at {index}",
        if is_full { "full" } else { "dev" }
    );

    // 1. Route level: the annotation path runs, so the response is
    //    `complete` and at least one hit carries its peak marker and
    //    abbreviation. (A native-engine run without a database would
    //    report `partial` here — the state `tests/search.rs` pins for
    //    the gn3 engine.)
    let (status, body) = get_json("/search?q=dataset%3Abxdpublish&limit=10", &token).await;
    assert_eq!(status, StatusCode::OK, "dataset:bxdpublish must serve");
    assert_eq!(
        body["meta"]["state"], "complete",
        "with a usable database the annotation lookup must succeed: {body}"
    );
    let annotated = body["data"]
        .as_array()
        .map(|hits| {
            hits.iter()
                .filter(|hit| !hit["locus"].is_null() && !hit["abbreviation"].is_null())
                .count()
        })
        .unwrap_or(0);
    assert!(
        annotated > 0,
        "hits must carry locus and abbreviation from PublishXRef: {body}"
    );
    eprintln!("  dataset:bxdpublish -> 200 complete, {annotated}/10 hits annotated");

    // 2. The boolean-prefix value is case-insensitive: the index holds no
    //    mixed-case terms (`XDSBXDPublish` does not exist), so writing
    //    the dataset name as it appears on the site must not come back
    //    empty. Pinned here as well as in the parser unit test, because
    //    this is the shape a caller actually sends.
    let lower = bxd_publish_hits().await;
    let upper = genenetwork4::search::search("dataset:BXDPublish", "phenotype", 1_000_000, 1)
        .await
        .expect("dataset:BXDPublish must parse and serve");
    assert!(
        !lower.is_empty(),
        "dataset:bxdpublish matched nothing in the index at {index}"
    );
    assert_eq!(
        upper.len(),
        lower.len(),
        "dataset:BXDPublish must match the same documents as dataset:bxdpublish"
    );

    // 3. Against the database. The engine's phenotype match set is the
    //    PublishXRef population, so for the BXD group it must be exactly
    //    `/groups/{group}/traits`'s row set — same ids, both ways.
    let bxd = genenetwork4::db::get_group_by_name("BXD")
        .expect("BXD group lookup must not fail")
        .expect("the BXD group must exist");
    let db_ids: BTreeSet<u32> = genenetwork4::db::get_group_traits(bxd.id)
        .expect("BXD traits must be readable")
        .into_iter()
        .map(|trait_info| trait_info.id)
        .collect();
    let index_ids = hit_ids(&lower);
    eprintln!(
        "  index {} ids vs database {} ids",
        index_ids.len(),
        db_ids.len()
    );
    if is_full {
        assert_eq!(
            index_ids, db_ids,
            "the full index must hold exactly the database's PublishXRef ids for BXD \
             (only in index: {:?}, only in database: {:?})",
            index_ids.difference(&db_ids).take(10).collect::<Vec<_>>(),
            db_ids.difference(&index_ids).take(10).collect::<Vec<_>>()
        );
    } else {
        assert!(
            !index_ids.is_disjoint(&db_ids),
            "the dev index must at least overlap the database's BXD ids"
        );
        eprintln!(
            "  dev index is a different snapshot: {} ids only in index, {} only in database \
             (reference only, never asserted)",
            index_ids.difference(&db_ids).count(),
            db_ids.difference(&index_ids).count()
        );
    }
}