about summary refs log tree commit diff
path: root/tests/search.rs
blob: 46485c6790b61b808c250bdea24b637716d5ed64 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
//! /search (phase A proxy) tests.
//!
//! No network: a stub axum app stands in for gn3, serving the recorded
//! production response (test/data/search_alcohol_phenotype.json) or a
//! parsererror, and GN3_URL points at it. /search now requires a bearer
//! token (AuthToken), so a second stub stands in for gn-auth, serving
//! GET /auth/public-jwks from the committed test JWKS fixture
//! (test/data/test_jwks.json); AUTH_SERVER_URL points at it and the
//! token is signed locally with the matching committed test private key
//! (test/data/test_rsa_private.pem), again via jsonwebtoken. This keeps
//! the whole suite hermetic -- no live gn-auth needed.
//!
//! All scenarios run inside a single test so the process-global env
//! vars (GN3_URL and AUTH_SERVER_URL) are set and read sequentially
//! (std::env::set_var is unsafe in edition 2024 because other threads
//! could read env concurrently; here no other test in this binary
//! touches them). The test uses a multi-thread runtime because
//! src/auth.rs fetches the JWKS with blocking `ureq`: on a
//! current-thread runtime that blocks the only worker and the axum
//! stubs -- spawned onto the same runtime -- never get to serve.

use axum::Router;
use axum::body::Body;
use axum::extract::Query;
use axum::http::{Request, StatusCode};
use axum::routing::get;
use http_body_util::BodyExt;
use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
use serde::Serialize;
use serde_json::{Value, json};
use std::collections::HashMap;
use tower::ServiceExt;

#[allow(dead_code)]
mod common;

/// Recorded from production on 2026-10-03:
/// https://genenetwork.org/api3/search/?query=alcohol&type=phenotype&per_page=5
const FIXTURE: &str = include_str!("../test/data/search_alcohol_phenotype.json");

/// Recorded from production on 2026-10-07:
/// https://genenetwork.org/api3/search/?query=serotonin+AND+midbrain+AND+group:BXD&type=phenotype&per_page=1000
/// The hits behind the "highest LOD ... serotonin in BXD male midbrain"
/// scenario (7 below): 119 real BXD phenotype hits, noise included.
const SEROTONIN_FIXTURE: &str = include_str!("../test/data/search_serotonin_midbrain_bxd.json");

/// Committed test-only key material: the JWKS gn-auth is stubbed with,
/// and the matching RSA private key used to sign tokens locally. See
/// test/data/test_jwks.json.
const TEST_JWKS: &str = include_str!("../test/data/test_jwks.json");
const TEST_PRIVATE_KEY: &str = include_str!("../test/data/test_rsa_private.pem");

/// Stands in for gn3's /api/search during tests.
async fn serve_gn3(Query(params): Query<HashMap<String, String>>) -> (StatusCode, String) {
    if params.get("query").is_some_and(|q| q.contains("serotonin")) {
        // gn3 has no sort of its own: a sorted /search must ask for a
        // wide page (1000), not just `limit`, to rank across it.
        if params.get("per_page").map(String::as_str) != Some("1000") {
            return (
                StatusCode::INTERNAL_SERVER_ERROR,
                "per_page must be 1000".into(),
            );
        }
        return (StatusCode::OK, SEROTONIN_FIXTURE.to_string());
    }
    if params.get("query").is_some_and(|q| q == "wide") {
        // 1000 hits, the size of a sorted search's window: more matches
        // may exist beyond it.
        let hits: Vec<Value> = (0..1000)
            .map(|i| json!({"name": i.to_string(), "dataset": "WideUnknown", "lrs": i as f64}))
            .collect();
        return (StatusCode::OK, Value::Array(hits).to_string());
    }
    if params.get("query").is_some_and(|q| q.contains("alcohol"))
        && params.get("page").map(String::as_str) == Some("2")
    {
        // Past the end: an unsorted /search forwards `page` to gn3.
        return (StatusCode::OK, "[]".to_string());
    }
    if params.get("query").is_some_and(|q| q.contains("&&")) {
        // Mirrors production gn3's shape for rejected xapian
        // queries: the diagnostic is under "error", and "error_type"
        // leaks a Python repr.
        return (
            StatusCode::BAD_REQUEST,
            json!({
                "error": "Syntax: <expression> AND <expression>",
                "error_type": "b'QueryParserError'"
            })
            .to_string(),
        );
    }
    if params.get("query").is_some_and(|q| q.contains("boom")) {
        return (StatusCode::INTERNAL_SERVER_ERROR, String::new());
    }
    (StatusCode::OK, FIXTURE.to_string())
}

async fn gn3_stub_url() -> String {
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let port = listener.local_addr().unwrap().port();
    let app = Router::new().route("/api/search/", get(serve_gn3));
    tokio::spawn(async move {
        axum::serve(listener, app).await.unwrap();
    });
    format!("http://127.0.0.1:{port}/api/")
}

/// Stands in for gn-auth's GET /auth/public-jwks, so token verification
/// never needs a live gn-auth.
async fn serve_jwks() -> (StatusCode, &'static str) {
    (StatusCode::OK, TEST_JWKS)
}

async fn gn_auth_stub_url() -> String {
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let port = listener.local_addr().unwrap().port();
    let app = Router::new().route("/auth/public-jwks", get(serve_jwks));
    tokio::spawn(async move {
        axum::serve(listener, app).await.unwrap();
    });
    format!("http://127.0.0.1:{port}")
}

/// Signs a token locally with the committed test private key, so the
/// stub's JWKS verifies it. `issuer` must equal the AUTH_SERVER_URL the
/// app is configured with (src/auth.rs validates the `iss` claim).
fn sign_test_token(issuer: &str) -> String {
    #[derive(Serialize)]
    struct Claims {
        sub: String,
        scope: String,
        iss: String,
        iat: u64,
        exp: u64,
    }

    let now = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap()
        .as_secs();
    let claims = Claims {
        sub: "search-test-user".to_string(),
        scope: "profile".to_string(),
        iss: issuer.to_string(),
        iat: now,
        exp: now + 3600,
    };
    let key = EncodingKey::from_rsa_pem(TEST_PRIVATE_KEY.as_bytes()).unwrap();
    encode(&Header::new(Algorithm::RS256), &claims, &key).unwrap()
}

fn set(key: &str, value: &str) {
    // SAFETY: this test file is the only place that touches these two
    // keys, in one single-threaded sequence (see module doc comment).
    unsafe { std::env::set_var(key, value) };
}

async fn get_json(uri: &str, token: Option<&str>) -> (StatusCode, Value) {
    let mut builder = Request::builder().uri(uri);
    if let Some(token) = token {
        builder = builder.header("Authorization", format!("Bearer {token}"));
    }
    let response = genenetwork4::app()
        .oneshot(builder.body(Body::empty()).unwrap())
        .await
        .unwrap();
    let status = response.status();
    let body = response.into_body().collect().await.unwrap().to_bytes();
    (status, serde_json::from_slice(&body).unwrap())
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn search_proxy_scenarios() {
    let gn3_stub = gn3_stub_url().await;
    let gn_auth_stub = gn_auth_stub_url().await;
    set("GN3_URL", &gn3_stub);
    set("AUTH_SERVER_URL", &gn_auth_stub);
    // The scenarios below are the gn3 *proxy*: pin the engine, or an
    // ambient SEARCH_ENGINE=native (as the dev box sets) answers them from
    // the local index and every expectation here is meaningless.
    set("SEARCH_ENGINE", "forward");

    // 0. Auth is required even for parameter validation: a request with
    //    no token is rejected by the extractor before the handler (and
    //    gn3) is ever reached. No gn-auth call happens for a missing
    //    header, so this scenario needs no token.
    let (status, body) = get_json("/search?q=alcohol", None).await;
    assert_eq!(status, StatusCode::UNAUTHORIZED);
    assert_eq!(body["code"], "invalid_token");
    // Errors, like successes, say which request they answer and how long
    // the server spent (even the token check's rejection).
    assert!(body["request_id"].is_string());
    assert!(body["took_ms"].is_u64());

    // Every remaining scenario needs a verifiable token. It's signed
    // locally and checked against the JWKS the gn-auth stub serves, so
    // no live gn-auth is involved.
    let token = sign_test_token(&gn_auth_stub);
    let token = Some(token.as_str());

    // 1. Happy path: hits pass through in gn3's shape, wrapped in the
    //    envelope.
    let (status, body) = get_json("/search?q=alcohol&type=phenotype&limit=5", token).await;
    assert_eq!(status, StatusCode::OK);
    let hits = &body["data"];
    assert_eq!(hits.as_array().unwrap().len(), 5);
    assert_eq!(hits[0]["name"], "10199");
    assert_eq!(hits[0]["dataset"], "C57BL-6JxC57BL-6NJF2Publish");
    assert!(hits[0]["lrs"].is_number());
    // The envelope: meta, a note on `lod`, and follow-ups -- the next page
    // (5 hits came back for limit=5) and the top hit's metadata.
    assert!(body["meta"]["took_ms"].is_u64());
    assert!(body["meta"]["request_id"].is_string());
    assert!(
        body["notes"]
            .as_array()
            .unwrap()
            .contains(&json!("lod = lrs / 4.604"))
    );
    let next: Vec<&str> = body["next"]
        .as_array()
        .unwrap()
        .iter()
        .map(|n| n["href"].as_str().unwrap())
        .collect();
    // The search-specific follow-ups come first, and every listing reply
    // then carries the root links (see `suggest::root_links`): the results
    // are a launch point, so `/groups`, `/species`, `/mapping-methods` and
    // the API document are offered after them.
    assert_eq!(
        &next[..2],
        [
            "/search?q=alcohol&limit=5&type=phenotype&page=2",
            "/traits/C57BL-6JxC57BL-6NJF2Publish/10199"
        ]
    );
    for root in ["/groups", "/species", "/openapi.json"] {
        assert!(next.contains(&root), "missing root link {root}: {next:?}");
    }

    // Unknown gn3 fields survive the round trip (transparent proxy).
    // (None in the fixture today; the envelope shape is what we pin.)

    // 2. Parameter validation happens before gn3 is touched.
    let (status, body) = get_json("/search?q=%20%20", token).await;
    assert_eq!(status, StatusCode::BAD_REQUEST);
    assert_eq!(body["code"], "invalid_parameter");
    assert!(body["request_id"].is_string());
    assert!(body["took_ms"].is_u64());

    let (status, body) = get_json("/search?q=alcohol&type=genotype", token).await;
    assert_eq!(status, StatusCode::BAD_REQUEST);
    assert_eq!(body["code"], "invalid_parameter");
    assert!(body["detail"].as_str().unwrap().contains("phenotype"));

    // 3. gn3 parse errors surface as 422 carrying gn3's diagnostic
    //    (assigned issue: report-invalid-search-queries-over-api).
    let (status, body) = get_json("/search?q=a%26%26b", token).await;
    assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY);
    assert_eq!(body["code"], "invalid_query");
    assert!(
        body["detail"]
            .as_str()
            .unwrap()
            .contains("Syntax: <expression> AND <expression>")
    );

    // 4. limit is capped: ask for more than 100, the stub still sees a
    //    bounded request and the envelope comes back fine.
    let (status, _body) = get_json("/search?q=alcohol&limit=9999", token).await;
    assert_eq!(status, StatusCode::OK);

    // 5. gn3 failures (5xx) surface as one 502 shape.
    let (status, body) = get_json("/search?q=boom", token).await;
    assert_eq!(status, StatusCode::BAD_GATEWAY);
    assert_eq!(body["code"], "backend_error");

    // 6. sort and order are validated before gn3 is touched.
    let (status, body) = get_json("/search?q=alcohol&sort=mean", token).await;
    assert_eq!(status, StatusCode::BAD_REQUEST);
    assert_eq!(body["code"], "invalid_parameter");
    assert!(body["detail"].as_str().unwrap().contains("lrs"));

    let (status, body) = get_json("/search?q=alcohol&sort=lod&order=sideways", token).await;
    assert_eq!(status, StatusCode::BAD_REQUEST);
    assert!(body["detail"].as_str().unwrap().contains("asc"));

    // 7. The question: "What is the highest LOD score, locus marker,
    //    and additive effect for serotonin in BXD male midbrain?"
    //    Search the BXD group ranked by LOD, then take the best hit
    //    whose description is a male-midbrain trait. (Males only:
    //    " males [" matches "adult males [" but not "adult females ["
    //    or "adult males and females [".)
    let (status, body) = get_json(
        "/search?q=serotonin%20AND%20midbrain%20AND%20group:BXD&sort=lod&limit=100",
        token,
    )
    .await;
    assert_eq!(status, StatusCode::OK);
    let hits = body["data"].as_array().unwrap();
    assert_eq!(hits.len(), 100, "119 recorded hits, truncated to limit");
    let lods: Vec<f64> = hits.iter().map(|h| h["lod"].as_f64().unwrap()).collect();
    assert!(lods.windows(2).all(|w| w[0] >= w[1]), "descending by lod");
    for hit in hits {
        let (lrs, lod) = (hit["lrs"].as_f64().unwrap(), hit["lod"].as_f64().unwrap());
        assert!((lod - lrs / 4.604).abs() < 1e-9, "lod is lrs / 4.604");
    }
    let male_midbrain = |hit: &&Value| {
        let d = hit["description"].as_str().unwrap();
        d.contains("Midbrain") && d.contains(" males [")
    };
    let is_5ht_level = |hit: &&Value| {
        hit["description"]
            .as_str()
            .unwrap()
            .contains("5-hydroxytryptamine (5-HT, serotonin) level")
    };

    // Serotonin itself (the 5-HT level): trait 17985.
    let serotonin = hits
        .iter()
        .filter(male_midbrain)
        .find(is_5ht_level)
        .expect("a male midbrain 5-HT level hit");
    assert_eq!(serotonin["name"], "17985");
    assert_eq!(serotonin["dataset"], "BXDPublish");
    assert!((serotonin["lod"].as_f64().unwrap() - 2.6063).abs() < 0.0001);
    assert!((serotonin["additive"].as_f64().unwrap() - -1.3041).abs() < 0.0001);
    assert_eq!(serotonin["geno_chr"], "16");

    // The serotonin transporter (SERT) protein in the same tissue and
    // sex has the higher LOD, and is the top male-midbrain hit overall:
    // "serotonin" in the question is ambiguous between the two.
    let top = hits.iter().find(male_midbrain).unwrap();
    assert_eq!(top["name"], "17994");
    assert!((top["lod"].as_f64().unwrap() - 3.5819).abs() < 0.0001);
    assert!((top["additive"].as_f64().unwrap() - 233.99).abs() < 0.01);

    // The peak marker's name comes from PublishXRef, so this part needs
    // a live MariaDB; without one, `locus` and `abbreviation` are null and the rest above
    // still holds.
    // gn3's pubmed_id is a declared field, not just passed through.
    assert_eq!(serotonin["pubmed_id"], 24102824);
    if common::db_available() {
        assert_eq!(serotonin["locus"], "rsm10000012461");
        assert_eq!(top["locus"], "rsm10000013922");
        assert_eq!(serotonin["abbreviation"], "5HT_MB_M");
    } else {
        eprintln!("skipping locus checks: no usable database (check SQL_URI credentials)");
    }

    // 8. order=asc reverses the ranking.
    let recorded: Vec<Value> = serde_json::from_str(SEROTONIN_FIXTURE).unwrap();
    let min_lrs = recorded
        .iter()
        .map(|h| h["lrs"].as_f64().unwrap())
        .fold(f64::INFINITY, f64::min);
    let (status, body) = get_json("/search?q=serotonin&sort=lrs&order=asc&limit=3", token).await;
    assert_eq!(status, StatusCode::OK);
    let hits = body["data"].as_array().unwrap();
    assert_eq!(hits.len(), 3);
    assert_eq!(hits[0]["lrs"].as_f64().unwrap(), min_lrs);
    assert!(hits[0]["lrs"].as_f64() <= hits[1]["lrs"].as_f64());
    assert!(hits[1]["lrs"].as_f64() <= hits[2]["lrs"].as_f64());

    // 9. page: unsorted, it is forwarded to gn3 (the stub's page 2 is
    //    empty); sorted, pages come from the ranked list (the recorded
    //    119 serotonin hits: 100, then 19, then none).
    let (status, body) = get_json("/search?q=alcohol&page=2", token).await;
    assert_eq!(status, StatusCode::OK);
    assert_eq!(body["data"], json!([]));
    let sorted = "/search?q=serotonin%20AND%20midbrain%20AND%20group:BXD&sort=lod&limit=100";
    let (_, page1) = get_json(sorted, token).await;
    let (_, page2) = get_json(&format!("{sorted}&page=2"), token).await;
    let (_, page3) = get_json(&format!("{sorted}&page=3"), token).await;
    assert_eq!(page1["data"].as_array().unwrap().len(), 100);
    assert_eq!(page2["data"].as_array().unwrap().len(), 19);
    assert_eq!(page3["data"].as_array().unwrap().len(), 0);
    let last_of_1 = page1["data"][99]["lod"].as_f64().unwrap();
    let first_of_2 = page2["data"][0]["lod"].as_f64().unwrap();
    assert!(last_of_1 >= first_of_2, "page 2 continues the ranking");
    let (status, body) = get_json("/search?q=alcohol&page=0", token).await;
    assert_eq!(status, StatusCode::BAD_REQUEST);
    assert!(body["detail"].as_str().unwrap().contains("page"));

    // 10. A sorted search over a full 1000-hit window cannot know it saw
    //     every match: the response is `partial` and says why. (These
    //     hits are not phenotype datasets, so no database is involved.)
    let (status, body) = get_json("/search?q=wide&sort=lrs&limit=3", token).await;
    assert_eq!(status, StatusCode::OK);
    assert_eq!(body["meta"]["state"], "partial");
    assert!(
        body["notes"][1]
            .as_str()
            .unwrap()
            .contains("first 1000 matches")
    );
    assert_eq!(body["data"][0]["lrs"], 999.0);
    // ... and a window that was not full is complete (the 119 recorded
    // serotonin hits are BXDPublish ones: complete only if their locus
    // lookup could reach the database).
    let (_, body) = get_json(
        "/search?q=serotonin%20AND%20midbrain%20AND%20group:BXD&sort=lod&limit=100",
        token,
    )
    .await;
    let expected = if common::db_available() {
        "complete"
    } else {
        "partial"
    };
    assert_eq!(body["meta"]["state"], expected);

    // A locally signed token with the wrong key (or a tampered one) is
    // rejected instead of reaching gn3 -- the point of the JWKS stub.
    let (status, body) = get_json("/search?q=alcohol", Some("not-a-real-token")).await;
    assert_eq!(status, StatusCode::UNAUTHORIZED);
    assert_eq!(body["code"], "invalid_token");
}