about summary refs log tree commit diff
path: root/src/lib.rs
blob: 976b2f187da26f851255299d9258fdc7a5fc3fc7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
use axum::Json;
use axum::Router;
use gnauth_macros::gnauth;
use tower_http::request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer};
use utoipa::OpenApi;
use utoipa_axum::router::OpenApiRouter;
use utoipa_axum::routes;

pub mod auth;
pub mod baskets;
pub mod bnw;
pub mod bulklmm;
pub mod compute_gate;
pub mod correlation;
pub mod partial;
pub mod pca;
pub mod covariates;
pub mod config;
pub mod db;
pub mod embedded;
pub mod fitqtl;
pub mod frequencies;
pub mod genoupload;
pub mod genotypes;
pub mod heatmap;
pub mod normalize;
mod plot;
mod error;
mod gemma;
pub mod indexer;
pub mod qtl2;
pub mod response;
mod routes;
mod rqtl;
pub mod search;
mod streaming;
pub mod suggest;
pub mod tokens;
pub mod wikidata;

use error::ApiError;
use response::{Ctx, Envelope};

#[derive(utoipa::OpenApi)]
#[openapi(info(
    title = "GeneNetwork4 API",
    description = "GeneNetwork4 REST API",
    license(name = "AGPL-3.0-or-later")
), security(("bearer_token" = [])), modifiers(&SecurityAddon))]
struct ApiDoc;

/// Registers the "bearer_token" scheme `ApiDoc`'s own `security(...)`
/// (above) requires by default on every operation -- every endpoint
/// needs a token unless it opts out with its own empty `security()`
/// (`hello_world`, `version`, `routes::dev::get_test_token`: see each).
struct SecurityAddon;

impl utoipa::Modify for SecurityAddon {
    fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) {
        use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityScheme};
        if let Some(components) = openapi.components.as_mut() {
            components.add_security_scheme(
                "bearer_token",
                SecurityScheme::Http(
                    HttpBuilder::new()
                        .scheme(HttpAuthScheme::Bearer)
                        .bearer_format("JWT")
                        .build(),
                ),
            );
        }
    }
}

/// Name the caller on the response, so the proxy's access log can say who
/// asked and not just from where.
///
/// The subject is only known *inside* the handler (the `AuthToken`
/// extractor verifies the token there), so this middleware opens a slot in
/// the request extensions, the extractor fills it, and the header is set on
/// the way out. Unauthenticated requests (and `401`s) carry no header --
/// there is no identity to name, and inventing one would be worse than a
/// dash.
async fn tag_caller(
    mut request: axum::extract::Request,
    next: axum::middleware::Next,
) -> axum::response::Response {
    let slot = crate::auth::CallerSlot::default();
    request.extensions_mut().insert(slot.clone());
    let started = std::time::Instant::now();
    let method = request.method().clone();
    // The path only: a query string can carry a credential (`?token=`) and has
    // no business in a log line.
    let path = request.uri().path().to_string();
    let request_id = request
        .headers()
        .get("x-request-id")
        .and_then(|value| value.to_str().ok())
        .map(str::to_string);
    let mut response = next.run(request).await;
    let caller = slot.get();
    let subject = caller.as_ref().map(|caller| caller.subject.clone());
    if let Some(subject) = &subject {
        // The subject is a gn-auth user id (an ASCII username); anything
        // else is dropped rather than turned into an invalid header.
        if let Ok(value) = axum::http::HeaderValue::from_str(subject) {
            response
                .headers_mut()
                .insert(crate::auth::USER_HEADER, value);
        }
    }
    // One line per request, with the user: the proxy's access log names the
    // caller through `x-gn4-user`, but the process log is what an operator
    // tails while watching a job, and "which user ran this" belongs in it.
    if config::Config::load().access_log {
        eprintln!(
            "{}",
            request_log_line(
                method.as_str(),
                &path,
                response.status().as_u16(),
                started.elapsed().as_millis(),
                request_id.as_deref(),
                caller.as_ref(),
            )
        );
    }
    response
}

/// The access-log line: one compact JSON object, so it greps and parses like
/// the error lines. `user` is absent rather than `"-"` when nobody was
/// identified -- an unauthenticated request has no subject, and a dash would be
/// a value a reader could mistake for a user name.
///
/// `scheme` and (for an opaque token) `token` say which credential was
/// used, never the credential itself: `{"scheme":"opaque","token":"agent-1"}`
/// lets `scripts/gn4-log.py` resolve the handle against the registry -- the
/// subject as it is *now*, the token's status and its note -- without the log
/// ever holding a secret or a hash.
fn request_log_line(
    method: &str,
    path: &str,
    status: u16,
    took_ms: u128,
    request_id: Option<&str>,
    caller: Option<&crate::auth::Caller>,
) -> String {
    let mut line = serde_json::json!({
        "event": "request",
        "method": method,
        "path": path,
        "status": status,
        "took_ms": took_ms,
    });
    if let Some(object) = line.as_object_mut() {
        object.insert("ts".to_string(), serde_json::json!(rfc3339_utc_now()));
    }
    if let (Some(request_id), Some(object)) = (request_id, line.as_object_mut()) {
        object.insert("request_id".to_string(), serde_json::json!(request_id));
    }
    if let (Some(caller), Some(object)) = (caller, line.as_object_mut()) {
        object.insert("user".to_string(), serde_json::json!(caller.subject));
        object.insert("scheme".to_string(), serde_json::json!(caller.scheme.as_str()));
        if let Some(id) = &caller.id {
            object.insert("token".to_string(), serde_json::json!(id));
        }
    }
    line.to_string()
}

/// The current time as RFC 3339 UTC, e.g. `2026-11-14T14:02:11Z`.
///
/// Hand-rolled rather than pulling in a date crate: the access log needs
/// one format, `time` is only a transitive dependency (see docs/dev/SKILL.md
/// on Cargo.lock drift), and this is the civil-from-days inverse of
/// `auth.rs`'s `days_from_civil`.
fn rfc3339_utc_now() -> String {
    let seconds = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .map(|since| since.as_secs() as i64)
        .unwrap_or(0);
    let days = seconds.div_euclid(86_400);
    let of_day = seconds.rem_euclid(86_400);
    // Howard Hinnant's civil_from_days.
    let z = days + 719_468;
    let era = z.div_euclid(146_097);
    let day_of_era = z - era * 146_097;
    let year_of_era =
        (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
    let year = year_of_era + era * 400;
    let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
    let mp = (5 * day_of_year + 2) / 153;
    let day = day_of_year - (153 * mp + 2) / 5 + 1;
    let month = if mp < 10 { mp + 3 } else { mp - 9 };
    let year = if month <= 2 { year + 1 } else { year };
    format!(
        "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z",
        of_day / 3600,
        (of_day % 3600) / 60,
        of_day % 60
    )
}

/// Build the application router. Public so integration tests (tests/) and
/// the binary (src/main.rs) can both use it without binding a real socket.
pub fn app() -> Router {
    let mut api_router = OpenApiRouter::with_openapi(ApiDoc::openapi())
        .routes(routes!(hello_world))
        .routes(routes!(version))
        .routes(routes!(routes::accessions::accession))
        .routes(routes!(routes::mapping_methods::mapping_methods))
        .routes(routes!(routes::species::species))
        .routes(routes!(routes::species::species_by_name))
        .routes(routes!(routes::species::species_datasets))
        .routes(routes!(routes::groups::groups))
        .routes(routes!(routes::groups::group_by_name))
        .routes(routes!(routes::groups::group_strains))
        .routes(routes!(routes::groups::group_trait_values))
        .routes(routes!(routes::groups::group_trait_info))
        .routes(routes!(routes::groups::group_traits))
        .routes(routes!(routes::groups::group_datasets))
        .routes(routes!(routes::groups::group_case_attributes))
        .routes(routes!(routes::groups::group_trait_case_attribute))
        .routes(routes!(routes::datasets::dataset_by_name))
        .routes(routes!(routes::datasets::dataset_probesets))
        .routes(routes!(routes::datasets::dataset_probeset_info))
        .routes(routes!(routes::datasets::dataset_probeset_values))
        .routes(routes!(routes::traits::traits))
        .routes(routes!(routes::traits::trait_info))
        .routes(routes!(routes::traits::trait_values))
        .routes(routes!(routes::publications::publication_traits))
        .routes(routes!(routes::search::search))
        .routes(routes!(routes::correlation::correlation))
        .routes(routes!(routes::correlation_matrix::correlation_matrix))
        .routes(routes!(routes::baskets::list_baskets))
        .routes(routes!(routes::baskets::create_basket))
        .routes(routes!(routes::baskets::get_basket))
        .routes(routes!(routes::baskets::basket_values))
        .routes(routes!(routes::baskets::basket_revisions))
        .routes(routes!(routes::baskets::revise_basket))
        .routes(routes!(routes::baskets::delete_basket))
        .routes(routes!(routes::peakgenotypes::peak_genotypes))
        .routes(routes!(routes::genoupload::upload_genotype))
        .routes(routes!(routes::genoupload::list_genotypes))
        .routes(routes!(routes::genoupload::get_genotype))
        .routes(routes!(routes::genoupload::genotype_markers))
        .routes(routes!(routes::genoupload::genotype_geno))
        .routes(routes!(routes::genoupload::delete_genotype))
        .routes(routes!(routes::matrices::upload_matrix))
        .routes(routes!(routes::matrices::list_matrices))
        .routes(routes!(routes::matrices::get_matrix))
        .routes(routes!(routes::matrices::matrix_values))
        .routes(routes!(routes::matrices::delete_matrix))
        .routes(routes!(routes::probesets::probeset_by_name))
        .routes(routes!(routes::probesets::gene_traits))
        .routes(routes!(routes::probesets::gene_aliases))
        .routes(routes!(routes::plot::plot_rqtl))
        .routes(routes!(routes::plot::plot_gemma))
        .routes(routes!(routes::plot::plot_genotypes))
        .routes(routes!(routes::plot::plot_bulklmm))
        .routes(routes!(routes::plot::plot_bnw))
        .routes(routes!(routes::plot::plot_pca))
        .routes(routes!(routes::plot::plot_heatmap))
        .routes(routes!(routes::phewas::group_phewas))
        .routes(routes!(routes::frequencies::group_version_frequencies))
        .routes(routes!(routes::frequencies::group_marker_frequencies))
        .routes(routes!(routes::frequencies::unlisted_frequencies))
        .routes(routes!(routes::frequencies::unlisted_marker_frequencies))
        .routes(routes!(routes::frequencies::marker_classes))
        .routes(routes!(routes::groups::group_parental_design))
        .routes(routes!(routes::genotypes::group_genotypes))
        .routes(routes!(routes::genotypes::group_genotype_version))
        .routes(routes!(routes::genotypes::group_genotype_markers))
        .routes(routes!(routes::genotypes::group_marker_location))
        .routes(routes!(routes::genotypes::unlisted_genotype_markers))
        .routes(routes!(routes::genotypes::unlisted_marker_location))
        .routes(routes!(routes::bnw::compute_bnw))
        .routes(routes!(routes::bnw::compute_bnw_features))
        .routes(routes!(routes::bnw::bnw_examples))
        .routes(routes!(routes::bnw::bnw_example))
        .routes(routes!(routes::bulklmm::compute_bulklmm))
        .routes(routes!(routes::gemma::compute_gemma))
        .routes(routes!(routes::gemma::compute_gemma_ravanan))
        .routes(routes!(routes::gemma::gemma_status))
        .routes(routes!(routes::fitqtl::compute_fitqtl))
        .routes(routes!(routes::qtl2::compute_qtl2))
        .routes(routes!(routes::rqtl::compute_rqtl))
        .routes(routes!(routes::rqtl::rqtl_status));
    // Registered -- and so reachable and listed in /openapi.json -- unless
    // GENENETWORK4_TEST_TOKEN switches it off; when registered it decides
    // per request whether the caller is local. See src/routes/dev.rs.
    if config::Config::load().test_token_enabled {
        api_router = api_router.routes(routes!(routes::dev::get_test_token));
    }
    // Administering the short API tokens. Always registered, and each
    // handler refuses anything that did not arrive directly from this
    // machine (see src/routes/tokens.rs): minting credentials must not be
    // reachable from the network, so there is no switch to turn that off.
    // A sub-router so the localhost guard is a layer over exactly these
    // four routes: it must refuse a remote caller before anything of the
    // request is parsed (see routes::tokens::local_only).
    let token_router = OpenApiRouter::new()
        .routes(routes!(routes::tokens::list_tokens))
        .routes(routes!(routes::tokens::mint_token))
        .routes(routes!(routes::tokens::revoke_token))
        .routes(routes!(routes::tokens::who_am_i))
        .route_layer(axum::middleware::from_fn(routes::tokens::local_only));
    api_router = api_router.merge(token_router);
    let (router, api) = api_router.split_for_parts();
    router
        .route("/openapi.json", axum::routing::get(|| async { Json(api) }))
        // axum's default is 2 MB, which a request legitimately exceeds:
        // /plot/* takes a whole mapping response back (a real GEMMA run
        // on the BXD WGS panel is ~1.9 MB, and a genome-sized run is
        // tens of MB), and /compute/* carries one value per individual
        // plus up to eight covariates. A body over the limit is refused
        // with axum's own plain-text 413 before any handler sees it, so
        // the limit has to sit above what the API accepts. nginx's
        // client_max_body_size must be at least as large (64m in
        // etc/nginx-vesuvio.pu.ac.ke.conf), or it refuses first.
        .layer(axum::extract::DefaultBodyLimit::max(BODY_LIMIT_BYTES))
        // Assigns each request an x-request-id (if the client didn't
        // already send one) and copies it onto the response, so it can be
        // included in ApiError bodies (see error.rs) and correlated
        // with logs.
        .layer(PropagateRequestIdLayer::x_request_id())
        .layer(SetRequestIdLayer::x_request_id(MakeRequestUuid))
        // Outermost, so the slot exists before the handler's extractors
        // run: see `tag_caller`.
        .layer(axum::middleware::from_fn(tag_caller))
}

/// The largest request body the API accepts: 64 MiB, comfortably above a
/// whole mapping result posted back to /plot/* and above a panel's worth
/// of samples plus covariates, and still small enough that a mistake
/// cannot exhaust memory.
const BODY_LIMIT_BYTES: usize = 64 * 1024 * 1024;

// hello_world and version predate the domain-module split and don't
// belong to a data domain; kept here as top-level utility routes.
// hello_world (a plain-text liveness greeting) is the one handler
// whose response is not wrapped in the envelope, like /openapi.json.
// Both opt out of ApiDoc's default security requirement with an empty
// security() -- neither takes an AuthToken, so documenting them as
// needing one would be wrong, not just inconsistent. #[gnauth(disabled)]
// below makes that a compile-time assertion, not just this comment.
#[utoipa::path(get, path = "/", summary = "Greeting; confirms the server is up", responses(
    (status = 200, description = "Greeting", body = String),
    (status = 500, description = "Internal error", body = ApiError)
), security())]
#[gnauth(disabled)]
async fn hello_world() -> &'static str {
    "Hello, world!"
}

#[derive(serde::Serialize, utoipa::ToSchema)]
struct VersionInfo {
    name: &'static str,
    short_name: &'static str,
    version: f64,
    release: u32,
    /// The HEAD commit's date (`YYYYMMDD`), not the build time.
    date: &'static str,
    /// The commit this binary was built from, e.g. "18d18b1", or
    /// "18d18b1-dirty" when the working tree had uncommitted changes at
    /// build time. Compare the part before "-dirty" with a clone's
    /// `git log -1 --format=%h`.
    git: &'static str,
    /// Whether that build was from a dirty tree. When true, `git` names
    /// the commit whose *neighbourhood* the binary was built from, and
    /// `source_digest` is what identifies the code.
    dirty: bool,
    /// A digest of the compiled source files: two builds of the same
    /// commit (or of the same dirty tree) can be told apart by this,
    /// which a bare commit hash cannot do.
    source_digest: &'static str,
    /// The branch the build's HEAD was on, or "HEAD" when detached.
    branch: &'static str,
    /// Whether the database answers, checked at most every ten seconds.
    /// A service that answers `/version` while every data endpoint fails
    /// is the state this makes visible.
    database: &'static str,
}

#[utoipa::path(get, path = "/version", summary = "Build information: the git commit the binary was built from, and a source digest", responses(
    (status = 200, description = "Build information", body = Envelope<VersionInfo>),
    (status = 500, description = "Internal error", body = ApiError)
), security())]
#[gnauth(disabled)]
async fn version(ctx: Ctx) -> Json<Envelope<VersionInfo>> {
    let git = env!("GN4_GIT_HASH");
    // No `notes` in the ordinary case: the fields say what they are (see
    // VersionInfo's doc comments), and /version is the cheapest endpoint
    // for the envelope test's "notes is an empty array" case. The
    // exception is a database failure in the last few minutes, which is
    // the one thing a caller cannot otherwise see from here -- `database`
    // is a *now* probe, so an intermittent fault leaves every data
    // endpoint failing with nothing on this endpoint to say so.
    let mut envelope = ctx.envelope(VersionInfo {
        name: "GeneNetwork4",
        short_name: "GN4AI",
        version: 4.0,
        release: 1,
        date: env!("GN4_COMMIT_DATE"),
        git,
        dirty: git.ends_with("-dirty"),
        source_digest: env!("GN4_SOURCE_DIGEST"),
        branch: env!("GN4_GIT_BRANCH"),
        database: if crate::db::health() {
            "reachable"
        } else {
            "unreachable"
        },
    });
    if let Some(failure) = crate::db::recent_failure(std::time::Duration::from_secs(600)) {
        let seconds = failure.at.elapsed().as_secs();
        envelope = envelope.with_note(&format!(
            "A database request failed {}s ago ({}): {}. `database` is a probe of the \
             connection pool as well, so `reachable` here does not mean the failure did not \
             happen -- it means one is not happening now. The pool is rebuilt when a \
             connection dies, so a single such line is usually recovered from.",
            seconds,
            failure.kind.as_str(),
            failure.detail.chars().take(200).collect::<String>()
        ));
    }
    Json(envelope.with_links(crate::suggest::root_links()))
}

#[cfg(test)]
mod access_log_tests {
    use super::request_log_line;
    use crate::auth::{Caller, Scheme};

    fn caller(scheme: Scheme, id: Option<&str>, subject: &str) -> Caller {
        Caller {
            scheme,
            id: id.map(str::to_string),
            subject: subject.to_string(),
        }
    }

    #[test]
    fn the_line_names_the_user_and_never_the_query() {
        let line = request_log_line(
            "GET",
            "/groups/BXD",
            200,
            17,
            Some("abc"),
            Some(&caller(Scheme::Jwt, None, "zachs")),
        );
        let parsed: serde_json::Value = serde_json::from_str(&line).unwrap();
        assert_eq!(parsed["event"], "request");
        assert_eq!(parsed["method"], "GET");
        assert_eq!(parsed["path"], "/groups/BXD");
        assert_eq!(parsed["status"], 200);
        assert_eq!(parsed["took_ms"], 17);
        assert_eq!(parsed["request_id"], "abc");
        assert_eq!(parsed["user"], "zachs");
        assert_eq!(parsed["scheme"], "jwt");
        assert!(
            parsed["ts"].as_str().is_some_and(|ts| ts.len() == 20 && ts.ends_with('Z')),
            "the line is self-describing: {}",
            parsed["ts"]
        );
        assert!(parsed.get("token").is_none(), "a JWT has no registry handle");
    }

    /// An opaque token names its registry handle, never the token or its
    /// hash: the handle is what a reader resolves against `tokens.sqlite`.
    #[test]
    fn an_opaque_token_logs_its_handle_and_not_the_secret() {
        let line = request_log_line(
            "POST",
            "/compute/gemma",
            200,
            900,
            None,
            Some(&caller(Scheme::Opaque, Some("agent-ravanan-3"), "zachs")),
        );
        let parsed: serde_json::Value = serde_json::from_str(&line).unwrap();
        assert_eq!(parsed["user"], "zachs");
        assert_eq!(parsed["scheme"], "opaque");
        assert_eq!(parsed["token"], "agent-ravanan-3");
        assert!(!line.contains("gn4ai_"), "{line}");
    }

    /// An unauthenticated request has no subject; a dash would be a value a
    /// reader could mistake for a user name.
    #[test]
    fn no_user_field_when_nobody_was_identified() {
        let line = request_log_line("GET", "/groups", 401, 0, None, None);
        let parsed: serde_json::Value = serde_json::from_str(&line).unwrap();
        assert!(parsed.get("user").is_none(), "{line}");
        assert!(parsed.get("request_id").is_none(), "{line}");
    }
}