//! `GET /probesets/{name}` and `GET /genes/{symbol}/traits`: cross-dataset //! lookups over the expression traits. //! //! Both are questions the search index cannot answer. A probeset id is not a //! Xapian field, so `1427571_at` finds nothing exact and the only route to it //! is guessing a dataset and calling `/datasets/{ds}/probesets/{name}`. //! A gene symbol can be searched, but the reply is capped, ungrouped and //! counts different probesets and platforms in one list. These two endpoints //! are the indexed lookups: `ProbeSet.Name` and `ProbeSet.Symbol` both have //! one, and the answer carries the panel and the mapping statistics. //! //! `/genes/{symbol}/aliases` is the annotation lookup beside them: Wikidata's //! `skos:altLabel`s for a gene, cached in LMDB for seven days (see //! src/wikidata.rs) and served stale rather than dropped when Wikidata is //! unreachable. //! //! What these return is **metadata**: which datasets carry a probeset, and //! what its QTL scan found in each. It deliberately does not return trait //! values -- those are per-dataset and authorized per dataset, and the `next` //! link goes to the endpoint that checks. So a token buys the index; the //! values stay where they always were. use axum::Json; use axum::extract::{Path, Query}; use axum::http::StatusCode; use serde::Deserialize; use gnauth_macros::gnauth; use crate::auth::AuthToken; use crate::db::{self, ProbesetHit}; use crate::config::Config; use crate::error::{ApiError, Unauthorized}; use crate::response::{Ctx, Envelope, path_segment}; use crate::suggest::{self, Link}; use crate::wikidata::{self, GeneAliases, WikidataError}; #[derive(Deserialize, utoipa::IntoParams)] pub struct ProbesetParams { /// Only this group (an InbredSet name, e.g. BXD). pub group: Option, /// Only this species (e.g. mouse). pub species: Option, /// Only datasets whose platform label contains this, e.g. M430. pub platform: Option, /// One-based page. Default 1. pub page: Option, /// Rows per page. Default 100, at most 5000. pub limit: Option, } #[derive(Deserialize, utoipa::IntoParams)] pub struct GeneParams { /// Only this group (an InbredSet name, e.g. BXD). pub group: Option, /// Only this species (e.g. mouse). pub species: Option, /// Only datasets whose platform label contains this, e.g. M430. pub platform: Option, /// `dataset` (default) or `lod`, strongest peak first. pub sort: Option, /// Only traits whose peak reached at least this LOD. pub min_lod: Option, /// One-based page. Default 1. pub page: Option, /// Rows per page. Default 100, at most 5000. pub limit: Option, } /// The columns of both replies, in this order. const COLUMNS: [&str; 13] = [ "dataset", "group", "species", "platform", "name", "symbol", "chr", "mb", "mean", "lrs", "lod", "locus", "additive", ]; #[utoipa::path(get, path = "/probesets/{name}", summary = "Every dataset that carries a probeset with this name", description = "A cross-dataset lookup on `ProbeSet.Name` (indexed): one row per dataset, \ with the panel (group, species, platform) and the probeset's mapping statistics there (mean, \ LRS, LOD, peak locus, additive). Suffixed variants match too -- `1427571_at` also finds \ `1427571_at_A` -- and each row's `name` says which one it was. Metadata only: values are \ per-dataset and live at /datasets/{dataset}/probesets/{name}/values.", params(("name" = String, Path, description = "Probeset id, e.g. 1427571_at"), ProbesetParams), responses( (status = 200, description = "One row per dataset, positional (meta.columns names them)", body = Envelope), Unauthorized, (status = 404, description = "No probeset with that name", body = ApiError) ) )] #[gnauth(token_only)] pub async fn probeset_by_name( ctx: Ctx, Path(name): Path, Query(params): Query, _token: AuthToken, ) -> Result>, ApiError> { let rid = ctx.request_info(); let lookup = name.clone(); let hits = tokio::task::spawn_blocking(move || db::probeset_hits(Some(&lookup), None)) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|error| ApiError::from_db(error, "probeset_by_name").with_request_id(rid.clone()))?; if hits.is_empty() { return Err(ApiError::new( StatusCode::NOT_FOUND, "probeset_not_found", "Probeset not found", format!("No probeset named '{name}' (or a suffixed variant of it)."), "probeset_by_name", ) .with_request_id(rid)); } let total = hits.len(); let filtered = filter(hits, ¶ms.group, ¶ms.species, ¶ms.platform); let matched = filtered.len(); let page = page(&filtered, params.page, params.limit); let datasets: Vec = filtered .iter() .map(|hit| hit.dataset.clone()) .collect::>() .into_iter() .collect(); let variants: std::collections::BTreeSet<&str> = filtered .iter() .map(|hit| hit.name.as_str()) .filter(|candidate| *candidate != name) .collect(); let mut note = format!( "{total} row(s) over {} dataset(s) carry '{name}'", datasets.len() ); if !variants.is_empty() { note.push_str(&format!( ", including the suffixed variant(s) {}", variants.into_iter().collect::>().join(", ") )); } note.push_str(&format!( "; {matched} remain after the filters, {} on this page", page.rows_len )); let mut envelope = ctx .envelope(serde_json::Value::Array(page.rows)) .with_columns(COLUMNS.iter().map(|name| name.to_string()).collect(), page.rows_len) .with_note(¬e) .with_note( "Metadata only: which datasets carry the probeset and what their scans found. The \ trait's values are per-dataset and authorized per dataset -- follow `next`.", ); if let Some(dataset) = datasets.first() { envelope = envelope.with_next( &format!( "/datasets/{}/probesets/{}/values", path_segment(dataset), path_segment(&name) ), "The probeset's value per strain in the first matching dataset (with mapping \ suggestions attached)", ); } envelope = envelope.with_links(probeset_links(&datasets, &name)); Ok(Json(envelope)) } #[utoipa::path(get, path = "/genes/{symbol}/traits", summary = "Every expression trait of a gene symbol, across datasets", description = "A cross-dataset lookup on `ProbeSet.Symbol` (indexed): one row per dataset \ and probeset, with a real total (unlike /search's capped list) and the panel each trait belongs \ to. Filter with ?group=, ?species=, ?platform= and ?min_lod=; sort with ?sort=lod for the \ strongest peaks first. Metadata only: values are per-dataset.", params(("symbol" = String, Path, description = "Gene symbol, e.g. Shh"), GeneParams), responses( (status = 200, description = "One row per dataset and probeset, positional (meta.columns names them)", body = Envelope), Unauthorized, (status = 404, description = "No probeset carries that symbol", body = ApiError) ) )] #[gnauth(token_only)] pub async fn gene_traits( ctx: Ctx, Path(symbol): Path, Query(params): Query, _token: AuthToken, ) -> Result>, ApiError> { let rid = ctx.request_info(); let lookup = symbol.clone(); let hits = tokio::task::spawn_blocking(move || db::probeset_hits(None, Some(&lookup))) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|error| ApiError::from_db(error, "gene_traits").with_request_id(rid.clone()))?; if hits.is_empty() { return Err(ApiError::new( StatusCode::NOT_FOUND, "gene_not_found", "No probeset for that symbol", format!("No probeset in any dataset carries the symbol '{symbol}'."), "gene_traits", ) .with_request_id(rid)); } let total = hits.len(); let mut filtered = filter(hits, ¶ms.group, ¶ms.species, ¶ms.platform); if let Some(min_lod) = params.min_lod { filtered.retain(|hit| hit.lod.is_some_and(|lod| lod >= min_lod)); } let matched = filtered.len(); let probesets: Vec = filtered .iter() .map(|hit| hit.name.clone()) .collect::>() .into_iter() .collect(); let datasets: Vec = filtered .iter() .map(|hit| hit.dataset.clone()) .collect::>() .into_iter() .collect(); match params.sort.as_deref() { Some("lod") => filtered.sort_by(|left, right| { right .lod .unwrap_or(f64::NEG_INFINITY) .partial_cmp(&left.lod.unwrap_or(f64::NEG_INFINITY)) .unwrap_or(std::cmp::Ordering::Equal) }), Some("dataset") | None => filtered.sort_by(|left, right| { left.dataset .cmp(&right.dataset) .then(left.name.cmp(&right.name)) }), Some(other) => { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_sort", "Invalid sort", format!("sort '{other}' not recognized; use 'dataset' or 'lod'."), "gene_traits", ) .with_request_id(rid)); } } let page = page(&filtered, params.page, params.limit); let mut envelope = ctx .envelope(serde_json::Value::Array(page.rows)) .with_columns(COLUMNS.iter().map(|name| name.to_string()).collect(), page.rows_len) .with_note(&format!( "{} expression trait row(s) across {} dataset(s) and {} probeset(s) carry the symbol \ '{symbol}'{}; {matched} remain after the filters, {} on this page.", total, datasets.len(), probesets.len(), match (params.group.as_deref(), params.species.as_deref(), params.platform.as_deref()) { (None, None, None) => "".to_string(), _ => " (all unfiltered)".to_string(), }, page.rows_len, )) .with_note( "Metadata only: values are per-dataset and authorized per dataset. A peak's `locus` \ is a marker name in that dataset's genotype version; resolving it to a position is \ /groups/{group}/genotypes/{version}/markers/{marker}/location.", ); if let Some(dataset) = datasets.first() { if let Some(name) = probesets.first() { envelope = envelope.with_next( &format!( "/datasets/{}/probesets/{}/values", path_segment(dataset), path_segment(name) ), "One of these traits' values per strain, with the mapping suggestions attached", ); } } envelope = envelope.with_links(probeset_links(&datasets, probesets.first().map(String::as_str).unwrap_or(""))); Ok(Json(envelope)) } /// The rows one page holds. struct Page { rows: Vec, rows_len: usize, } /// Rounded for reading: the database's floats carry more digits than a reply /// needs, and 6.63333333333333 is noise where 6.633 is the number. pub(crate) fn round(value: Option, places: f64) -> Option { let factor = 10f64.powf(places); value.map(|value| (value * factor).round() / factor) } fn page(hits: &[ProbesetHit], page: Option, limit: Option) -> Page { let limit = limit.unwrap_or(100).clamp(1, 5_000) as usize; let page = page.unwrap_or(1).max(1) as usize; let rows: Vec = hits .iter() .skip((page - 1) * limit) .take(limit) .map(|hit| { serde_json::json!([ hit.dataset, hit.group, hit.species, hit.platform, hit.name, hit.symbol, hit.chr, round(hit.mb, 4.0), round(hit.mean, 3.0), round(hit.lrs, 3.0), round(hit.lod, 3.0), hit.locus, round(hit.additive, 4.0), ]) }) .collect(); let rows_len = rows.len(); Page { rows, rows_len } } /// The `group`/`species`/`platform` filters, all optional and all substring /// for `platform` (there is no platform id in this schema). fn filter( hits: Vec, group: &Option, species: &Option, platform: &Option, ) -> Vec { hits.into_iter() .filter(|hit| { group .as_deref() .is_none_or(|wanted| hit.group.as_deref() == Some(wanted)) }) .filter(|hit| { species .as_deref() .is_none_or(|wanted| hit.species.as_deref() == Some(wanted)) }) .filter(|hit| { platform.as_deref().is_none_or(|wanted| { hit.platform .as_deref() .is_some_and(|actual| actual.to_lowercase().contains(&wanted.to_lowercase())) }) }) .collect() } /// Where a cross-dataset answer leads: the first dataset's values, the /// group's genotype surface, and the listings. fn probeset_links(datasets: &[String], name: &str) -> Vec { let mut links = Vec::new(); if !datasets.is_empty() && !name.is_empty() { links.push(( format!("/datasets/{}/probesets", path_segment(&datasets[0])), "The other probesets of the first matching dataset", )); } links.push(( "/groups".to_string(), "Every group, to pick a panel to map against", )); links.extend(suggest::root_links()); links } fn internal(message: String, rid: crate::response::RequestInfo) -> ApiError { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", message, "probeset_lookup", ) .with_request_id(rid) } #[derive(Deserialize, utoipa::IntoParams)] pub struct AliasParams { /// Restrict the lookup to one species: human, mouse or rat. Without it /// the first entity Wikidata finds in those taxa is used. pub species: Option, /// Ask Wikidata again even if the cached answer is inside its seven-day /// TTL. An operator's switch, not a caller's. pub refresh: Option, } #[utoipa::path(get, path = "/genes/{symbol}/aliases", summary = "A gene symbol's aliases from Wikidata, cached for seven days", description = "The `skos:altLabel`s Wikidata records for the gene, in English: the alias \ list a trait page shows next to the symbol. Two SPARQL queries (the entity, then its labels) \ against WIKIDATA_ENDPOINT, kept in an LMDB cache (WIKIDATA_CACHE_DIR) for WIKIDATA_TTL_SECONDS \ (seven days by default). When Wikidata cannot be reached the cached answer is served and marked \ `stale` rather than reported as \"no aliases\" -- an empty list and an unreachable upstream mean \ different things.", params(("symbol" = String, Path, description = "Gene symbol, e.g. Shh"), AliasParams), responses( (status = 200, description = "The aliases, with where and when they came from", body = Envelope), Unauthorized, (status = 502, description = "Wikidata could not be reached and nothing was cached", body = ApiError) ) )] #[gnauth(token_only)] pub async fn gene_aliases( ctx: Ctx, Path(symbol): Path, Query(params): Query, _token: AuthToken, ) -> Result>, ApiError> { let rid = ctx.request_info(); let config = Config::load(); let file = wikidata::cache_file(&config.wikidata_cache_dir) .to_string_lossy() .into_owned(); let endpoint = config.wikidata_endpoint.clone(); let ttl = config.wikidata_ttl; let wanted = symbol.clone(); let species = params.species.clone(); let refresh = params.refresh.unwrap_or(false); let answer = tokio::task::spawn_blocking(move || { wikidata::lookup( std::path::Path::new(&file), &endpoint, ttl, &wanted, species.as_deref(), refresh, ) }) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|error| match error { WikidataError::Unavailable(detail) => ApiError::new( StatusCode::BAD_GATEWAY, "wikidata_unavailable", "Wikidata could not be reached", format!("{detail}; nothing was cached for '{symbol}' to fall back on."), "gene_aliases", ) .with_request_id(rid.clone()), WikidataError::Cache(detail) => ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "wikidata_cache_error", "The alias cache could not be read", detail, "gene_aliases", ) .with_request_id(rid.clone()), })?; let age_hours = (chrono_now() - answer.fetched_at).max(0) as f64 / 3600.0; // Where else to send a reader about this gene, keyed on the symbol and // species that were matched (GTEx only for a human entity: it is human-only). let mut answer = answer.clone(); answer.links = wikidata::link_outs( &answer.symbol, answer.species.as_deref(), answer.label.as_deref(), answer.wikidata_id.as_deref(), ); let mut envelope = ctx .envelope(answer.clone()) .with_note(&format!( "Wikidata's English `skos:altLabel`s for {} {}; the symbol itself is not repeated in \ `aliases`.", answer.symbol, match &answer.wikidata_id { Some(id) => format!("({id})"), None => "(no matching gene entity)".to_string(), } )) .with_note(&format!( "Cached in gene-aliases.lmdb for {} day(s); this answer was fetched {age_hours:.1} h \ ago (add ?refresh=true to ask Wikidata again).", ttl / 86_400 )); if answer.stale { envelope = envelope.partial( "Wikidata could not be refreshed, so this is the last value that was fetched -- \ still the gene's aliases, but not today's.", ); } envelope = envelope.with_next( &format!("/genes/{}/traits", path_segment(&symbol)), "This gene's expression traits across datasets", ); envelope = envelope.with_links(suggest::root_links()); Ok(Json(envelope)) } /// Seconds since the epoch, for a reply's "fetched N hours ago". fn chrono_now() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|elapsed| elapsed.as_secs() as i64) .unwrap_or(0) }