//! Gene aliases from Wikidata, cached locally in LMDB. //! //! GeneNetwork2/3 get a gene's aliases from Wikidata by two SPARQL queries: //! first the entity (a gene, `wdt:P31 wd:Q7187`, in human/mouse/rat, labelled //! with the symbol), then its `skos:altLabel`s. gn-guile does the same //! (`gn/db/sources/wikidata.scm`), which is where these queries come from. //! //! Wikidata is queried only when the cache is older than [`DEFAULT_TTL`] //! (seven days) -- it is a shared public service, and an alias list is not //! news -- and the answer is kept in LMDB so a restart, or a Wikidata outage, //! does not cost the caller the data. **A failed refresh keeps the last //! value**: the reply says it is stale rather than pretending the gene has no //! aliases, because an empty list and an unreachable upstream mean different //! things and only one of them is about the gene. //! //! The cache is a single LMDB file, `WIKIDATA_CACHE_DIR/gene-aliases.lmdb` //! (LMDB's no-subdir layout, so it is one file and its `-lock` companion //! rather than a directory of two): key = `":"`, //! value = the [`Stored`] JSON. use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use lmdb::Transaction; use serde::{Deserialize, Serialize}; /// Seven days, in seconds. pub const DEFAULT_TTL: i64 = 7 * 24 * 60 * 60; /// The cache file inside `WIKIDATA_CACHE_DIR`. pub const CACHE_FILE: &str = "gene-aliases.lmdb"; /// The taxa the gene lookup accepts, and the QIDs they are. const TAXA: [(&str, &str); 3] = [ ("human", "Q15978631"), ("mouse", "Q83310"), ("rat", "Q184224"), ]; /// What the endpoint answers with. #[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] pub struct GeneAliases { /// The symbol as asked for. pub symbol: String, /// The Wikidata entity that matched, e.g. "Q24420953". Absent when /// Wikidata knows no gene with this symbol in the accepted taxa. #[serde(skip_serializing_if = "Option::is_none")] pub wikidata_id: Option, /// Which of human/mouse/rat it matched, when the request did not say. #[serde(skip_serializing_if = "Option::is_none")] pub species: Option, /// The entity's own `rdfs:label` in that species -- the canonical symbol, /// which is what a service keyed on the human symbol needs when the caller /// asked for `?species=human` (a mouse `Shh` gives `SHH`). #[serde(skip_serializing_if = "Option::is_none")] pub label: Option, /// `skos:altLabel`s, sorted, the symbol itself removed. pub aliases: Vec, /// When Wikidata last answered, in Unix seconds. pub fetched_at: i64, /// True when Wikidata could not be reached and this is the cached answer. pub stale: bool, /// Where else to send a reader about this gene: the services GeneNetwork2's /// trait page links to, keyed on the symbol and species that were actually /// matched. GTEx appears only for a human entity, because it is human-only /// and a mouse symbol there is an empty report -- ask `?species=human` for /// the ortholog's label when you need that link. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub links: Vec, } /// One external gene page. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] pub struct GeneLink { /// The service, e.g. "RGD". pub name: String, pub href: String, /// What it is for, in one sentence. pub description: String, } /// What is stored in LMDB: the answer without the per-request flags. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Stored { pub symbol: String, pub wikidata_id: Option, pub species: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub label: Option, pub aliases: Vec, pub fetched_at: i64, } /// Why aliases could not be produced at all. #[derive(Debug)] pub enum WikidataError { /// Neither Wikidata nor the cache had anything. Unavailable(String), /// The cache could not be read or written. Cache(String), } impl std::fmt::Display for WikidataError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { WikidataError::Unavailable(what) | WikidataError::Cache(what) => write!(f, "{what}"), } } } fn now() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|elapsed| elapsed.as_secs() as i64) .unwrap_or(0) } /// The cache key: species-scoped when the request named one, so `?species=rat` /// and `?species=mouse` do not overwrite each other. pub fn key(symbol: &str, species: Option<&str>) -> String { format!( "{}:{}", species.unwrap_or("any").to_lowercase(), symbol.to_lowercase() ) } /// Whether a stored answer is still inside the TTL. pub fn is_fresh(stored: &Stored, ttl: i64, now: i64) -> bool { now.saturating_sub(stored.fetched_at) < ttl } /// The gene entity query: a gene in one of the accepted taxa, labelled with /// the symbol. Binds `?species` when the caller named one. pub fn gene_id_query(symbol: &str, species: Option<&str>) -> String { let taxa: Vec<&str> = match species { Some(wanted) => TAXA .iter() .filter(|(name, _)| name.eq_ignore_ascii_case(wanted)) .map(|(_, qid)| *qid) .collect(), None => TAXA.iter().map(|(_, qid)| *qid).collect(), }; let values: Vec = taxa.iter().map(|qid| format!("(wd:{qid})")).collect(); format!( "SELECT DISTINCT ?wikidata_id ?species ?label WHERE {{\n \ ?wikidata_id wdt:P31 wd:Q7187 ;\n wdt:P703 ?species ;\n \ rdfs:label ?label .\n \ VALUES (?species) {{ {} }}\n \ FILTER(LCASE(STR(?label)) = LCASE(\"{}\"))\n \ FILTER(LANG(?label) = \"en\")\n}}", values.join(" "), symbol.replace('"', "") ) } /// The aliases query for one entity. pub fn aliases_query(wikidata_id: &str) -> String { let id = wikidata_id.trim_start_matches("wd:").trim(); format!( "SELECT DISTINCT ?alias WHERE {{\n \ wd:{id} rdfs:label ?name ;\n skos:altLabel ?alias .\n \ FILTER(LANG(?name) = \"en\" && LANG(?alias) = \"en\")\n}}" ) } /// One row of a SPARQL JSON reply, reduced to its `value` strings. fn binding_values(body: &serde_json::Value, field: &str) -> Vec { body.get("results") .and_then(|results| results.get("bindings")) .and_then(|bindings| bindings.as_array()) .map(|rows| { rows.iter() .filter_map(|row| row.get(field)?.get("value")?.as_str()) .map(str::to_string) .collect() }) .unwrap_or_default() } /// The local QID from a Wikidata entity URL. fn qid_of(url: &str) -> String { url.rsplit('/').next().unwrap_or(url).to_string() } /// The species a taxon QID names. fn species_of(qid: &str) -> Option { TAXA .iter() .find(|(_, taxon)| *taxon == qid) .map(|(name, _)| (*name).to_string()) } /// Fetch a gene's aliases from Wikidata: two SPARQL queries. fn fetch( endpoint: &str, symbol: &str, species: Option<&str>, ) -> Result { let ids = sparql(endpoint, &gene_id_query(symbol, species))?; let rows: Vec<(String, Option)> = binding_values(&ids, "wikidata_id") .into_iter() .map(|url| { let qid = qid_of(&url); (qid.clone(), None) }) .collect(); let species_rows = binding_values(&ids, "species"); let label_rows = binding_values(&ids, "label"); let mut entities: Vec<(String, Option)> = rows .into_iter() .enumerate() .map(|(index, (qid, _))| { (qid, species_rows.get(index).and_then(|url| species_of(&qid_of(url)))) }) .collect(); // Deterministic: the species the caller named first, then mouse, rat, // human -- the same gene in three species is three entities, and a reply // that changes which one it picked would be worse than one that picks a // fixed order. let priority = |found: &Option| -> u8 { match found.as_deref() { Some(name) if species.is_some_and(|wanted| name.eq_ignore_ascii_case(wanted)) => 0, Some("mouse") => 1, Some("rat") => 2, Some("human") => 3, _ => 4, } }; entities.sort_by_key(|(qid, found)| (priority(found), qid.clone())); let Some((wikidata_id, found_species)) = entities.into_iter().next() else { return Ok(Stored { symbol: symbol.to_string(), wikidata_id: None, species: species.map(str::to_string), label: None, aliases: Vec::new(), fetched_at: now(), }); }; let label = label_rows.first().cloned(); let alias_body = sparql(endpoint, &aliases_query(&wikidata_id))?; let mut aliases: Vec = binding_values(&alias_body, "alias"); aliases.retain(|alias| !alias.eq_ignore_ascii_case(symbol) && !alias.trim().is_empty()); aliases.sort_by_key(|alias| alias.to_lowercase()); aliases.dedup(); Ok(Stored { symbol: symbol.to_string(), wikidata_id: Some(wikidata_id), species: found_species, label, aliases, fetched_at: now(), }) } /// How long one SPARQL call may take. Wikidata's query service answers in /// one to three seconds usually, but is a shared public service and has been /// measured at nine; two queries in a row need the room, and only the first /// call for a symbol pays it (the answer is cached for a week). const SPARQL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); /// `POST query=…` to a SPARQL endpoint, asking for SPARQL JSON. fn sparql(endpoint: &str, query: &str) -> Result { let agent = ureq::Agent::config_builder() .timeout_global(Some(SPARQL_TIMEOUT)) .build() .new_agent(); let response = agent .post(endpoint) .header("Accept", "application/sparql-results+json") .header("User-Agent", "genenetwork4 (+https://genenetwork.org)") .send_form([("query", query)]) .map_err(|error| { WikidataError::Unavailable(format!( "{endpoint}: {error} (gave up after {}s)", SPARQL_TIMEOUT.as_secs() )) })?; response .into_body() .read_json::() .map_err(|error| WikidataError::Unavailable(format!("{endpoint}: {error}"))) } /// The LMDB environment: one file, `gene-aliases.lmdb`, via LMDB's no-subdir /// layout (which also creates `gene-aliases.lmdb-lock`, as LMDB always does). fn environment(file: &Path) -> Result { if let Some(parent) = file.parent() { std::fs::create_dir_all(parent) .map_err(|error| WikidataError::Cache(format!("{}: {error}", parent.display())))?; } lmdb::Environment::new() .set_max_dbs(2) .set_map_size(16 * 1024 * 1024) .set_flags(lmdb::EnvironmentFlags::NO_SUB_DIR) .open_with_permissions(file, 0o600) .map_err(|error| WikidataError::Cache(format!("{}: {error}", file.display()))) } /// The cached answer for a key, if any. pub fn cached(file: &Path, key: &str) -> Result, WikidataError> { let env = environment(file)?; let db = env .create_db(Some("aliases"), lmdb::DatabaseFlags::empty()) .map_err(|error| WikidataError::Cache(error.to_string()))?; let txn = env .begin_ro_txn() .map_err(|error| WikidataError::Cache(error.to_string()))?; match txn.get(db, &key) { Ok(bytes) => { let stored: Stored = serde_json::from_slice(bytes) .map_err(|error| WikidataError::Cache(format!("{key}: {error}")))?; Ok(Some(stored)) } Err(lmdb::Error::NotFound) => Ok(None), Err(error) => Err(WikidataError::Cache(error.to_string())), } } /// Store an answer. pub fn store(file: &Path, key: &str, stored: &Stored) -> Result<(), WikidataError> { let env = environment(file)?; let db = env .create_db(Some("aliases"), lmdb::DatabaseFlags::empty()) .map_err(|error| WikidataError::Cache(error.to_string()))?; let value = serde_json::to_vec(stored) .map_err(|error| WikidataError::Cache(format!("{key}: {error}")))?; let mut txn = env .begin_rw_txn() .map_err(|error| WikidataError::Cache(error.to_string()))?; txn.put(db, &key, &value, lmdb::WriteFlags::empty()) .map_err(|error| WikidataError::Cache(error.to_string()))?; txn.commit() .map_err(|error| WikidataError::Cache(error.to_string())) } /// Aliases for a symbol: cache when fresh, Wikidata when not, the last value /// when Wikidata fails. pub fn lookup( file: &Path, endpoint: &str, ttl: i64, symbol: &str, species: Option<&str>, refresh: bool, ) -> Result { let key = key(symbol, species); let previous = cached(file, &key).unwrap_or(None); if let Some(previous) = &previous { // An entry written before the label was kept is incomplete: refetch it // rather than serve a reply whose link-outs would use the wrong symbol. if !refresh && previous.label.is_some() && is_fresh(previous, ttl, now()) { return Ok(GeneAliases { symbol: previous.symbol.clone(), wikidata_id: previous.wikidata_id.clone(), species: previous.species.clone(), label: previous.label.clone(), aliases: previous.aliases.clone(), fetched_at: previous.fetched_at, stale: false, links: Vec::new(), }); } } match fetch(endpoint, symbol, species) { Ok(fresh) => { // A cache that cannot be written is not a reason to fail the // request; the answer is good and the next call refetches. let _ = store(file, &key, &fresh); Ok(GeneAliases { symbol: fresh.symbol, wikidata_id: fresh.wikidata_id, species: fresh.species, label: fresh.label, aliases: fresh.aliases, fetched_at: fresh.fetched_at, stale: false, links: Vec::new(), }) } Err(error) => match previous { Some(previous) => Ok(GeneAliases { symbol: previous.symbol, wikidata_id: previous.wikidata_id, species: previous.species, label: previous.label, aliases: previous.aliases, fetched_at: previous.fetched_at, stale: true, links: Vec::new(), }), None => Err(error), }, } } /// The cache file for a configured `WIKIDATA_CACHE_DIR`. pub fn cache_file(configured: &str) -> PathBuf { PathBuf::from(configured).join(CACHE_FILE) } #[cfg(test)] mod tests { use super::*; #[test] fn the_key_is_species_scoped_and_case_insensitive() { assert_eq!(key("Shh", None), "any:shh"); assert_eq!(key("Shh", Some("Rat")), "rat:shh"); assert_ne!(key("Shh", Some("rat")), key("Shh", Some("mouse"))); } #[test] fn freshness_is_the_ttl() { let stored = Stored { symbol: "Shh".to_string(), wikidata_id: Some("Q24420953".to_string()), species: Some("rat".to_string()), label: Some("Shh".to_string()), aliases: vec!["HHG1".to_string()], fetched_at: 1_000_000, }; assert!(is_fresh(&stored, DEFAULT_TTL, 1_000_000 + DEFAULT_TTL - 1)); assert!(!is_fresh(&stored, DEFAULT_TTL, 1_000_000 + DEFAULT_TTL)); // A clock that went backwards does not make the cache stale. assert!(is_fresh(&stored, DEFAULT_TTL, 999_999)); } #[test] fn the_gene_query_binds_the_taxa_and_the_label() { let query = gene_id_query("Shh", None); assert!(query.contains("wdt:P31 wd:Q7187"), "{query}"); assert!(query.contains("wd:Q15978631"), "human: {query}"); assert!(query.contains("wd:Q83310"), "mouse: {query}"); assert!(query.contains("wd:Q184224"), "rat: {query}"); // The label is matched case-insensitively: human genes are labelled // SHH, mouse ones Shh, and both are the same question. assert!(query.contains("LCASE(STR(?label)) = LCASE(\"Shh\")"), "{query}"); assert!(query.contains("LANG(?label) = \"en\""), "{query}"); let mouse_only = gene_id_query("Shh", Some("mouse")); assert!(mouse_only.contains("wd:Q83310"), "{mouse_only}"); assert!(!mouse_only.contains("wd:Q184224"), "{mouse_only}"); // A quote in the symbol cannot break out of the literal. assert!(!gene_id_query("a\"b", None).contains("\"a\"b\")")); } #[test] fn the_aliases_query_accepts_a_bare_or_prefixed_id() { assert!(aliases_query("Q24420953").contains("wd:Q24420953")); assert!(aliases_query("wd:Q24420953").contains("wd:Q24420953")); assert!(!aliases_query("wd:Q24420953").contains("wd:wd:")); } #[test] fn sparql_bindings_are_flattened() { let body = serde_json::json!({ "results": {"bindings": [ {"alias": {"type": "literal", "value": "HHG1"}}, {"alias": {"type": "literal", "value": "TPT"}}, {} ]} }); assert_eq!(binding_values(&body, "alias"), vec!["HHG1", "TPT"]); assert!(binding_values(&body, "nonesuch").is_empty()); assert_eq!(qid_of("http://www.wikidata.org/entity/Q24420953"), "Q24420953"); assert_eq!(species_of("Q83310"), Some("mouse".to_string())); assert_eq!(species_of("Q1"), None); } #[test] fn the_cache_round_trips_through_lmdb() { let dir = std::env::temp_dir().join(format!("gn4-wikidata-test-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); let dir = cache_file(&dir.to_string_lossy()); let stored = Stored { symbol: "Shh".to_string(), wikidata_id: Some("Q24420953".to_string()), species: Some("rat".to_string()), label: Some("Shh".to_string()), aliases: vec!["HHG1".to_string(), "TPT".to_string()], fetched_at: 1_700_000_000, }; store(&dir, &key("Shh", None), &stored).unwrap(); let back = cached(&dir, &key("Shh", None)).unwrap().unwrap(); assert_eq!(back.aliases, stored.aliases); assert_eq!(back.wikidata_id, stored.wikidata_id); assert!(cached(&dir, &key("Nosuch", None)).unwrap().is_none()); assert!(dir.ends_with("gene-aliases.lmdb"), "{}", dir.display()); assert!(dir.is_file(), "one file, as asked: {}", dir.display()); if let Some(parent) = dir.parent() { std::fs::remove_dir_all(parent).ok(); } } /// Wikidata is a public service; a test that cannot reach it proves the /// stale path rather than failing. #[test] fn an_unreachable_endpoint_serves_the_last_value() { let dir = std::env::temp_dir().join(format!("gn4-wikidata-stale-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); let dir = cache_file(&dir.to_string_lossy()); let stored = Stored { symbol: "Shh".to_string(), wikidata_id: Some("Q24420953".to_string()), species: Some("rat".to_string()), label: Some("Shh".to_string()), aliases: vec!["HHG1".to_string()], // Long expired, so the refresh is attempted. fetched_at: 1_600_000_000, }; store(&dir, &key("Shh", None), &stored).unwrap(); let answer = lookup( &dir, "http://127.0.0.1:1/sparql", DEFAULT_TTL, "Shh", None, false, ) .unwrap(); assert!(answer.stale, "an unreachable upstream keeps the last value"); assert_eq!(answer.aliases, vec!["HHG1".to_string()]); // And with nothing cached, the failure is reported. let empty = std::env::temp_dir().join(format!("gn4-wikidata-none-{}", std::process::id())); let _ = std::fs::remove_dir_all(&empty); let empty = cache_file(&empty.to_string_lossy()); assert!(lookup(&empty, "http://127.0.0.1:1/sparql", DEFAULT_TTL, "Nosuch", None, false).is_err()); // The file and LMDB's lock companion; remove the parent directories. for path in [&dir, &empty] { if let Some(parent) = path.parent() { std::fs::remove_dir_all(parent).ok(); } } } } /// Percent-encode a symbol for a URL component. Gene symbols are mostly /// alphanumeric and hyphenated; the exception is worth handling rather than /// pasting into a link unescaped. fn encode(text: &str) -> String { let mut out = String::with_capacity(text.len()); for byte in text.bytes() { if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { out.push(byte as char); } else { out.push_str(&format!("%{byte:02X}")); } } out } /// The taxon id for a species name, for the services that want one. fn taxon(species: Option<&str>) -> Option<&'static str> { match species { Some(name) if name.eq_ignore_ascii_case("mouse") => Some("10090"), Some(name) if name.eq_ignore_ascii_case("rat") => Some("10116"), Some(name) if name.eq_ignore_ascii_case("human") => Some("9606"), _ => None, } } /// The external gene pages for a matched entity: what GeneNetwork2's trait page /// offers, restricted to the ones that answer (see SKILL.md, "Gene link-outs"). /// The symbol used is the entity's own label when there is one -- a mouse `Shh` /// asked for as `?species=human` gives `SHH`, which is what GTEx needs. pub fn link_outs(symbol: &str, species: Option<&str>, label: Option<&str>, qid: Option<&str>) -> Vec { let name = label.unwrap_or(symbol); let encoded = encode(name); let species_name = species.unwrap_or("mouse"); let rgd_species = match species_name.to_lowercase().as_str() { "rat" => "Rat", "human" => "Human", _ => "Mouse", }; let mut links = Vec::new(); let push = |links: &mut Vec, name: &str, href: String, description: &str| { links.push(GeneLink { name: name.to_string(), href, description: description.to_string(), }) }; if let Some(qid) = qid { push( &mut links, "Wikidata", format!("https://www.wikidata.org/wiki/{}", encode(qid)), "Identifiers, aliases and cross-references for the gene entity", ); } push( &mut links, "RGD", format!( "https://rgd.mcw.edu/rgdweb/elasticResults.html?term={encoded}&category=Gene&species={rgd_species}" ), "Orthologs, phenotypes, expression and QTLs (mouse and rat)", ); if species_name.eq_ignore_ascii_case("mouse") { push( &mut links, "MGI", format!("https://www.informatics.jax.org/searchtool/SearchResults?query={encoded}"), "Mouse gene authority: alleles, knockouts, phenotypes", ); push( &mut links, "BioGPS", format!("http://biogps.org/?org=10090#goto=genereport&id={encoded}"), "Mouse expression across tissues and datasets", ); push( &mut links, "Allen Brain Atlas", format!("https://mouse.brain-map.org/search/show?search_type=gene&search_term={encoded}"), "In-situ hybridisation in mouse brain", ); } push( &mut links, "Alliance of Genome Resources", format!("https://www.alliancegenome.org/search?category=gene&q={encoded}"), "Cross-species reference: orthologs and disease models", ); let organism = match species_name.to_lowercase().as_str() { "rat" => "rat", "human" => "human", _ => "mouse", }; push( &mut links, "NCBI Gene", format!("https://www.ncbi.nlm.nih.gov/gene/?term={encoded}%5Bgene%5D%20AND%20{organism}%5Borgn%5D"), "The canonical gene record", ); if let Some(taxon) = taxon(species) { push( &mut links, "UniProt", format!( "https://www.uniprot.org/uniprotkb?query=gene%3A{encoded}%20AND%20organism_id%3A{taxon}" ), "Protein entry, domains and variants", ); push( &mut links, "STRING", format!("https://string-db.org/cgi/network?identifier={encoded}&species={taxon}"), "Protein-protein interaction network", ); } push( &mut links, "GeneMANIA", format!("https://genemania.org/search/{}/{encoded}", species_name.to_lowercase()), "Functional association network", ); if species_name.eq_ignore_ascii_case("human") { push( &mut links, "GTEx", format!("https://www.gtexportal.org/home/gene/{encoded}"), "Human tissue expression (this link needs the human symbol -- ?species=human)", ); push( &mut links, "Open Targets", format!("https://platform.opentargets.org/search?q={encoded}"), "Target-disease associations", ); push( &mut links, "Human Protein Atlas", format!("https://www.proteinatlas.org/search/{encoded}"), "Protein-level expression across human tissues", ); push( &mut links, "GWAS Catalog", format!("https://www.ebi.ac.uk/gwas/search?query={encoded}"), "Published GWAS associations", ); } push( &mut links, "GeneCup", format!("https://genecup.org/progress?type=GWAS&query={encoded}"), "Literature-mined gene-disease links", ); links }