diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-12 19:08:38 +0200 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-10-12 19:08:38 +0200 |
| commit | 8db38664938c4db10573efc0be04b3d12b249f41 (patch) | |
| tree | a587d6ceb1d72485096b175555572822aadddb85 /src | |
| parent | 9cc1631324216d564aca4448cbe71f49dbf2a4af (diff) | |
| parent | ad3bb1e37e5feb22d3a8eb9e99e072f613fa186c (diff) | |
| download | genenetwork4-8db38664938c4db10573efc0be04b3d12b249f41.tar.gz | |
Merge branch 'main' into gemma
Diffstat (limited to 'src')
| -rw-r--r-- | src/bin/gn-indexer.rs | 91 | ||||
| -rw-r--r-- | src/indexer/build.rs | 372 | ||||
| -rw-r--r-- | src/indexer/cache.rs | 241 | ||||
| -rw-r--r-- | src/indexer/mod.rs | 32 | ||||
| -rw-r--r-- | src/indexer/pyjson.rs | 307 | ||||
| -rw-r--r-- | src/indexer/rows.rs | 229 | ||||
| -rw-r--r-- | src/indexer/run.rs | 319 | ||||
| -rw-r--r-- | src/indexer/schema.rs | 96 | ||||
| -rw-r--r-- | src/indexer/sha256.rs | 196 | ||||
| -rw-r--r-- | src/indexer/sidecar.rs | 58 | ||||
| -rw-r--r-- | src/lib.rs | 1 |
11 files changed, 1942 insertions, 0 deletions
diff --git a/src/bin/gn-indexer.rs b/src/bin/gn-indexer.rs new file mode 100644 index 0000000..3b70945 --- /dev/null +++ b/src/bin/gn-indexer.rs @@ -0,0 +1,91 @@ +//! `gn-indexer`: the GeneNetwork4 native search-index builder. +//! +//! Usage: +//! gn-indexer full [--index PATH] [--rdf-cache PATH] [--no-publish-filter] +//! gn-indexer incremental [--index PATH] [--rdf-cache PATH] [--no-publish-filter] +//! +//! The target index comes from `--index`, else `XAPIAN_DB_PATH` (the +//! same key the search engine reads), and is written IN PLACE. The RDF +//! caches come from `--rdf-cache`, else `GN4_RDF_CACHE`, else +//! `/tmp/gn4-rdf-cache.json`. `SQL_URI` comes from the crate's normal +//! layered configuration. +//! +//! Test-only RED controls for the incremental gates are read from +//! `GN4_INDEXER_RED` (values `hash`, `delete`, `commit`); they sabotage +//! one guarantee each and are never set in a normal run. + +use std::path::PathBuf; +use std::process::ExitCode; + +use genenetwork4::config::Config; +use genenetwork4::indexer::{RunConfig, run_full, run_incremental}; + +fn usage() -> ExitCode { + eprintln!( + "usage: gn-indexer <full|incremental> [--index PATH] [--rdf-cache PATH] \ + [--no-publish-filter]" + ); + ExitCode::from(2) +} + +fn main() -> ExitCode { + let mut args = std::env::args().skip(1); + let Some(mode) = args.next() else { + return usage(); + }; + let mut index_arg: Option<PathBuf> = None; + let mut rdf_arg: Option<PathBuf> = None; + let mut publish_filter = true; + while let Some(arg) = args.next() { + match arg.as_str() { + "--index" => index_arg = args.next().map(PathBuf::from), + "--rdf-cache" => rdf_arg = args.next().map(PathBuf::from), + "--no-publish-filter" => publish_filter = false, + other => { + eprintln!("unknown argument: {other}"); + return usage(); + } + } + } + + let config = Config::load(); + let index_path = index_arg.unwrap_or_else(|| PathBuf::from(&config.xapian_db_path)); + if index_path.as_os_str().is_empty() { + eprintln!("no index path: pass --index or set XAPIAN_DB_PATH"); + return ExitCode::from(1); + } + let rdf_cache = rdf_arg.unwrap_or_else(|| { + PathBuf::from( + std::env::var("GN4_RDF_CACHE").unwrap_or_else(|_| "/tmp/gn4-rdf-cache.json".into()), + ) + }); + + let run_config = RunConfig { + sql_uri: config.sql_uri, + index_path, + rdf_cache, + publish_filter, + }; + + let result = match mode.as_str() { + "full" => run_full(&run_config), + "incremental" => run_incremental(&run_config), + _ => return usage(), + }; + + match result { + Ok(stats) => { + println!("mode={}", stats.mode); + println!("built={}", stats.built); + println!("replaced={}", stats.replaced); + println!("deleted={}", stats.deleted); + println!("unchanged={}", stats.unchanged); + println!("index_doc_count={}", stats.index_doc_count); + ExitCode::SUCCESS + } + Err(error) => { + eprintln!("gn-indexer: {error}"); + ExitCode::from(1) + } + } +} diff --git a/src/indexer/build.rs b/src/indexer/build.rs new file mode 100644 index 0000000..e72b0a7 --- /dev/null +++ b/src/indexer/build.rs @@ -0,0 +1,372 @@ +//! Document construction, mirroring `index_genes` / `index_phenotypes` +//! in `~/project/genenetwork3/scripts/index-genenetwork` call for call. +//! +//! The whole point of this module is *not* to improve on the indexer: +//! the TermGenerator configuration (English stemmer, default stemming +//! strategy — never set, `STOP_ALL` with an empty `SimpleStopper`), +//! the order of the `index_text` / `index_text_without_positions` +//! calls, the prefixes, the value slots, the popped-before-storage +//! fields and the identity term are reproduced exactly, because the +//! twin gate compares term sets, wdf, positions, slots and docdata +//! byte for byte. + +use xapian_rs::{Document, SimpleStopper, Stem, StopStrategy, TermGenerator}; + +use super::cache::RdfCaches; +use super::pyjson::{JsonObject, JsonValue}; +use super::rows::{GeneRow, PhenotypeRow}; +use super::schema::{self, prefix, slot}; +use super::sha256::hex_digest; + +/// A document ready to write, its identity term and its source hash. +pub struct BuiltDocument { + pub idterm: String, + pub document: Document, + pub source_hash: String, +} + +/// The writer's TermGenerator, configured exactly as the indexer. +fn writer_termgen() -> TermGenerator { + let mut termgen = TermGenerator::default(); + termgen.set_stemmer(Stem::for_language("en")); + termgen.set_stopper_strategy(StopStrategy::All); + termgen.set_stopper(SimpleStopper::new()); + termgen +} + +/// Canonical encoder for a document's complete source, used for the +/// incremental change hash. Deterministic and delimiter-safe. +#[derive(Default)] +struct Canon { + buffer: Vec<u8>, +} + +impl Canon { + fn field(&mut self, key: &str, value: &str) { + self.buffer.extend_from_slice(key.as_bytes()); + self.buffer.push(b'='); + self.buffer + .extend_from_slice(value.len().to_string().as_bytes()); + self.buffer.push(b':'); + self.buffer.extend_from_slice(value.as_bytes()); + self.buffer.push(b'\n'); + } + + fn opt_str(&mut self, key: &str, value: &Option<String>) { + match value { + Some(text) => self.field(key, text), + None => { + self.buffer.extend_from_slice(key.as_bytes()); + self.buffer.extend_from_slice(b"=N\n"); + } + } + } + + fn opt_f64(&mut self, key: &str, value: &Option<f64>) { + match value { + Some(number) => self.field(key, &format!("{:016x}", number.to_bits())), + None => { + self.buffer.extend_from_slice(key.as_bytes()); + self.buffer.extend_from_slice(b"=N\n"); + } + } + } + + fn opt_i64(&mut self, key: &str, value: &Option<i64>) { + match value { + Some(number) => self.field(key, &number.to_string()), + None => { + self.buffer.extend_from_slice(key.as_bytes()); + self.buffer.extend_from_slice(b"=N\n"); + } + } + } +} + +/// Build a gene document. `None` when name or dataset is absent (the +/// indexer then writes no document at all). +pub fn build_gene(row: &GeneRow, caches: &RdfCaches) -> Option<BuiltDocument> { + let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?); + + let mut doc = Document::default(); + { + let mut termgen = writer_termgen(); + termgen.set_document(&doc); + + // Values (slots 0,1,2,3,4). + if let Some(mean) = row.mean { + doc.set_value(slot::MEAN, mean); + } + if let Some(lrs) = row.lrs { + doc.set_value(slot::PEAK, lrs); + } + if let Some(mb) = row.mb { + doc.set_value(slot::POSITION, mb); + } + if let Some(geno_mb) = row.geno_mb { + doc.set_value(slot::PEAK_MB, geno_mb); + } + if let Some(additive) = row.additive { + doc.set_value(slot::ADDITIVE, additive); + } + + // Free text WITH positions. + for text in [&row.description, &row.tissue, &row.dataset] { + if let Some(text) = text { + termgen.index_text::<&str>(text, None, None); + termgen.increase_termpos(None); + } + } + if let Some(text) = &row.probe_target_description { + termgen.index_text::<&str>(text, None, None); + termgen.increase_termpos(None); + } + // Free text WITHOUT positions, unprefixed. + for text in [&row.name, &row.symbol, &row.species, &row.group] { + if let Some(text) = text { + termgen.index_text_without_positions::<&str>(text, None, None); + } + } + for text in [&row.alias, &row.genbankid, &row.unigeneid] { + if let Some(text) = text { + termgen.index_text_without_positions::<&str>(text, None, None); + } + } + // Prefixed: WITH positions for XI/XDS, without for the rest. + if let Some(text) = &row.species { + termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::SPECIES)); + } + if let Some(text) = &row.group { + termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::GROUP)); + } + if let Some(text) = &row.tissue { + termgen.index_text::<&str>(text, Some(0u32), Some(prefix::TISSUE)); + } + if let Some(text) = &row.dataset { + termgen.index_text::<&str>(text, Some(0u32), Some(prefix::DATASET)); + } + if let Some(text) = &row.symbol { + termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::SYMBOL)); + } + if let Some(text) = &row.chr { + termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::CHR)); + } + if let Some(text) = &row.geno_chr { + termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::PEAK_CHR)); + } + // RDF dictionaries (XRF / XWK), keyed on (species, symbol). + if let (Some(species), Some(symbol)) = (&row.species, &row.symbol) { + if let Some(text) = caches.rif.get(species, symbol) { + termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::RIF)); + } + if let Some(text) = caches.wiki.get(species, symbol) { + termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::WIKI)); + } + } + } + + // Stored data: insertion order minus the popped fields. + let mut data = JsonObject::new(); + push_opt_str(&mut data, "name", &row.name); + push_opt_str(&mut data, "symbol", &row.symbol); + push_opt_str(&mut data, "description", &row.description); + push_opt_str(&mut data, "chr", &row.chr); + push_opt_f64(&mut data, "mb", &row.mb); + push_opt_str(&mut data, "dataset", &row.dataset); + push_opt_str(&mut data, "dataset_fullname", &row.dataset_fullname); + push_opt_str(&mut data, "species", &row.species); + push_opt_str(&mut data, "group", &row.group); + push_opt_str(&mut data, "tissue", &row.tissue); + push_opt_f64(&mut data, "mean", &row.mean); + push_opt_f64(&mut data, "lrs", &row.lrs); + push_opt_f64(&mut data, "additive", &row.additive); + push_opt_str(&mut data, "geno_chr", &row.geno_chr); + push_opt_f64(&mut data, "geno_mb", &row.geno_mb); + doc.set_data(data.to_string()); + + let idterm = schema::identity_term(schema::DOCTYPE_GENE, name, dataset); + doc.add_boolean_term(schema::type_term(schema::DOCTYPE_GENE)); + doc.add_boolean_term(&idterm); + + // Source hash: full row + the monolithic cache hashes. + let mut canon = Canon::default(); + canon.opt_str("name", &row.name); + canon.opt_str("symbol", &row.symbol); + canon.opt_str("description", &row.description); + canon.opt_str("chr", &row.chr); + canon.opt_f64("mb", &row.mb); + canon.opt_str("alias", &row.alias); + canon.opt_str("genbankid", &row.genbankid); + canon.opt_str("unigeneid", &row.unigeneid); + canon.opt_str("probe_target_description", &row.probe_target_description); + canon.opt_str("dataset", &row.dataset); + canon.opt_str("dataset_fullname", &row.dataset_fullname); + canon.opt_str("species", &row.species); + canon.opt_str("group", &row.group); + canon.opt_str("tissue", &row.tissue); + canon.opt_f64("mean", &row.mean); + canon.opt_f64("lrs", &row.lrs); + canon.opt_f64("additive", &row.additive); + canon.opt_str("geno_chr", &row.geno_chr); + canon.opt_f64("geno_mb", &row.geno_mb); + canon.field("rif-cache", &caches.rif.hash); + canon.field("wiki-cache", &caches.wiki.hash); + + Some(BuiltDocument { + idterm, + document: doc, + source_hash: hex_digest(&canon.buffer), + }) +} + +/// Build a phenotype document. +pub fn build_phenotype(row: &PhenotypeRow) -> Option<BuiltDocument> { + let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?); + // Python converts the integer name to str for the document (and for + // the identity term's identifier). + let name_text = name.to_string(); + + // The raw comma-separated authors string is indexed before storage. + let authors_raw = row.authors.clone(); + let authors_list: Option<Vec<String>> = authors_raw + .as_ref() + .map(|text| text.split(',').map(|a| a.trim().to_string()).collect()); + + let mut doc = Document::default(); + { + let mut termgen = writer_termgen(); + termgen.set_document(&doc); + + // Values (slots 0,1,3,4,5,6 — no position slot). + if let Some(mean) = row.mean { + doc.set_value(slot::MEAN, mean); + } + if let Some(lrs) = row.lrs { + doc.set_value(slot::PEAK, lrs); + } + if let Some(geno_mb) = row.geno_mb { + doc.set_value(slot::PEAK_MB, geno_mb); + } + if let Some(additive) = row.additive { + doc.set_value(slot::ADDITIVE, additive); + } + if let Some(year) = row.year { + doc.set_value(slot::YEAR, year as f64); + } + if let Some(nsamples) = row.nsamples { + doc.set_value(slot::NSAMPLES, nsamples as f64); + } + + // Free text WITH positions. + for text in [&row.description, &authors_raw, &row.dataset] { + if let Some(text) = text { + termgen.index_text::<&str>(text, None, None); + termgen.increase_termpos(None); + } + } + for text in [&row.abstract_, &row.title] { + if let Some(text) = text { + termgen.index_text::<&str>(text, None, None); + termgen.increase_termpos(None); + } + } + // Free text WITHOUT positions, unprefixed. + for text in [&row.species, &row.group, &row.inbredsetcode] { + if let Some(text) = text { + termgen.index_text_without_positions::<&str>(text, None, None); + } + } + for text in [&row.abbreviation, &row.lab_code] { + if let Some(text) = text { + termgen.index_text_without_positions::<&str>(text, None, None); + } + } + // Prefixed: WITH positions for A/XDS. + if let Some(text) = &row.species { + termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::SPECIES)); + } + if let Some(text) = &row.group { + termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::GROUP)); + } + if let Some(text) = &authors_raw { + termgen.index_text::<&str>(text, Some(0u32), Some(prefix::AUTHOR)); + } + if let Some(text) = &row.geno_chr { + termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::PEAK_CHR)); + } + if let Some(text) = &row.dataset { + termgen.index_text::<&str>(text, Some(0u32), Some(prefix::DATASET)); + } + } + + let mut data = JsonObject::new(); + push_opt_str(&mut data, "species", &row.species); + push_opt_str(&mut data, "group", &row.group); + push_opt_str(&mut data, "dataset", &row.dataset); + push_opt_str(&mut data, "dataset_fullname", &row.dataset_fullname); + data.push("name", JsonValue::Str(name_text.clone())); + push_opt_str(&mut data, "description", &row.description); + if let Some(authors) = &authors_list { + data.push("authors", JsonValue::StrList(authors.clone())); + } + push_opt_i64(&mut data, "year", &row.year); + push_opt_i64(&mut data, "pubmed_id", &row.pubmed_id); + push_opt_f64(&mut data, "lrs", &row.lrs); + push_opt_f64(&mut data, "additive", &row.additive); + push_opt_str(&mut data, "inbredsetcode", &row.inbredsetcode); + push_opt_f64(&mut data, "mean", &row.mean); + push_opt_i64(&mut data, "nsamples", &row.nsamples); + push_opt_str(&mut data, "geno_chr", &row.geno_chr); + push_opt_f64(&mut data, "geno_mb", &row.geno_mb); + doc.set_data(data.to_string()); + + let idterm = schema::identity_term(schema::DOCTYPE_PHENOTYPE, &name_text, dataset); + doc.add_boolean_term(schema::type_term(schema::DOCTYPE_PHENOTYPE)); + doc.add_boolean_term(&idterm); + + let mut canon = Canon::default(); + canon.opt_str("species", &row.species); + canon.opt_str("group", &row.group); + canon.opt_str("dataset", &row.dataset); + canon.opt_str("dataset_fullname", &row.dataset_fullname); + canon.opt_i64("name", &row.name); + canon.opt_str("abbreviation", &row.abbreviation); + canon.opt_str("description", &row.description); + canon.opt_str("Lab_code", &row.lab_code); + canon.opt_str("Abstract", &row.abstract_); + canon.opt_str("Title", &row.title); + canon.opt_str("authors", &row.authors); + canon.opt_i64("year", &row.year); + canon.opt_i64("pubmed_id", &row.pubmed_id); + canon.opt_f64("lrs", &row.lrs); + canon.opt_f64("additive", &row.additive); + canon.opt_str("inbredsetcode", &row.inbredsetcode); + canon.opt_f64("mean", &row.mean); + canon.opt_i64("nsamples", &row.nsamples); + canon.opt_str("geno_chr", &row.geno_chr); + canon.opt_f64("geno_mb", &row.geno_mb); + + Some(BuiltDocument { + idterm, + document: doc, + source_hash: hex_digest(&canon.buffer), + }) +} + +fn push_opt_str(object: &mut JsonObject, key: &str, value: &Option<String>) { + if let Some(text) = value { + object.push(key, JsonValue::Str(text.clone())); + } +} + +fn push_opt_f64(object: &mut JsonObject, key: &str, value: &Option<f64>) { + if let Some(number) = value { + object.push(key, JsonValue::Float(*number)); + } +} + +fn push_opt_i64(object: &mut JsonObject, key: &str, value: &Option<i64>) { + if let Some(number) = value { + object.push(key, JsonValue::Int(*number)); + } +} diff --git a/src/indexer/cache.rs b/src/indexer/cache.rs new file mode 100644 index 0000000..0145e43 --- /dev/null +++ b/src/indexer/cache.rs @@ -0,0 +1,241 @@ +//! The RDF-derived dictionaries (RIF and wiki), reproducing the +//! indexer's `build_rdf_cache(..., remove_common_words=True)`. +//! +//! The indexer fetches this text over SPARQL; the lane replays it from +//! the authoritative cache file written during the reference build +//! (`~/spikes/index-self-build/rdf-cache.json`, produced by the +//! batched patched script — the cache format is defined by +//! `index-genenetwork-batched`, not by live SPARQL). +//! +//! Each cache is treated as ONE monolithic hashed input: a change to +//! either dictionary re-derives every gene document (the design's +//! "cache as one input", not per-symbol attribution). + +use std::collections::HashMap; +use std::path::Path; + +use serde_json::Value; + +use super::sha256::hex_digest; + +/// Species batching order, verbatim from `index-genenetwork-batched` +/// (`ALL_SPECIES`). Dictionary values are keyed on +/// `(speciesName, symbolName)` and the word counter's first-seen order +/// follows this order. +pub const ALL_SPECIES: [&str; 14] = [ + "arabidopsis", + "barley", + "bat", + "drosophila", + "human", + "medaka", + "monkey", + "mouse", + "poplar", + "rat", + "soybean", + "tomato", + "macaca", + "Nematode", +]; + +/// One derived dictionary plus its monolithic content hash. +pub struct Dictionary { + entries: HashMap<(String, String), String>, + pub hash: String, +} + +impl Dictionary { + /// The dictionary text for `(species, symbol)`, if any. + pub fn get(&self, species: &str, symbol: &str) -> Option<&str> { + self.entries + .get(&(species.to_string(), symbol.to_string())) + .map(String::as_str) + } +} + +/// Both RDF dictionaries (RIF, wiki) plus their content hashes. +pub struct RdfCaches { + pub rif: Dictionary, + pub wiki: Dictionary, +} + +impl RdfCaches { + /// Load and derive both dictionaries from the cache file. + pub fn load(path: &Path) -> Result<RdfCaches, String> { + let text = std::fs::read_to_string(path) + .map_err(|e| format!("cannot read RDF cache {}: {e}", path.display()))?; + let store: Value = serde_json::from_str(&text) + .map_err(|e| format!("RDF cache {} is not valid JSON: {e}", path.display()))?; + Ok(RdfCaches { + rif: derive(&store, "rif")?, + wiki: derive(&store, "wiki")?, + }) + } +} + +/// Indexer word tokenisation: Python `re.compile(r"\w+")` with a +/// `str` subject (Unicode word characters: alphanumerics plus `_`). +/// Callers join the runs with single spaces. +fn tokenize_words(text: &str) -> Vec<&str> { + let mut tokens = Vec::new(); + let mut start: Option<usize> = None; + for (index, ch) in text.char_indices() { + let is_word = ch.is_alphanumeric() || ch == '_'; + match (is_word, start) { + (true, None) => start = Some(index), + (false, Some(begin)) => { + tokens.push(&text[begin..index]); + start = None; + } + _ => {} + } + } + if let Some(begin) = start { + tokens.push(&text[begin..]); + } + tokens +} + +/// Derive one dictionary from the parsed cache store. +fn derive(store: &Value, kind: &str) -> Result<Dictionary, String> { + let mut cache: HashMap<(String, String), String> = HashMap::new(); + // Word counts with first-seen insertion order (Python Counter). + let mut counts: HashMap<String, u64> = HashMap::new(); + let mut order: Vec<String> = Vec::new(); + let kind_store = store + .get(kind) + .and_then(Value::as_object) + .ok_or_else(|| format!("RDF cache has no `{kind}` section"))?; + for species in ALL_SPECIES { + let Some(bindings) = kind_store.get(species).and_then(Value::as_array) else { + continue; + }; + for binding in bindings { + let species_name = binding + .get("speciesName") + .and_then(|v| v.get("value")) + .and_then(Value::as_str) + .ok_or("binding missing speciesName.value")?; + let symbol_name = binding + .get("symbolName") + .and_then(|v| v.get("value")) + .and_then(Value::as_str) + .ok_or("binding missing symbolName.value")?; + let comment = binding + .get("comment") + .and_then(|v| v.get("value")) + .and_then(Value::as_str) + .ok_or("binding missing comment.value")?; + let value = tokenize_words(comment).join(" "); + cache.insert( + (species_name.to_string(), symbol_name.to_string()), + value.clone(), + ); + // Counter.update(value.lower().strip().split()) + for word in value.to_lowercase().split_whitespace() { + match counts.get_mut(word) { + Some(count) => *count += 1, + None => { + counts.insert(word.to_string(), 1); + order.push(word.to_string()); + } + } + } + } + } + + // remove_common_words: top-1000 by count (ties by first-seen order), + // then drop words with len<4 or cnt>3000. + let mut items: Vec<(String, u64)> = order + .iter() + .map(|word| (word.clone(), counts[word])) + .collect(); + // Stable sort by count descending == heapq.nlargest(1000, ...) set + // (verified against CPython on the authoritative cache; see + // logs/cache-nlargest-check.log). + items.sort_by(|a, b| b.1.cmp(&a.1)); + let mut dropped: std::collections::HashSet<&str> = std::collections::HashSet::new(); + for (word, count) in items.iter().take(1000) { + if word.chars().count() < 4 || *count > 3000 { + dropped.insert(word); + } + } + + let mut entries: HashMap<(String, String), String> = HashMap::new(); + for (key, value) in cache { + let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new(); + let mut words: Vec<&str> = Vec::new(); + let lowered = value.to_lowercase(); + for word in lowered.split_whitespace() { + if dropped.contains(word) || !seen.insert(word) { + continue; + } + words.push(word); + } + entries.insert(key, words.join(" ")); + } + + let hash = content_hash(&entries); + Ok(Dictionary { entries, hash }) +} + +/// Monolithic content hash: a canonical, order-independent +/// serialization of the derived dictionary. +fn content_hash(entries: &HashMap<(String, String), String>) -> String { + let mut keys: Vec<&(String, String)> = entries.keys().collect(); + keys.sort(); + let mut buffer: Vec<u8> = Vec::new(); + for key in keys { + buffer.extend_from_slice(key.0.as_bytes()); + buffer.push(0); + buffer.extend_from_slice(key.1.as_bytes()); + buffer.push(0); + buffer.extend_from_slice(entries[key].as_bytes()); + buffer.push(1); + } + hex_digest(&buffer) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn punctuation_strip_joins_word_runs() { + assert_eq!(tokenize_words("hello, world!").join(" "), "hello world"); + assert_eq!( + tokenize_words("beta-1,4-galactosyl").join(" "), + "beta 1 4 galactosyl" + ); + assert_eq!(tokenize_words("a_b c").join(" "), "a_b c"); + } + + #[test] + fn remove_common_words_matches_reference_slice() { + // Mirrors derive-dictionary.py for (mouse, Cd3g): + // rif='identifies cd3gamma infiltrating animals tcr graft cd3cd4'. + let json = std::fs::read_to_string("/home/kabui/spikes/index-self-build/rdf-cache.json"); + let Ok(json) = json else { + eprintln!("skipping: authoritative RDF cache not present"); + return; + }; + let store: Value = serde_json::from_str(&json).unwrap(); + let rif = derive(&store, "rif").unwrap(); + // The dictionary text is fed to `index_text_without_positions` + // (positions off, wdf increment 0), so only its word SET + // reaches the index; Python's `" ".join(set(...))` order is + // irrelevant and deliberately not reproduced. + let mut got: Vec<&str> = rif + .get("mouse", "Cd3g") + .expect("mouse|Cd3g present") + .split_whitespace() + .collect(); + got.sort_unstable(); + let mut want: Vec<&str> = "identifies cd3gamma infiltrating animals tcr graft cd3cd4" + .split_whitespace() + .collect(); + want.sort_unstable(); + assert_eq!(got, want); + } +} diff --git a/src/indexer/mod.rs b/src/indexer/mod.rs new file mode 100644 index 0000000..7486feb --- /dev/null +++ b/src/indexer/mod.rs @@ -0,0 +1,32 @@ +//! The native indexer: crawl, filter, detect, update in place. +//! +//! This module tree is the Rust port of gn3's Python indexer +//! (`~/project/genenetwork3/scripts/index-genenetwork`) described by +//! `~/gn-gemtext-threads/topics/gn4/gn4-search-indexing.gmi`. It shares +//! the crate's configuration (`SQL_URI`, `XAPIAN_DB_PATH`) with the +//! native search engine and writes documents into the live index by +//! their stable identity term, so a per-document update is visible with +//! no service stop. +//! +//! Two modes, one mechanism: +//! +//! * FULL (bootstrap): crawl every public, non-confidential freeze and +//! build a fresh index. A missing or empty sidecar means FULL. +//! * INCREMENTAL: crawl, hash each document's complete source, diff +//! against the sidecar, replace only changed documents, delete +//! vanished identity terms, and write nothing when nothing changed. +//! +//! The phenotype crawl carries `confidentiality < 1 AND public > 0` +//! (the Python query has no WHERE clause, which is the metadata leak +//! this module closes by construction). + +pub mod build; +pub mod cache; +pub mod pyjson; +pub mod rows; +pub mod run; +pub mod schema; +pub mod sha256; +pub mod sidecar; + +pub use run::{RunConfig, Stats, run_full, run_incremental}; diff --git a/src/indexer/pyjson.rs b/src/indexer/pyjson.rs new file mode 100644 index 0000000..01519d1 --- /dev/null +++ b/src/indexer/pyjson.rs @@ -0,0 +1,307 @@ +//! Byte-exact reproduction of Python's `json.dumps` for the indexer's +//! document data. +//! +//! The reference index stores `json.dumps(trait.data)` (see +//! `~/project/genenetwork3/scripts/index-genenetwork`). The twin gate +//! compares document data byte-for-byte, so serde_json's defaults are +//! not usable: Python emits `", "` / `": "` separators, keeps dict +//! insertion order, escapes every non-ASCII character as `\uXXXX` +//! (`ensure_ascii=True`), and formats floats with `repr` (shortest +//! round-trip, fixed notation for decimal exponents in `[-4, 16)` and +//! scientific `d.ddde±XX` outside it). +//! +//! [`JsonObject`] builds one document's data in insertion order from +//! typed values; [`escape_string`] and [`float_repr`] are the two +//! primitives that must match CPython exactly. + +use std::fmt::Write as _; + +/// One value in a document-data object. +pub enum JsonValue { + Str(String), + Int(i64), + Float(f64), + StrList(Vec<String>), +} + +/// Builds a JSON object in insertion order with Python's separators. +#[derive(Default)] +pub struct JsonObject { + entries: Vec<(String, JsonValue)>, +} + +impl JsonObject { + pub fn new() -> Self { + Self::default() + } + + /// Append `key: value`; insertion order is the output order. + pub fn push(&mut self, key: &str, value: JsonValue) { + self.entries.push((key.to_string(), value)); + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Render the object exactly as `json.dumps` would. + pub fn to_string(&self) -> String { + let mut out = String::new(); + out.push('{'); + for (index, (key, value)) in self.entries.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + out.push('"'); + escape_string(key, &mut out); + out.push_str("\": "); + write_value(value, &mut out); + } + out.push('}'); + out + } +} + +fn write_value(value: &JsonValue, out: &mut String) { + match value { + JsonValue::Str(text) => { + out.push('"'); + escape_string(text, out); + out.push('"'); + } + JsonValue::Int(number) => { + let _ = write!(out, "{number}"); + } + JsonValue::Float(number) => out.push_str(&float_repr(*number)), + JsonValue::StrList(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + out.push('"'); + escape_string(item, out); + out.push('"'); + } + out.push(']'); + } + } +} + +/// Escape a string exactly as Python's `json.dumps` with +/// `ensure_ascii=True`: the two-character escapes for the standard +/// control characters, `\u00XX` for other control characters, and +/// `\uXXXX` (a surrogate pair above the BMP) for every non-ASCII +/// character. +pub fn escape_string(text: &str, out: &mut String) { + for ch in text.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{08}' => out.push_str("\\b"), + '\u{0c}' => out.push_str("\\f"), + c if (c as u32) < 0x20 => { + let _ = write!(out, "\\u{:04x}", c as u32); + } + c if (c as u32) < 0x7f => out.push(c), + c => { + // ensure_ascii: BMP code units as \uXXXX, astral as a + // UTF-16 surrogate pair. + let mut buf = [0u16; 2]; + for unit in c.encode_utf16(&mut buf) { + let _ = write!(out, "\\u{:04x}", *unit); + } + } + } + } +} + +/// Format an `f64` exactly as CPython's `repr` (and therefore +/// `json.dumps`). +/// +/// CPython uses the shortest decimal string that round-trips, rounded +/// to nearest with ties to even, then chooses fixed notation when the +/// decimal exponent `e` (value = `d.ddd × 10^e`) satisfies +/// `-4 <= e < 16`, and scientific `d.ddde±XX` otherwise. +/// +/// Rust's `{:e}` is shortest but rounds ties away from zero (e.g. +/// `859.44000244140625` becomes `859.4400024414063`, CPython gives +/// `859.4400024414062`). Instead, search for the smallest significant +/// digit count whose round-half-even rendering round-trips; that is +/// exactly CPython's choice, and the reference docdata has such ties. +pub fn float_repr(value: f64) -> String { + if value == 0.0 { + // Handles +0.0 and -0.0 (repr(-0.0) == "-0.0"). + return if value.is_sign_negative() { + "-0.0".to_string() + } else { + "0.0".to_string() + }; + } + let mut rendered = None; + for significant in 1..=17u32 { + let candidate = format!("{:.*e}", (significant - 1) as usize, value); + if candidate.parse::<f64>() == Ok(value) { + rendered = Some(candidate); + break; + } + } + let rendered = rendered.expect("17 significant digits always round-trip an f64"); + let (mantissa, exponent) = rendered + .split_once('e') + .expect("LowerExp output always contains 'e'"); + let exponent: i32 = exponent.parse().expect("LowerExp exponent is an integer"); + + let (sign, mantissa) = match mantissa.strip_prefix('-') { + Some(rest) => ("-", rest), + None => ("", mantissa), + }; + let digits: String = mantissa.chars().filter(|c| *c != '.').collect(); + + let mut out = String::new(); + out.push_str(sign); + if (-4..16).contains(&exponent) { + if exponent >= 0 { + let integer_digits = exponent as usize + 1; + if digits.len() > integer_digits { + out.push_str(&digits[..integer_digits]); + out.push('.'); + out.push_str(&digits[integer_digits..]); + } else { + out.push_str(&digits); + for _ in digits.len()..integer_digits { + out.push('0'); + } + out.push_str(".0"); + } + } else { + out.push_str("0."); + for _ in 0..(-exponent - 1) { + out.push('0'); + } + out.push_str(&digits); + } + } else { + out.push_str(&digits[..1]); + if digits.len() > 1 { + out.push('.'); + out.push_str(&digits[1..]); + } + out.push('e'); + out.push(if exponent < 0 { '-' } else { '+' }); + let magnitude = exponent.unsigned_abs(); + if magnitude < 10 { + let _ = write!(out, "0{magnitude}"); + } else { + let _ = write!(out, "{magnitude}"); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Values sampled verbatim from the reference index docdata + /// (`logs/probe-docdata.log`, `logs/scan-docdata.log`): the exact + /// byte forms Python produced, including the float edge shapes + /// (integral floats keep `.0`, tiny values go scientific). + #[test] + fn float_repr_matches_reference_bytes() { + let cases: &[(f64, &str)] = &[ + (52.13529418496525, "52.13529418496525"), + (44.970689, "44.970689"), + (8.14033666666667, "8.14033666666667"), + (22.004269639323, "22.004269639323"), + (0.0, "0.0"), + (-0.0, "-0.0"), + (1.0, "1.0"), + (20.0, "20.0"), + (460.0, "460.0"), + (-17.0, "-17.0"), + (2.39444435069444, "2.39444435069444"), + (-3844.23775308921, "-3844.23775308921"), + (6350.554222735148, "6350.554222735148"), + // exponent shapes observed in reference docdata + (6.031019218040234e-08, "6.031019218040234e-08"), + (-2.977410049149484e-09, "-2.977410049149484e-09"), + (-1.4110948100234523e-11, "-1.4110948100234523e-11"), + (-2.8504644121442524e-05, "-2.8504644121442524e-05"), + (3.2393828682277513e-10, "3.2393828682277513e-10"), + // boundary shapes + (1e15, "1000000000000000.0"), + (1e16, "1e+16"), + (1e-4, "0.0001"), + (1e-5, "1e-05"), + (1.5e300, "1.5e+300"), + // observed tie cases where Rust's `{:e}` rounds the wrong + // way; CPython rounds half to even + (859.44000244140625, "859.4400024414062"), + (819.85333251953125, "819.8533325195312"), + (636.80230712890625, "636.8023071289062"), + (13.4032135009765625, "13.403213500976562"), + (3709.29327392578125, "3709.2932739257812"), + (8.1207427978515625, "8.120742797851562"), + ]; + for (value, expected) in cases { + assert_eq!(float_repr(*value), *expected, "for {value:?}"); + } + } + + /// Every distinct float token shape from the reference docdata + /// (all exponent and `.0` forms, plus a deterministic plain + /// sample): parsing each token back to f64 and re-rendering must + /// reproduce the exact token bytes. Fixture generated read-only + /// from the reference index (`scripts/gen-float-fixture.py`, + /// `logs/gen-float-fixture.log`). + #[test] + fn float_repr_matches_reference_fixture() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/pyfloat.txt"); + let Ok(text) = std::fs::read_to_string(path) else { + eprintln!("skipping: float fixture not present at {path}"); + return; + }; + let mut checked = 0usize; + for line in text.lines() { + let token = line.trim(); + if token.is_empty() { + continue; + } + let value: f64 = token.parse().expect("fixture token is a float"); + assert_eq!(float_repr(value), token, "for {value:?}"); + checked += 1; + } + assert!(checked > 1000, "fixture unexpectedly small: {checked}"); + } + + #[test] + fn escape_string_matches_python() { + let mut out = String::new(); + escape_string("a\"b\\c\n\t\r\u{08}\u{0c}\u{01}", &mut out); + assert_eq!(out, "a\\\"b\\\\c\\n\\t\\r\\b\\f\\u0001"); + out.clear(); + escape_string("parré", &mut out); + assert_eq!(out, "parr\\u00e9"); + } + + #[test] + fn object_matches_python_layout() { + let mut object = JsonObject::new(); + object.push("species", JsonValue::Str("mouse".into())); + object.push("year", JsonValue::Int(2001)); + object.push("mean", JsonValue::Float(52.13529418496525)); + object.push( + "authors", + JsonValue::StrList(vec!["Airey DC".into(), "Lu L".into()]), + ); + assert_eq!( + object.to_string(), + "{\"species\": \"mouse\", \"year\": 2001, \"mean\": 52.13529418496525, \ + \"authors\": [\"Airey DC\", \"Lu L\"]}" + ); + } +} diff --git a/src/indexer/rows.rs b/src/indexer/rows.rs new file mode 100644 index 0000000..b8669e1 --- /dev/null +++ b/src/indexer/rows.rs @@ -0,0 +1,229 @@ +//! The crawl: the SQL queries and typed row structs the document +//! builder consumes. +//! +//! The queries and joins are the indexer's own (`genes_query` and +//! `phenotypes_query` in index-genenetwork), rendered by its +//! `serialize_sql`. The one deliberate difference is the phenotype +//! filter: the Python query has NO `WHERE` clause, which is the leak +//! this lane closes by construction — both queries now carry +//! `confidentiality < 1 AND public > 0`. +//! +//! The phenotype join fans out on `InbredSetId` only, so a group's two +//! PublishFreezes (e.g. BXDPublish and GITrMetPublish) each index every +//! trait under their own dataset label. This cross-product is INTENTIONAL +//! reference semantics and is reproduced, not "cleaned up". + +use mysql::prelude::Queryable; +use mysql::{Conn, Opts, OptsBuilder, Row}; + +/// The phenotype filter, fully qualified (the table's `public` column +/// collides with InbredSet.public, so both predicates need the table +/// name). +const PHENOTYPE_PUBLIC_FILTER: &str = + "PublishFreeze.confidentiality < 1 AND PublishFreeze.public > 0"; +/// The gene filter, fully qualified against ProbeSetFreeze. +const GENE_PUBLIC_FILTER: &str = "ProbeSetFreeze.confidentiality < 1 AND ProbeSetFreeze.public > 0"; + +/// One expression (gene) crawl row; each field is absent when SQL +/// returned NULL (the indexer's `MonadicDict` drops NULLs). +#[derive(Default, Clone)] +pub struct GeneRow { + pub name: Option<String>, + pub symbol: Option<String>, + pub description: Option<String>, + pub chr: Option<String>, + pub mb: Option<f64>, + pub alias: Option<String>, + pub genbankid: Option<String>, + pub unigeneid: Option<String>, + pub probe_target_description: Option<String>, + pub dataset: Option<String>, + pub dataset_fullname: Option<String>, + pub species: Option<String>, + pub group: Option<String>, + pub tissue: Option<String>, + pub mean: Option<f64>, + pub lrs: Option<f64>, + pub additive: Option<f64>, + pub geno_chr: Option<String>, + pub geno_mb: Option<f64>, +} + +/// One published (phenotype) crawl row. +#[derive(Default, Clone)] +pub struct PhenotypeRow { + pub species: Option<String>, + pub group: Option<String>, + pub dataset: Option<String>, + pub dataset_fullname: Option<String>, + pub name: Option<i64>, + pub abbreviation: Option<String>, + pub description: Option<String>, + pub lab_code: Option<String>, + pub abstract_: Option<String>, + pub title: Option<String>, + pub authors: Option<String>, + pub year: Option<i64>, + pub pubmed_id: Option<i64>, + pub lrs: Option<f64>, + pub additive: Option<f64>, + pub inbredsetcode: Option<String>, + pub mean: Option<f64>, + pub nsamples: Option<i64>, + pub geno_chr: Option<String>, + pub geno_mb: Option<f64>, +} + +fn connect(sql_uri: &str) -> Result<Conn, String> { + let opts = Opts::from_url(sql_uri).map_err(|e| format!("bad SQL_URI: {e}"))?; + Conn::new(OptsBuilder::from_opts(opts)).map_err(|e| format!("cannot connect: {e}")) +} + +fn text(row: &Row, column: &str) -> Result<Option<String>, String> { + match row.get_opt::<Option<String>, _>(column) { + Some(Ok(value)) => Ok(value), + Some(Err(e)) => Err(format!("column {column}: {e}")), + None => Err(format!("column {column} not in result set")), + } +} + +fn float(row: &Row, column: &str) -> Result<Option<f64>, String> { + match row.get_opt::<Option<f64>, _>(column) { + Some(Ok(value)) => Ok(value), + Some(Err(e)) => Err(format!("column {column}: {e}")), + None => Err(format!("column {column} not in result set")), + } +} + +fn integer(row: &Row, column: &str) -> Result<Option<i64>, String> { + match row.get_opt::<Option<i64>, _>(column) { + Some(Ok(value)) => Ok(value), + Some(Err(e)) => Err(format!("column {column}: {e}")), + None => Err(format!("column {column} not in result set")), + } +} + +fn genes_sql() -> String { + format!( + "SELECT ProbeSet.Name AS name, ProbeSet.Symbol AS symbol, \ + ProbeSet.description AS description, ProbeSet.Chr AS chr, ProbeSet.Mb as mb, \ + ProbeSet.alias AS alias, ProbeSet.GenbankId AS genbankid, \ + ProbeSet.UniGeneId AS unigeneid, \ + ProbeSet.Probe_Target_Description AS probe_target_description, \ + ProbeSetFreeze.Name AS dataset, ProbeSetFreeze.FullName AS dataset_fullname, \ + Species.Name AS species, InbredSet.Name AS `group`, Tissue.Name AS tissue, \ + ProbeSetXRef.Mean AS mean, ProbeSetXRef.LRS AS lrs, \ + ProbeSetXRef.additive AS additive, Geno.Chr AS geno_chr, Geno.Mb as geno_mb \ + FROM Species \ + INNER JOIN InbredSet ON InbredSet.SpeciesId = Species.Id \ + INNER JOIN ProbeFreeze ON ProbeFreeze.InbredSetId = InbredSet.Id \ + INNER JOIN Tissue ON ProbeFreeze.TissueId = Tissue.Id \ + INNER JOIN ProbeSetFreeze ON ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id \ + INNER JOIN ProbeSetXRef ON ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id \ + INNER JOIN ProbeSet ON ProbeSet.Id = ProbeSetXRef.ProbeSetId \ + LEFT JOIN Geno ON ProbeSetXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id \ + WHERE {GENE_PUBLIC_FILTER}" + ) +} + +fn phenotypes_sql(filtered: bool) -> String { + let where_clause = if filtered { + format!(" WHERE {PHENOTYPE_PUBLIC_FILTER}") + } else { + String::new() + }; + format!( + "SELECT Species.Name AS species, InbredSet.Name AS `group`, \ + PublishFreeze.Name AS dataset, PublishFreeze.FullName AS dataset_fullname, \ + PublishXRef.Id AS name, \ + COALESCE(Phenotype.Post_publication_abbreviation, \ + Phenotype.Pre_publication_abbreviation) AS abbreviation, \ + COALESCE(Phenotype.Post_publication_description, \ + Phenotype.Pre_publication_description) AS description, \ + Phenotype.Lab_code, Publication.Abstract, Publication.Title, \ + Publication.Authors AS authors, \ + IF(CONVERT(Publication.Year, UNSIGNED)=0, NULL, \ + CONVERT(Publication.Year, UNSIGNED)) AS year, \ + Publication.PubMed_ID AS pubmed_id, PublishXRef.LRS as lrs, \ + PublishXRef.additive, InbredSet.InbredSetCode AS inbredsetcode, \ + PublishXRef.mean, PublishXRef.NSamples AS nsamples, \ + Geno.Chr as geno_chr, Geno.Mb as geno_mb \ + FROM Species \ + INNER JOIN InbredSet ON InbredSet.SpeciesId = Species.Id \ + INNER JOIN PublishFreeze ON PublishFreeze.InbredSetId = InbredSet.Id \ + INNER JOIN PublishXRef ON PublishXRef.InbredSetId = InbredSet.Id \ + INNER JOIN Phenotype ON PublishXRef.PhenotypeId = Phenotype.Id \ + INNER JOIN Publication ON PublishXRef.PublicationId = Publication.Id \ + LEFT JOIN Geno ON PublishXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id\ + {where_clause}" + ) +} + +/// Crawl every public, non-confidential expression record. +pub fn crawl_genes(sql_uri: &str) -> Result<Vec<GeneRow>, String> { + let mut conn = connect(sql_uri)?; + let rows: Vec<Row> = conn + .query(genes_sql()) + .map_err(|e| format!("gene crawl failed: {e}"))?; + let mut out = Vec::with_capacity(rows.len()); + for row in &rows { + out.push(GeneRow { + name: text(row, "name")?, + symbol: text(row, "symbol")?, + description: text(row, "description")?, + chr: text(row, "chr")?, + mb: float(row, "mb")?, + alias: text(row, "alias")?, + genbankid: text(row, "genbankid")?, + unigeneid: text(row, "unigeneid")?, + probe_target_description: text(row, "probe_target_description")?, + dataset: text(row, "dataset")?, + dataset_fullname: text(row, "dataset_fullname")?, + species: text(row, "species")?, + group: text(row, "group")?, + tissue: text(row, "tissue")?, + mean: float(row, "mean")?, + lrs: float(row, "lrs")?, + additive: float(row, "additive")?, + geno_chr: text(row, "geno_chr")?, + geno_mb: float(row, "geno_mb")?, + }); + } + Ok(out) +} + +/// Crawl published phenotype records. `filtered=false` drops the +/// public/confidentiality predicate: that is the leak control, used +/// only by the gate-4 RED run. +pub fn crawl_phenotypes(sql_uri: &str, filtered: bool) -> Result<Vec<PhenotypeRow>, String> { + let mut conn = connect(sql_uri)?; + let rows: Vec<Row> = conn + .query(phenotypes_sql(filtered)) + .map_err(|e| format!("phenotype crawl failed: {e}"))?; + let mut out = Vec::with_capacity(rows.len()); + for row in &rows { + out.push(PhenotypeRow { + species: text(row, "species")?, + group: text(row, "group")?, + dataset: text(row, "dataset")?, + dataset_fullname: text(row, "dataset_fullname")?, + name: integer(row, "name")?, + abbreviation: text(row, "abbreviation")?, + description: text(row, "description")?, + lab_code: text(row, "Lab_code")?, + abstract_: text(row, "Abstract")?, + title: text(row, "Title")?, + authors: text(row, "authors")?, + year: integer(row, "year")?, + pubmed_id: integer(row, "pubmed_id")?, + lrs: float(row, "lrs")?, + additive: float(row, "additive")?, + inbredsetcode: text(row, "inbredsetcode")?, + mean: float(row, "mean")?, + nsamples: integer(row, "nsamples")?, + geno_chr: text(row, "geno_chr")?, + geno_mb: float(row, "geno_mb")?, + }); + } + Ok(out) +} diff --git a/src/indexer/run.rs b/src/indexer/run.rs new file mode 100644 index 0000000..5d26981 --- /dev/null +++ b/src/indexer/run.rs @@ -0,0 +1,319 @@ +//! Run orchestration: FULL (bootstrap) and INCREMENTAL (per-document +//! in-place update) over one mechanism — build documents from a crawl, +//! then write them into the index by identity term. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use xapian_rs::{Database, DbAction, DbBackend, Enquire, Query, WritableDatabase}; + +use super::build::{BuiltDocument, build_gene, build_phenotype}; +use super::cache::RdfCaches; +use super::rows::{self, GeneRow, PhenotypeRow}; +use super::schema; +use super::sidecar::Sidecar; + +/// What a run did. +#[derive(Debug, Default, Clone)] +pub struct Stats { + pub mode: String, + pub built: usize, + pub replaced: usize, + pub deleted: usize, + pub unchanged: usize, + pub index_doc_count: u32, + pub duplicate_identity_terms: Vec<String>, +} + +/// Inputs for one run. +pub struct RunConfig { + pub sql_uri: String, + pub index_path: PathBuf, + pub rdf_cache: PathBuf, + /// Apply the public/confidentiality filter to phenotypes. `false` + /// is the leak RED control only. + pub publish_filter: bool, +} + +/// Test-only sabotage hooks for the RED controls (gate 5). Never set in +/// a normal run; each value deliberately breaks one guarantee so the +/// gate can be shown to fail. +fn red_mode() -> Option<String> { + std::env::var("GN4_INDEXER_RED").ok() +} + +/// Build every document for a crawl, asserting identity-term +/// uniqueness (a duplicate `name:dataset` pair is a reported error, not +/// last-writer-wins). +fn build_all( + genes: &[GeneRow], + phenotypes: &[PhenotypeRow], + caches: &RdfCaches, + red_hash: bool, +) -> Result<(Vec<BuiltDocument>, Vec<String>), String> { + let mut built = Vec::with_capacity(genes.len() + phenotypes.len()); + let mut seen: BTreeMap<String, usize> = BTreeMap::new(); + let mut duplicates = Vec::new(); + for row in genes { + if let Some(mut document) = build_gene(row, caches) { + if let Some(nonce) = red_nonce(red_hash) { + document.source_hash.push_str(&nonce); + } + record(&mut seen, &mut duplicates, &mut built, document); + } + } + for row in phenotypes { + if let Some(mut document) = build_phenotype(row) { + if let Some(nonce) = red_nonce(red_hash) { + document.source_hash.push_str(&nonce); + } + record(&mut seen, &mut duplicates, &mut built, document); + } + } + Ok((built, duplicates)) +} + +/// Record one built document, flagging a duplicate identity term +/// instead of silently overwriting. +fn record( + seen: &mut BTreeMap<String, usize>, + duplicates: &mut Vec<String>, + built: &mut Vec<BuiltDocument>, + document: BuiltDocument, +) { + if let Some(previous) = seen.get(&document.idterm) { + duplicates.push(format!( + "{} (first at position {previous}, duplicate at {})", + document.idterm, + built.len() + )); + } else { + seen.insert(document.idterm.clone(), built.len()); + } + built.push(document); +} + +fn red_nonce(red_hash: bool) -> Option<String> { + if !red_hash { + return None; + } + // Include the process id so every RUN produces different hashes: + // the point of the RED control is that a re-run over unchanged + // data still rewrites (i.e. the idempotence gate can fail). + static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Some(format!("-red{}-{n}", std::process::id())) +} + +/// Enumerate the identity (`Q`-prefixed) terms the index actually holds. +fn index_identity_terms(db: &WritableDatabase) -> BTreeSet<String> { + let read: Database = Database::from(db); + let mut enquire = Enquire::new(&read); + enquire.set_query(Query::match_all(), None); + let mset = enquire.mset(0, u32::MAX, None, None); + let mut terms = BTreeSet::new(); + for item in mset.matches() { + let document = item.document(); + for term in document.terms() { + let name: &str = term.as_ref(); + if name.starts_with('Q') { + terms.insert(name.to_string()); + break; + } + } + } + terms +} + +/// Run a FULL (bootstrap) build: a fresh index written document by +/// document through `replace_document_by_term`. +pub fn run_full(config: &RunConfig) -> Result<Stats, String> { + let caches = RdfCaches::load(&config.rdf_cache)?; + let genes = rows::crawl_genes(&config.sql_uri)?; + let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?; + let red = red_mode(); + let (built, duplicates) = + build_all(&genes, &phenotypes, &caches, red.as_deref() == Some("hash"))?; + + let mut stats = Stats { + mode: "full".to_string(), + built: built.len(), + replaced: built.len(), + duplicate_identity_terms: duplicates, + ..Default::default() + }; + if !stats.duplicate_identity_terms.is_empty() { + return Err(format!( + "duplicate identity terms in crawl: {}", + stats.duplicate_identity_terms.join(", ") + )); + } + + let mut db = WritableDatabase::open( + &config.index_path, + Some(DbAction::CreateOrOverwrite), + Some(DbBackend::Glass), + None, + None, + ); + db.begin_transaction(None); + for document in &built { + let _ = db.replace_document_by_term(&document.idterm, &document.document); + } + commit(&mut db, red.as_deref() == Some("commit"))?; + stats.index_doc_count = db.doc_count(); + + // Sidecar strictly after the index commit. + let sidecar = Sidecar { + version: 1, + mode: "full".to_string(), + doc_count: built.len(), + rif_hash: caches.rif.hash.clone(), + wiki_hash: caches.wiki.hash.clone(), + documents: built + .iter() + .map(|d| (d.idterm.clone(), d.source_hash.clone())) + .collect(), + }; + sidecar.store(&config.index_path)?; + Ok(stats) +} + +/// Run an INCREMENTAL pass: crawl, hash, diff against the sidecar, +/// replace only changed documents, delete vanished identity terms, and +/// write nothing when nothing changed. +pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { + let Some(previous) = Sidecar::load(&config.index_path) else { + // Missing or unreadable sidecar means FULL. + return run_full(config); + }; + if previous.version != 1 { + return run_full(config); + } + + let caches = RdfCaches::load(&config.rdf_cache)?; + let genes = rows::crawl_genes(&config.sql_uri)?; + let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?; + let red = red_mode(); + let (built, duplicates) = + build_all(&genes, &phenotypes, &caches, red.as_deref() == Some("hash"))?; + if !duplicates.is_empty() { + return Err(format!( + "duplicate identity terms in crawl: {}", + duplicates.join(", ") + )); + } + + let mut db = WritableDatabase::open( + &config.index_path, + Some(DbAction::CreateOrOpen), + Some(DbBackend::Glass), + None, + None, + ); + + // Startup mismatch detection: the sidecar's key set must equal the + // index's actual identity terms, else the sidecar is behind the + // index (or the index was written elsewhere) and is not trusted; do + // a full compare against the index's own terms. + let index_terms = index_identity_terms(&db); + let sidecar_terms: BTreeSet<String> = previous.documents.keys().cloned().collect(); + let mismatched = index_terms != sidecar_terms; + + let built_by_term: BTreeMap<&str, &BuiltDocument> = + built.iter().map(|d| (d.idterm.as_str(), d)).collect(); + + let mut to_write: Vec<&BuiltDocument> = Vec::new(); + let mut unchanged = 0usize; + for document in &built { + let known = if mismatched { + None + } else { + previous.documents.get(&document.idterm) + }; + match known { + Some(hash) if *hash == document.source_hash => unchanged += 1, + _ => to_write.push(document), + } + } + + // Existence truth: the sidecar's key set (or, on mismatch, the + // index's own terms). Anything there but not in the crawl is gone. + let mut to_delete: BTreeSet<String> = if mismatched { + index_terms.clone() + } else { + sidecar_terms.clone() + }; + if red.as_deref() == Some("delete") { + to_delete.clear(); + } else { + for term in built_by_term.keys() { + to_delete.remove(*term); + } + } + + let mut stats = Stats { + mode: "incremental".to_string(), + built: built.len(), + unchanged, + ..Default::default() + }; + + let nothing_changed = to_write.is_empty() && to_delete.is_empty(); + if !nothing_changed { + db.begin_transaction(None); + for document in &to_write { + let _ = db.replace_document_by_term(&document.idterm, &document.document); + stats.replaced += 1; + } + for term in &to_delete { + db.delete_document_by_term(term); + stats.deleted += 1; + } + commit(&mut db, red.as_deref() == Some("commit"))?; + } + stats.index_doc_count = db.doc_count(); + + // Sidecar after the commit; skipped when nothing changed so an + // idempotent run touches neither the index nor the sidecar. + if !nothing_changed { + let sidecar = Sidecar { + version: 1, + mode: "incremental".to_string(), + doc_count: built.len(), + rif_hash: caches.rif.hash.clone(), + wiki_hash: caches.wiki.hash.clone(), + documents: built + .iter() + .map(|d| (d.idterm.clone(), d.source_hash.clone())) + .collect(), + }; + sidecar.store(&config.index_path)?; + } + Ok(stats) +} + +fn commit(db: &mut WritableDatabase, skip: bool) -> Result<(), String> { + if skip { + // RED control: writes are made but never committed, so they are + // invisible to readers. + db.cancel_transaction(); + return Ok(()); + } + db.commit_transaction(); + Ok(()) +} + +/// Where the sidecar lives for an index path. +pub fn sidecar_location(index_path: &Path) -> PathBuf { + super::sidecar::sidecar_path(index_path) +} + +/// The document type of an identity term (for reporting). +pub fn idterm_doctype(idterm: &str) -> &str { + if idterm.starts_with("Qgene:") { + schema::DOCTYPE_GENE + } else { + schema::DOCTYPE_PHENOTYPE + } +} diff --git a/src/indexer/schema.rs b/src/indexer/schema.rs new file mode 100644 index 0000000..5008864 --- /dev/null +++ b/src/indexer/schema.rs @@ -0,0 +1,96 @@ +//! The index schema shared by the writer (this module tree) and the +//! native reader (`crate::search::native`). +//! +//! Writer and reader compile against the same constants, so +//! writer/reader drift is a compile error rather than a search bug: +//! the identity-term shape, the document-type terms, the prefixed +//! field terms and the numeric value slots are defined once here. + +/// Document type for expression (ProbeSet) records; the type term is +/// `XTgene` (`write_document`, index-genenetwork). +pub const DOCTYPE_GENE: &str = "gene"; +/// Document type for published phenotype records; the type term is +/// `XTphenotype`. +pub const DOCTYPE_PHENOTYPE: &str = "phenotype"; + +/// Prefixed term prefixes, exactly the set the indexer emits +/// (`index_species`..`index_peakchr`, `index_from_dictionary`). +pub mod prefix { + /// Authors (indexed WITH positions, wdf increment 0). + pub const AUTHOR: &str = "A"; + /// Species (boolean, no positions). + pub const SPECIES: &str = "XS"; + /// Group (boolean, no positions). + pub const GROUP: &str = "XG"; + /// Tissue (WITH positions, wdf increment 0). + pub const TISSUE: &str = "XI"; + /// Dataset (WITH positions, wdf increment 0). + pub const DATASET: &str = "XDS"; + /// Symbol (boolean, no positions). + pub const SYMBOL: &str = "XY"; + /// Chromosome (boolean, no positions). + pub const CHR: &str = "XC"; + /// Peak chromosome (boolean, no positions). + pub const PEAK_CHR: &str = "XPC"; + /// RIF dictionary text (no positions, wdf increment 0). + pub const RIF: &str = "XRF"; + /// Wiki dictionary text (no positions, wdf increment 0). + pub const WIKI: &str = "XWK"; +} + +/// Numeric value slots (`add_mean`..`add_nsamples`). +pub mod slot { + /// Trait mean (`add_mean`). + pub const MEAN: u32 = 0; + /// Peak / LRS (`add_peak`). + pub const PEAK: u32 = 1; + /// Position in megabases (`add_mb`). + pub const POSITION: u32 = 2; + /// Peak position in megabases (`add_peakmb`). + pub const PEAK_MB: u32 = 3; + /// Additive effect (`add_additive`). + pub const ADDITIVE: u32 = 4; + /// Publication year (`add_year`). + pub const YEAR: u32 = 5; + /// Number of samples (`add_nsamples`). + pub const NSAMPLES: u32 = 6; +} + +/// The document-type term `XT<doctype>` written by `write_document`. +pub fn type_term(doctype: &str) -> String { + format!("XT{doctype}") +} + +/// The stable identity term `Q<doctype>:<name>:<dataset>` (the whole +/// `name:dataset` identifier lowercased), matching `write_document` in +/// the Python indexer. +pub fn identity_term(doctype: &str, name: &str, dataset: &str) -> String { + format!( + "Q{}:{}", + doctype, + format!("{name}:{dataset}").to_lowercase() + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn identity_term_is_lowercased_with_dataset() { + assert_eq!( + identity_term(DOCTYPE_PHENOTYPE, "10583", "BXDPublish"), + "Qphenotype:10583:bxdpublish" + ); + assert_eq!( + identity_term(DOCTYPE_GENE, "100001_at", "HC_U_0304_R"), + "Qgene:100001_at:hc_u_0304_r" + ); + } + + #[test] + fn type_term_matches_reference() { + assert_eq!(type_term(DOCTYPE_GENE), "XTgene"); + assert_eq!(type_term(DOCTYPE_PHENOTYPE), "XTphenotype"); + } +} diff --git a/src/indexer/sha256.rs b/src/indexer/sha256.rs new file mode 100644 index 0000000..0556491 --- /dev/null +++ b/src/indexer/sha256.rs @@ -0,0 +1,196 @@ +//! A small, dependency-free SHA-256. +//! +//! The sidecar and cache hashes need a stable cryptographic digest and +//! the crate must not gain a new dependency (the pinned lockfile stays +//! untouched apart from the documented scratch `[patch]`). Standard +//! FIPS 180-4; unit-tested against the published vectors. + +const K: [u32; 64] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +]; + +const H0: [u32; 8] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, +]; + +/// Streaming SHA-256. +pub struct Sha256 { + state: [u32; 8], + buffer: [u8; 64], + buffer_len: usize, + total_len: u64, +} + +impl Default for Sha256 { + fn default() -> Self { + Self::new() + } +} + +impl Sha256 { + pub fn new() -> Self { + Sha256 { + state: H0, + buffer: [0u8; 64], + buffer_len: 0, + total_len: 0, + } + } + + pub fn update(&mut self, mut data: &[u8]) { + self.total_len = self.total_len.wrapping_add(data.len() as u64); + if self.buffer_len > 0 { + let take = (64 - self.buffer_len).min(data.len()); + self.buffer[self.buffer_len..self.buffer_len + take].copy_from_slice(&data[..take]); + self.buffer_len += take; + data = &data[take..]; + if self.buffer_len == 64 { + let block = self.buffer; + self.compress(&block); + self.buffer_len = 0; + } + } + while data.len() >= 64 { + let mut block = [0u8; 64]; + block.copy_from_slice(&data[..64]); + self.compress(&block); + data = &data[64..]; + } + if !data.is_empty() { + self.buffer[..data.len()].copy_from_slice(data); + self.buffer_len = data.len(); + } + } + + pub fn finalize(mut self) -> [u8; 32] { + let bit_len = self.total_len.wrapping_mul(8); + self.update(&[0x80]); + while self.buffer_len != 56 { + self.update(&[0x00]); + } + // update() advances total_len, but the length field uses the + // pre-padding bit length captured above. + self.total_len = self.total_len; // no-op, kept for clarity + self.update(&bit_len.to_be_bytes()); + let mut out = [0u8; 32]; + for (index, word) in self.state.iter().enumerate() { + out[index * 4..index * 4 + 4].copy_from_slice(&word.to_be_bytes()); + } + out + } + + fn compress(&mut self, block: &[u8; 64]) { + let mut w = [0u32; 64]; + for i in 0..16 { + w[i] = u32::from_be_bytes([ + block[i * 4], + block[i * 4 + 1], + block[i * 4 + 2], + block[i * 4 + 3], + ]); + } + for i in 16..64 { + let s0 = w[i - 15].rotate_right(7) ^ w[i - 15].rotate_right(18) ^ (w[i - 15] >> 3); + let s1 = w[i - 2].rotate_right(17) ^ w[i - 2].rotate_right(19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + .wrapping_add(s0) + .wrapping_add(w[i - 7]) + .wrapping_add(s1); + } + let mut a = self.state[0]; + let mut b = self.state[1]; + let mut c = self.state[2]; + let mut d = self.state[3]; + let mut e = self.state[4]; + let mut f = self.state[5]; + let mut g = self.state[6]; + let mut h = self.state[7]; + for i in 0..64 { + let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); + let ch = (e & f) ^ (!e & g); + let temp1 = h + .wrapping_add(s1) + .wrapping_add(ch) + .wrapping_add(K[i]) + .wrapping_add(w[i]); + let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); + let maj = (a & b) ^ (a & c) ^ (b & c); + let temp2 = s0.wrapping_add(maj); + h = g; + g = f; + f = e; + e = d.wrapping_add(temp1); + d = c; + c = b; + b = a; + a = temp1.wrapping_add(temp2); + } + self.state[0] = self.state[0].wrapping_add(a); + self.state[1] = self.state[1].wrapping_add(b); + self.state[2] = self.state[2].wrapping_add(c); + self.state[3] = self.state[3].wrapping_add(d); + self.state[4] = self.state[4].wrapping_add(e); + self.state[5] = self.state[5].wrapping_add(f); + self.state[6] = self.state[6].wrapping_add(g); + self.state[7] = self.state[7].wrapping_add(h); + } +} + +/// Hex digest of `data`. +pub fn hex_digest(data: &[u8]) -> String { + let digest = { + let mut hasher = Sha256::new(); + hasher.update(data); + hasher.finalize() + }; + let mut out = String::with_capacity(64); + for byte in digest { + use std::fmt::Write as _; + let _ = write!(out, "{byte:02x}"); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn matches_published_vectors() { + assert_eq!( + hex_digest(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + hex_digest(b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + assert_eq!( + hex_digest(b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"), + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + ); + } + + #[test] + fn streaming_matches_one_shot() { + let data: Vec<u8> = (0..1000u32).map(|n| (n % 251) as u8).collect(); + let one = hex_digest(&data); + let mut hasher = Sha256::new(); + for chunk in data.chunks(7) { + hasher.update(chunk); + } + let mut out = String::new(); + for byte in hasher.finalize() { + use std::fmt::Write as _; + let _ = write!(out, "{byte:02x}"); + } + assert_eq!(one, out); + } +} diff --git a/src/indexer/sidecar.rs b/src/indexer/sidecar.rs new file mode 100644 index 0000000..8dcc5a3 --- /dev/null +++ b/src/indexer/sidecar.rs @@ -0,0 +1,58 @@ +//! The incremental sidecar: per-document source hashes stored next to +//! the index (not inside it), written atomically and strictly after the +//! index commit. +//! +//! The sidecar's key set is the existence truth for the incremental +//! path: a document absent from a crawl is deleted by its identity term +//! even if the index still holds it. Write order is index commit first, +//! sidecar rename after, so a crash between the two leaves a sidecar +//! that is behind the index — the startup key-set check detects exactly +//! that and falls back to a full compare. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// The on-disk sidecar. +#[derive(Serialize, Deserialize, Default, PartialEq)] +pub struct Sidecar { + /// Format version; a mismatch triggers a full rebuild. + pub version: u32, + /// How the sidecar was last written (`full` or `incremental`). + pub mode: String, + /// Document count of the index the sidecar describes. + pub doc_count: usize, + /// Monolithic RIF cache hash at the time of writing. + pub rif_hash: String, + /// Monolithic wiki cache hash at the time of writing. + pub wiki_hash: String, + /// Identity term -> source hash. + pub documents: BTreeMap<String, String>, +} + +/// The sidecar path for an index directory (`<index>.sidecar.json`). +pub fn sidecar_path(index_path: &Path) -> PathBuf { + PathBuf::from(format!( + "{}.sidecar.json", + index_path.to_string_lossy().trim_end_matches('/') + )) +} + +impl Sidecar { + pub fn load(index_path: &Path) -> Option<Sidecar> { + let text = std::fs::read_to_string(sidecar_path(index_path)).ok()?; + serde_json::from_str(&text).ok() + } + + /// Atomically replace the sidecar: write a sibling temp file, sync, + /// then rename over the target. + pub fn store(&self, index_path: &Path) -> Result<(), String> { + let path = sidecar_path(index_path); + let temp = PathBuf::from(format!("{}.tmp.{}", path.display(), std::process::id())); + let json = serde_json::to_string(self).map_err(|e| format!("sidecar encode: {e}"))?; + std::fs::write(&temp, json).map_err(|e| format!("sidecar write: {e}"))?; + std::fs::rename(&temp, &path).map_err(|e| format!("sidecar rename: {e}"))?; + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 4d90785..50db2b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ pub mod config; mod db; pub mod embedded; mod error; +pub mod indexer; mod gemma; pub mod response; mod routes; |
