diff options
| author | Alexander_Kabui <alexanderkabui@proton.me> | 2026-10-13 15:11:33 +0300 |
|---|---|---|
| committer | Alexander_Kabui <alexanderkabui@proton.me> | 2026-10-14 07:27:02 +0300 |
| commit | a6003ffbfe40aadaa1d9548bdc3294987b97d304 (patch) | |
| tree | c1bf7ef12eba17ac178090ae48fddb807e23acd7 /src | |
| parent | a188b75157516c5e00dd130d9ca8dfbd6cf36679 (diff) | |
| download | genenetwork4-a6003ffbfe40aadaa1d9548bdc3294987b97d304.tar.gz | |
* src/indexer/run.rs (write path): stream documents, not buffer all.
Building and writing each document as the crawl yields it is leaner than collecting the whole set in a Vec, so live documents stay O(1) and peak memory stops scaling with the corpus. * src/indexer/livemem.rs: the live-document counter that proves it. * src/indexer/build.rs, src/indexer/mod.rs, src/bin/gn-indexer.rs: adapt to the streamed build and write signatures. * Cargo.lock: repin xapian-rs 97fd343 -> 70c5eac (the write-API set).
Diffstat (limited to 'src')
| -rw-r--r-- | src/bin/gn-indexer.rs | 1 | ||||
| -rw-r--r-- | src/indexer/build.rs | 31 | ||||
| -rw-r--r-- | src/indexer/livemem.rs | 58 | ||||
| -rw-r--r-- | src/indexer/mod.rs | 1 | ||||
| -rw-r--r-- | src/indexer/run.rs | 381 |
5 files changed, 368 insertions, 104 deletions
diff --git a/src/bin/gn-indexer.rs b/src/bin/gn-indexer.rs index 3b70945..d81990e 100644 --- a/src/bin/gn-indexer.rs +++ b/src/bin/gn-indexer.rs @@ -81,6 +81,7 @@ fn main() -> ExitCode { println!("deleted={}", stats.deleted); println!("unchanged={}", stats.unchanged); println!("index_doc_count={}", stats.index_doc_count); + println!("max_live_documents={}", stats.max_live_documents); ExitCode::SUCCESS } Err(error) => { diff --git a/src/indexer/build.rs b/src/indexer/build.rs index e72b0a7..13074f8 100644 --- a/src/indexer/build.rs +++ b/src/indexer/build.rs @@ -83,10 +83,33 @@ impl Canon { } } +/// The stable identity term of a gene row, or `None` when name or +/// dataset is absent (the crawl then produces no document). +/// +/// This is the cheap identity pre-pass: it computes exactly the +/// `name:dataset` term `build_gene` would write, without building a +/// document, so identity uniqueness can be validated before any write. +pub fn gene_idterm(row: &GeneRow) -> Option<String> { + let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?); + Some(schema::identity_term(schema::DOCTYPE_GENE, name, dataset)) +} + +/// The stable identity term of a phenotype row, or `None` when name or +/// dataset is absent. The integer name is stringified exactly as the +/// document builder does. +pub fn phenotype_idterm(row: &PhenotypeRow) -> Option<String> { + let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?); + Some(schema::identity_term( + schema::DOCTYPE_PHENOTYPE, + &name.to_string(), + dataset, + )) +} + /// Build a gene document. `None` when name or dataset is absent (the /// indexer then writes no document at all). pub fn build_gene(row: &GeneRow, caches: &RdfCaches) -> Option<BuiltDocument> { - let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?); + let idterm = gene_idterm(row)?; let mut doc = Document::default(); { @@ -184,7 +207,6 @@ pub fn build_gene(row: &GeneRow, caches: &RdfCaches) -> Option<BuiltDocument> { push_opt_f64(&mut data, "geno_mb", &row.geno_mb); doc.set_data(data.to_string()); - let idterm = schema::identity_term(schema::DOCTYPE_GENE, name, dataset); doc.add_boolean_term(schema::type_term(schema::DOCTYPE_GENE)); doc.add_boolean_term(&idterm); @@ -221,10 +243,10 @@ pub fn build_gene(row: &GeneRow, caches: &RdfCaches) -> Option<BuiltDocument> { /// Build a phenotype document. pub fn build_phenotype(row: &PhenotypeRow) -> Option<BuiltDocument> { - let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?); // Python converts the integer name to str for the document (and for // the identity term's identifier). - let name_text = name.to_string(); + let name_text = row.name.as_ref()?.to_string(); + let idterm = phenotype_idterm(row)?; // The raw comma-separated authors string is indexed before storage. let authors_raw = row.authors.clone(); @@ -320,7 +342,6 @@ pub fn build_phenotype(row: &PhenotypeRow) -> Option<BuiltDocument> { push_opt_f64(&mut data, "geno_mb", &row.geno_mb); doc.set_data(data.to_string()); - let idterm = schema::identity_term(schema::DOCTYPE_PHENOTYPE, &name_text, dataset); doc.add_boolean_term(schema::type_term(schema::DOCTYPE_PHENOTYPE)); doc.add_boolean_term(&idterm); diff --git a/src/indexer/livemem.rs b/src/indexer/livemem.rs new file mode 100644 index 0000000..8d898cf --- /dev/null +++ b/src/indexer/livemem.rs @@ -0,0 +1,58 @@ +//! Live-document accounting for the streaming write path. +//! +//! The memory objective of the streaming refactor is that the number +//! of `Document` objects simultaneously alive in the run orchestrator +//! is O(1), not O(crawl size). [`CountedDocument`] wraps one built +//! document; constructing it increments a process-global live counter +//! and records the high-water mark, dropping it decrements. The peak +//! is reported on stdout as `max_live_documents` when +//! `GN4_INDEXER_LIVE_COUNTER=1`. +//! +//! The RED control (`GN4_INDEXER_LIVE_RED=retain`) makes the caller +//! retain every wrapped document instead of releasing it after the +//! write, so the counter must rise to O(docs) and the flatness +//! assertion must fail. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +static LIVE: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +/// A built document whose lifetime is tracked by the counter. +pub struct CountedDocument<T> { + inner: T, +} + +impl<T> CountedDocument<T> { + pub fn new(inner: T) -> Self { + let live = LIVE.fetch_add(1, Ordering::SeqCst) + 1; + PEAK.fetch_max(live, Ordering::SeqCst); + Self { inner } + } + + pub fn get(&self) -> &T { + &self.inner + } +} + +impl<T> Drop for CountedDocument<T> { + fn drop(&mut self) { + LIVE.fetch_sub(1, Ordering::SeqCst); + } +} + +/// Reset the live and peak counters before a run. +pub fn reset() { + LIVE.store(0, Ordering::SeqCst); + PEAK.store(0, Ordering::SeqCst); +} + +/// The high-water mark of simultaneously live documents this run. +pub fn max_live() -> usize { + PEAK.load(Ordering::SeqCst) +} + +/// Whether the run should retain every document (the RED control). +pub fn red_retain() -> bool { + std::env::var("GN4_INDEXER_LIVE_RED").as_deref() == Ok("retain") +} diff --git a/src/indexer/mod.rs b/src/indexer/mod.rs index 7486feb..7972b54 100644 --- a/src/indexer/mod.rs +++ b/src/indexer/mod.rs @@ -22,6 +22,7 @@ pub mod build; pub mod cache; +pub mod livemem; pub mod pyjson; pub mod rows; pub mod run; diff --git a/src/indexer/run.rs b/src/indexer/run.rs index 5d26981..6ca9a17 100644 --- a/src/indexer/run.rs +++ b/src/indexer/run.rs @@ -1,14 +1,24 @@ //! Run orchestration: FULL (bootstrap) and INCREMENTAL (per-document //! in-place update) over one mechanism — build documents from a crawl, //! then write them into the index by identity term. +//! +//! The write path is STREAMING: a cheap identity pre-pass validates +//! `name:dataset` uniqueness before any index is opened, then the crawl +//! is replayed once, building one document at a time and writing it +//! immediately, so the run never holds more than O(1) live `Document` +//! objects. FULL writes into a fresh temp directory and only renames it +//! into place on full success, so an error cannot destroy an existing +//! index; INCREMENTAL opens its write transaction lazily, so a +//! zero-change run writes nothing. use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use xapian_rs::{Database, DbAction, DbBackend, Enquire, Query, WritableDatabase}; -use super::build::{BuiltDocument, build_gene, build_phenotype}; +use super::build::{BuiltDocument, build_gene, build_phenotype, gene_idterm, phenotype_idterm}; use super::cache::RdfCaches; +use super::livemem::{self, CountedDocument}; use super::rows::{self, GeneRow, PhenotypeRow}; use super::schema; use super::sidecar::Sidecar; @@ -23,6 +33,8 @@ pub struct Stats { pub unchanged: usize, pub index_doc_count: u32, pub duplicate_identity_terms: Vec<String>, + /// High-water mark of simultaneously live built documents. + pub max_live_documents: usize, } /// Inputs for one run. @@ -37,60 +49,133 @@ pub struct RunConfig { /// Test-only sabotage hooks for the RED controls (gate 5). Never set in /// a normal run; each value deliberately breaks one guarantee so the -/// gate can be shown to fail. +/// gate can be shown to fail. `duplicate` skips the identity-uniqueness +/// error (last-writer-wins, the regression the pre-pass closes); `hash` +/// perturbs every source hash; `delete` drops the delete pass; +/// `commit` cancels instead of committing. fn red_mode() -> Option<String> { std::env::var("GN4_INDEXER_RED").ok() } -/// Build every document for a crawl, asserting identity-term -/// uniqueness (a duplicate `name:dataset` pair is a reported error, not -/// last-writer-wins). -fn build_all( +/// The sibling temp directory a FULL build writes before its atomic +/// swap into place. +fn temp_index_path(index_path: &Path) -> PathBuf { + let parent = index_path.parent().unwrap_or_else(|| Path::new(".")); + let name = index_path + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| "index".to_string()); + parent.join(format!(".{name}.tmp.{}", std::process::id())) +} + +/// Atomically replace `index_path` with the freshly built `temp` index +/// directory. The pre-existing index is moved aside first and restored +/// if the swap fails, so it is never left destroyed by a failed build. +fn swap_into_place(temp: &Path, index_path: &Path) -> Result<(), String> { + let parent = index_path.parent().unwrap_or_else(|| Path::new(".")); + let backup = parent.join(format!( + ".{}.old.{}", + index_path + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| "index".to_string()), + std::process::id() + )); + let had_existing = index_path.exists(); + if had_existing { + std::fs::rename(index_path, &backup) + .map_err(|e| format!("index swap (move aside): {e}"))?; + } + match std::fs::rename(temp, index_path) { + Ok(()) => { + if had_existing { + let _ = std::fs::remove_dir_all(&backup); + } + Ok(()) + } + Err(e) => { + if had_existing { + let _ = std::fs::rename(&backup, index_path); + } + Err(format!("index swap (rename into place): {e}")) + } + } +} + +/// The cheap identity pre-pass: enumerate every row's identity term +/// WITHOUT building documents, flag duplicate `name:dataset` pairs, and +/// return the full idterm set. Duplicates are an error to report, not +/// last-writer-wins. +fn identity_prepass( genes: &[GeneRow], phenotypes: &[PhenotypeRow], - caches: &RdfCaches, - red_hash: bool, -) -> Result<(Vec<BuiltDocument>, Vec<String>), String> { - let mut built = Vec::with_capacity(genes.len() + phenotypes.len()); +) -> (BTreeSet<String>, Vec<String>) { let mut seen: BTreeMap<String, usize> = BTreeMap::new(); let mut duplicates = Vec::new(); + let mut idterms = BTreeSet::new(); + let mut position = 0usize; for row in genes { - if let Some(mut document) = build_gene(row, caches) { - if let Some(nonce) = red_nonce(red_hash) { - document.source_hash.push_str(&nonce); - } - record(&mut seen, &mut duplicates, &mut built, document); + if let Some(idterm) = gene_idterm(row) { + record_idterm(&mut seen, &mut duplicates, &mut idterms, idterm, position); + position += 1; } } for row in phenotypes { - if let Some(mut document) = build_phenotype(row) { - if let Some(nonce) = red_nonce(red_hash) { - document.source_hash.push_str(&nonce); - } - record(&mut seen, &mut duplicates, &mut built, document); + if let Some(idterm) = phenotype_idterm(row) { + record_idterm(&mut seen, &mut duplicates, &mut idterms, idterm, position); + position += 1; } } - Ok((built, duplicates)) + (idterms, duplicates) } -/// Record one built document, flagging a duplicate identity term -/// instead of silently overwriting. -fn record( +fn record_idterm( seen: &mut BTreeMap<String, usize>, duplicates: &mut Vec<String>, - built: &mut Vec<BuiltDocument>, - document: BuiltDocument, + idterms: &mut BTreeSet<String>, + idterm: String, + position: usize, ) { - if let Some(previous) = seen.get(&document.idterm) { + if let Some(previous) = seen.get(&idterm) { duplicates.push(format!( - "{} (first at position {previous}, duplicate at {})", - document.idterm, - built.len() + "{idterm} (first at position {previous}, duplicate at {position})" )); } else { - seen.insert(document.idterm.clone(), built.len()); + seen.insert(idterm.clone(), position); + idterms.insert(idterm); + } +} + +/// Replay the crawl once, building one document at a time and handing +/// it to `emit`. Documents are produced in the same order (genes then +/// phenotypes) the batch builder used. +fn for_each_document<F>( + genes: &[GeneRow], + phenotypes: &[PhenotypeRow], + caches: &RdfCaches, + red_hash: bool, + mut emit: F, +) -> Result<(), String> +where + F: FnMut(BuiltDocument) -> Result<(), String>, +{ + for row in genes { + if let Some(mut document) = build_gene(row, caches) { + if let Some(nonce) = red_nonce(red_hash) { + document.source_hash.push_str(&nonce); + } + emit(document)?; + } + } + for row in phenotypes { + if let Some(mut document) = build_phenotype(row) { + if let Some(nonce) = red_nonce(red_hash) { + document.source_hash.push_str(&nonce); + } + emit(document)?; + } } - built.push(document); + Ok(()) } fn red_nonce(red_hash: bool) -> Option<String> { @@ -125,58 +210,117 @@ fn index_identity_terms(db: &WritableDatabase) -> BTreeSet<String> { terms } -/// Run a FULL (bootstrap) build: a fresh index written document by -/// document through `replace_document_by_term`. +/// Run a FULL (bootstrap) build: a fresh temp index written document by +/// document through `replace_document_by_term`, swapped into place only +/// on full success. pub fn run_full(config: &RunConfig) -> Result<Stats, String> { let caches = RdfCaches::load(&config.rdf_cache)?; let genes = rows::crawl_genes(&config.sql_uri)?; let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?; let red = red_mode(); - let (built, duplicates) = - build_all(&genes, &phenotypes, &caches, red.as_deref() == Some("hash"))?; - let mut stats = Stats { - mode: "full".to_string(), - built: built.len(), - replaced: built.len(), - duplicate_identity_terms: duplicates, - ..Default::default() - }; - if !stats.duplicate_identity_terms.is_empty() { + // Identity validation happens BEFORE any index is opened or + // written, so a duplicate crawl cannot touch an existing index. + let (_idterms, duplicates) = identity_prepass(&genes, &phenotypes); + if !duplicates.is_empty() && red.as_deref() != Some("duplicate") { return Err(format!( "duplicate identity terms in crawl: {}", - stats.duplicate_identity_terms.join(", ") + duplicates.join(", ") )); } - let mut db = WritableDatabase::open( - &config.index_path, - Some(DbAction::CreateOrOverwrite), - Some(DbBackend::Glass), - None, - None, + livemem::reset(); + let retain = livemem::red_retain(); + let temp = temp_index_path(&config.index_path); + let outcome = stream_full( + &temp, + &genes, + &phenotypes, + &caches, + red.as_deref() == Some("hash"), + retain, ); - db.begin_transaction(None); - for document in &built { - let _ = db.replace_document_by_term(&document.idterm, &document.document); + let (index_doc_count, hashes, built) = match outcome { + Ok(value) => value, + Err(error) => { + let _ = std::fs::remove_dir_all(&temp); + return Err(error); + } + }; + if let Err(error) = swap_into_place(&temp, &config.index_path) { + let _ = std::fs::remove_dir_all(&temp); + return Err(error); } - commit(&mut db, red.as_deref() == Some("commit"))?; - stats.index_doc_count = db.doc_count(); - // Sidecar strictly after the index commit. + // Sidecar strictly after the index commit (and the swap). let sidecar = Sidecar { version: 1, mode: "full".to_string(), - doc_count: built.len(), + doc_count: built, rif_hash: caches.rif.hash.clone(), wiki_hash: caches.wiki.hash.clone(), - documents: built - .iter() - .map(|d| (d.idterm.clone(), d.source_hash.clone())) - .collect(), + documents: hashes, }; sidecar.store(&config.index_path)?; - Ok(stats) + + Ok(Stats { + mode: "full".to_string(), + built, + replaced: built, + index_doc_count, + duplicate_identity_terms: Vec::new(), + max_live_documents: livemem::max_live(), + ..Default::default() + }) +} + +/// Stream a fresh FULL build into `temp`: open once, one transaction, +/// write each document as it is built. Returns the committed document +/// count, the idterm→source-hash map for the sidecar, and the built +/// count. +fn stream_full( + temp: &Path, + genes: &[GeneRow], + phenotypes: &[PhenotypeRow], + caches: &RdfCaches, + red_hash: bool, + retain: bool, +) -> Result<(u32, BTreeMap<String, String>, usize), String> { + let mut db = WritableDatabase::open( + temp, + Some(DbAction::CreateOrOverwrite), + Some(DbBackend::Glass), + None, + None, + ); + db.begin_transaction(None); + + let mut hashes: BTreeMap<String, String> = BTreeMap::new(); + let mut kept: Vec<CountedDocument<BuiltDocument>> = Vec::new(); + let mut count = 0usize; + { + let emit = |document: BuiltDocument| -> Result<(), String> { + let counted = CountedDocument::new(document); + let idterm = counted.get().idterm.clone(); + hashes.insert(idterm.clone(), counted.get().source_hash.clone()); + let _ = db.replace_document_by_term(&idterm, &counted.get().document); + count += 1; + if retain { + kept.push(counted); + } + Ok(()) + }; + if let Err(error) = for_each_document(genes, phenotypes, caches, red_hash, emit) { + db.cancel_transaction(); + return Err(error); + } + } + + commit(&mut db, red_mode().as_deref() == Some("commit"))?; + let doc_count = db.doc_count(); + drop(db); + drop(kept); + Ok((doc_count, hashes, count)) } /// Run an INCREMENTAL pass: crawl, hash, diff against the sidecar, @@ -195,15 +339,19 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { let genes = rows::crawl_genes(&config.sql_uri)?; let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?; let red = red_mode(); - let (built, duplicates) = - build_all(&genes, &phenotypes, &caches, red.as_deref() == Some("hash"))?; - if !duplicates.is_empty() { + + // Identity pre-pass, before any write. + let (idterms, duplicates) = identity_prepass(&genes, &phenotypes); + if !duplicates.is_empty() && red.as_deref() != Some("duplicate") { return Err(format!( "duplicate identity terms in crawl: {}", duplicates.join(", ") )); } + livemem::reset(); + let retain = livemem::red_retain(); + let mut db = WritableDatabase::open( &config.index_path, Some(DbAction::CreateOrOpen), @@ -220,56 +368,93 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { let sidecar_terms: BTreeSet<String> = previous.documents.keys().cloned().collect(); let mismatched = index_terms != sidecar_terms; - let built_by_term: BTreeMap<&str, &BuiltDocument> = - built.iter().map(|d| (d.idterm.as_str(), d)).collect(); - - let mut to_write: Vec<&BuiltDocument> = Vec::new(); + let mut hashes: BTreeMap<String, String> = BTreeMap::new(); let mut unchanged = 0usize; - for document in &built { - let known = if mismatched { - None - } else { - previous.documents.get(&document.idterm) + let mut replaced = 0usize; + let mut open_transaction = false; + let mut kept: Vec<CountedDocument<BuiltDocument>> = Vec::new(); + { + let previous_documents = &previous.documents; + let emit = |document: BuiltDocument| -> Result<(), String> { + let counted = CountedDocument::new(document); + let doc = counted.get(); + hashes.insert(doc.idterm.clone(), doc.source_hash.clone()); + let known = if mismatched { + None + } else { + previous_documents.get(&doc.idterm) + }; + match known { + Some(hash) if *hash == doc.source_hash => unchanged += 1, + _ => { + if !open_transaction { + db.begin_transaction(None); + open_transaction = true; + } + let _ = db.replace_document_by_term(&doc.idterm, &doc.document); + replaced += 1; + } + } + if retain { + kept.push(counted); + } + Ok(()) }; - match known { - Some(hash) if *hash == document.source_hash => unchanged += 1, - _ => to_write.push(document), + if let Err(error) = for_each_document( + &genes, + &phenotypes, + &caches, + red.as_deref() == Some("hash"), + emit, + ) { + if open_transaction { + db.cancel_transaction(); + } + return Err(error); } } // Existence truth: the sidecar's key set (or, on mismatch, the // index's own terms). Anything there but not in the crawl is gone. + // Deletes need the full crawl, so they are applied at the end. let mut to_delete: BTreeSet<String> = if mismatched { - index_terms.clone() + index_terms } else { - sidecar_terms.clone() + sidecar_terms }; if red.as_deref() == Some("delete") { to_delete.clear(); } else { - for term in built_by_term.keys() { - to_delete.remove(*term); + for term in &idterms { + to_delete.remove(term); + } + } + + let mut deleted = 0usize; + if !to_delete.is_empty() { + if !open_transaction { + db.begin_transaction(None); + open_transaction = true; + } + for term in &to_delete { + db.delete_document_by_term(term); + deleted += 1; } } let mut stats = Stats { mode: "incremental".to_string(), - built: built.len(), + built: hashes.len(), unchanged, + replaced, + deleted, + max_live_documents: livemem::max_live(), ..Default::default() }; - let nothing_changed = to_write.is_empty() && to_delete.is_empty(); - if !nothing_changed { - db.begin_transaction(None); - for document in &to_write { - let _ = db.replace_document_by_term(&document.idterm, &document.document); - stats.replaced += 1; - } - for term in &to_delete { - db.delete_document_by_term(term); - stats.deleted += 1; - } + let nothing_changed = replaced == 0 && deleted == 0; + if open_transaction { + // A zero-change run opens no transaction and writes nothing. commit(&mut db, red.as_deref() == Some("commit"))?; } stats.index_doc_count = db.doc_count(); @@ -280,16 +465,14 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { let sidecar = Sidecar { version: 1, mode: "incremental".to_string(), - doc_count: built.len(), + doc_count: hashes.len(), rif_hash: caches.rif.hash.clone(), wiki_hash: caches.wiki.hash.clone(), - documents: built - .iter() - .map(|d| (d.idterm.clone(), d.source_hash.clone())) - .collect(), + documents: hashes, }; sidecar.store(&config.index_path)?; } + let _ = kept; Ok(stats) } |
