diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/bin/gn-indexer.rs | 15 | ||||
| -rw-r--r-- | src/indexer/rows.rs | 284 | ||||
| -rw-r--r-- | src/indexer/run.rs | 1103 | ||||
| -rw-r--r-- | src/indexer/sidecar.rs | 446 |
4 files changed, 1477 insertions, 371 deletions
diff --git a/src/bin/gn-indexer.rs b/src/bin/gn-indexer.rs index d81990e..224edb8 100644 --- a/src/bin/gn-indexer.rs +++ b/src/bin/gn-indexer.rs @@ -82,6 +82,21 @@ fn main() -> ExitCode { println!("unchanged={}", stats.unchanged); println!("index_doc_count={}", stats.index_doc_count); println!("max_live_documents={}", stats.max_live_documents); + println!("shards_written={}", stats.shards_written); + println!("shards_deleted={}", stats.shards_deleted); + println!( + "max_resident_sidecar_entries={}", + stats.max_resident_sidecar_entries + ); + println!( + "max_resident_dataset_terms={}", + stats.max_resident_dataset_terms + ); + println!("peak_open_shards={}", stats.peak_open_shards); + println!("shard_bytes_read={}", stats.shard_bytes_read); + println!("commit_chunks={}", stats.commit_chunks); + println!("chunk_rss_peak_kib={}", stats.chunk_rss_peak_kib); + println!("startup={}", stats.startup); ExitCode::SUCCESS } Err(error) => { diff --git a/src/indexer/rows.rs b/src/indexer/rows.rs index b8669e1..69b94cc 100644 --- a/src/indexer/rows.rs +++ b/src/indexer/rows.rs @@ -1,5 +1,5 @@ //! The crawl: the SQL queries and typed row structs the document -//! builder consumes. +//! builder consumes, streamed and grouped by dataset. //! //! The queries and joins are the indexer's own (`genes_query` and //! `phenotypes_query` in index-genenetwork), rendered by its @@ -12,6 +12,18 @@ //! PublishFreezes (e.g. BXDPublish and GITrMetPublish) each index every //! trait under their own dataset label. This cross-product is INTENTIONAL //! reference semantics and is reproduced, not "cleaned up". +//! +//! # Streaming, dataset-grouped crawl +//! +//! [`stream_dataset_groups`] never materialises the crawl. Each query is +//! executed in its own producer thread with `ORDER BY dataset`; the merge +//! consumes the two sorted streams and hands the caller **one dataset at +//! a time** through a [`DatasetSink`], whose `begin_dataset` / `row` / +//! `end_dataset` callbacks see rows one at a time. The only per-dataset +//! state the caller keeps is bounded by the largest dataset, not by the +//! whole crawl. + +use std::sync::mpsc::{Receiver, Sender}; use mysql::prelude::Queryable; use mysql::{Conn, Opts, OptsBuilder, Row}; @@ -74,6 +86,34 @@ pub struct PhenotypeRow { pub geno_mb: Option<f64>, } +/// One crawled row, tagged with its document type. The dataset label is +/// the row's dataset column — never parsed back out of the identity +/// term, whose `name` part may itself contain a colon. +pub enum CrawlRow { + Gene(GeneRow), + Phenotype(PhenotypeRow), +} + +impl CrawlRow { + /// The row's dataset label (the shard key). + pub fn dataset(&self) -> Option<&str> { + match self { + CrawlRow::Gene(row) => row.dataset.as_deref(), + CrawlRow::Phenotype(row) => row.dataset.as_deref(), + } + } +} + +/// Receive one dataset group at a time from [`stream_dataset_groups`]. +pub trait DatasetSink { + /// Called once when a dataset group starts, before any of its rows. + fn begin_dataset(&mut self, label: &str) -> Result<(), String>; + /// Called for each row of the current dataset, in arrival order. + fn row(&mut self, row: CrawlRow) -> Result<(), String>; + /// Called once after the last row of the dataset. + fn end_dataset(&mut self, label: &str) -> Result<(), String>; +} + fn connect(sql_uri: &str) -> Result<Conn, String> { let opts = Opts::from_url(sql_uri).map_err(|e| format!("bad SQL_URI: {e}"))?; Conn::new(OptsBuilder::from_opts(opts)).map_err(|e| format!("cannot connect: {e}")) @@ -122,7 +162,8 @@ fn genes_sql() -> String { INNER JOIN ProbeSetXRef ON ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id \ INNER JOIN ProbeSet ON ProbeSet.Id = ProbeSetXRef.ProbeSetId \ LEFT JOIN Geno ON ProbeSetXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id \ - WHERE {GENE_PUBLIC_FILTER}" + WHERE {GENE_PUBLIC_FILTER} \ + ORDER BY dataset" ) } @@ -155,75 +196,194 @@ fn phenotypes_sql(filtered: bool) -> String { INNER JOIN Phenotype ON PublishXRef.PhenotypeId = Phenotype.Id \ INNER JOIN Publication ON PublishXRef.PublicationId = Publication.Id \ LEFT JOIN Geno ON PublishXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id\ - {where_clause}" + {where_clause} \ + ORDER BY dataset" ) } -/// Crawl every public, non-confidential expression record. -pub fn crawl_genes(sql_uri: &str) -> Result<Vec<GeneRow>, String> { +type Item = Result<(String, CrawlRow), String>; + +/// Producer thread for the gene crawl: stream rows ordered by dataset and +/// send each as `(dataset, CrawlRow)`. Rows with no dataset produce no +/// document and are skipped. +fn produce_genes(sql_uri: &str, tx: &Sender<Item>) -> Result<(), String> { let mut conn = connect(sql_uri)?; - let rows: Vec<Row> = conn - .query(genes_sql()) + let result = conn + .query_iter(genes_sql()) .map_err(|e| format!("gene crawl failed: {e}"))?; - let mut out = Vec::with_capacity(rows.len()); - for row in &rows { - out.push(GeneRow { - name: text(row, "name")?, - symbol: text(row, "symbol")?, - description: text(row, "description")?, - chr: text(row, "chr")?, - mb: float(row, "mb")?, - alias: text(row, "alias")?, - genbankid: text(row, "genbankid")?, - unigeneid: text(row, "unigeneid")?, - probe_target_description: text(row, "probe_target_description")?, - dataset: text(row, "dataset")?, - dataset_fullname: text(row, "dataset_fullname")?, - species: text(row, "species")?, - group: text(row, "group")?, - tissue: text(row, "tissue")?, - mean: float(row, "mean")?, - lrs: float(row, "lrs")?, - additive: float(row, "additive")?, - geno_chr: text(row, "geno_chr")?, - geno_mb: float(row, "geno_mb")?, - }); + for row in result { + let row = row.map_err(|e| format!("gene crawl failed: {e}"))?; + let Some(dataset) = text(&row, "dataset")? else { + continue; + }; + let row = GeneRow { + name: text(&row, "name")?, + symbol: text(&row, "symbol")?, + description: text(&row, "description")?, + chr: text(&row, "chr")?, + mb: float(&row, "mb")?, + alias: text(&row, "alias")?, + genbankid: text(&row, "genbankid")?, + unigeneid: text(&row, "unigeneid")?, + probe_target_description: text(&row, "probe_target_description")?, + dataset: Some(dataset.clone()), + dataset_fullname: text(&row, "dataset_fullname")?, + species: text(&row, "species")?, + group: text(&row, "group")?, + tissue: text(&row, "tissue")?, + mean: float(&row, "mean")?, + lrs: float(&row, "lrs")?, + additive: float(&row, "additive")?, + geno_chr: text(&row, "geno_chr")?, + geno_mb: float(&row, "geno_mb")?, + }; + if tx.send(Ok((dataset, CrawlRow::Gene(row)))).is_err() { + return Ok(()); + } } - Ok(out) + Ok(()) } -/// Crawl published phenotype records. `filtered=false` drops the -/// public/confidentiality predicate: that is the leak control, used -/// only by the gate-4 RED run. -pub fn crawl_phenotypes(sql_uri: &str, filtered: bool) -> Result<Vec<PhenotypeRow>, String> { +/// Producer thread for the phenotype crawl. +fn produce_phenotypes(sql_uri: &str, filtered: bool, tx: &Sender<Item>) -> Result<(), String> { let mut conn = connect(sql_uri)?; - let rows: Vec<Row> = conn - .query(phenotypes_sql(filtered)) + let result = conn + .query_iter(phenotypes_sql(filtered)) .map_err(|e| format!("phenotype crawl failed: {e}"))?; - let mut out = Vec::with_capacity(rows.len()); - for row in &rows { - out.push(PhenotypeRow { - species: text(row, "species")?, - group: text(row, "group")?, - dataset: text(row, "dataset")?, - dataset_fullname: text(row, "dataset_fullname")?, - name: integer(row, "name")?, - abbreviation: text(row, "abbreviation")?, - description: text(row, "description")?, - lab_code: text(row, "Lab_code")?, - abstract_: text(row, "Abstract")?, - title: text(row, "Title")?, - authors: text(row, "authors")?, - year: integer(row, "year")?, - pubmed_id: integer(row, "pubmed_id")?, - lrs: float(row, "lrs")?, - additive: float(row, "additive")?, - inbredsetcode: text(row, "inbredsetcode")?, - mean: float(row, "mean")?, - nsamples: integer(row, "nsamples")?, - geno_chr: text(row, "geno_chr")?, - geno_mb: float(row, "geno_mb")?, - }); + for row in result { + let row = row.map_err(|e| format!("phenotype crawl failed: {e}"))?; + let Some(dataset) = text(&row, "dataset")? else { + continue; + }; + let row = PhenotypeRow { + species: text(&row, "species")?, + group: text(&row, "group")?, + dataset: Some(dataset.clone()), + dataset_fullname: text(&row, "dataset_fullname")?, + name: integer(&row, "name")?, + abbreviation: text(&row, "abbreviation")?, + description: text(&row, "description")?, + lab_code: text(&row, "Lab_code")?, + abstract_: text(&row, "Abstract")?, + title: text(&row, "Title")?, + authors: text(&row, "authors")?, + year: integer(&row, "year")?, + pubmed_id: integer(&row, "pubmed_id")?, + lrs: float(&row, "lrs")?, + additive: float(&row, "additive")?, + inbredsetcode: text(&row, "inbredsetcode")?, + mean: float(&row, "mean")?, + nsamples: integer(&row, "nsamples")?, + geno_chr: text(&row, "geno_chr")?, + geno_mb: float(&row, "geno_mb")?, + }; + if tx.send(Ok((dataset, CrawlRow::Phenotype(row)))).is_err() { + return Ok(()); + } + } + Ok(()) +} + +fn spawn_producer<F>(tx: Sender<Item>, body: F) -> std::thread::JoinHandle<()> +where + F: FnOnce(&Sender<Item>) -> Result<(), String> + Send + 'static, +{ + std::thread::spawn(move || { + if let Err(error) = body(&tx) { + let _ = tx.send(Err(error)); + } + }) +} + +/// The dataset label at the head of a channel, propagating a producer +/// error. `Ok(None)` means the channel is exhausted. +fn head_label(head: &Option<Item>) -> Result<Option<&str>, String> { + match head { + None => Ok(None), + Some(Ok((label, _))) => Ok(Some(label.as_str())), + Some(Err(error)) => Err(error.clone()), + } +} + +/// Stream the crawl as dataset groups. Two producer threads (genes and +/// phenotypes) each return rows ordered by dataset; this merge consumes +/// them in dataset order and invokes `sink` once per dataset. No whole- +/// crawl buffer exists: at most one pending row per source is held. +pub fn stream_dataset_groups( + sql_uri: &str, + filtered: bool, + sink: &mut dyn DatasetSink, +) -> Result<(), String> { + // Unbounded channels: the producers stream the two SQL result sets in + // their own threads and the merge consumes them; this is the fast + // path (a bounded channel was tried and made the scaled run several + // times slower, so it was reverted). + let (gene_tx, gene_rx) = std::sync::mpsc::channel::<Item>(); + let (phen_tx, phen_rx) = std::sync::mpsc::channel::<Item>(); + let gene_uri = sql_uri.to_string(); + let phen_uri = sql_uri.to_string(); + let genes = spawn_producer(gene_tx, move |tx| produce_genes(&gene_uri, tx)); + let phenotypes = spawn_producer(phen_tx, move |tx| { + produce_phenotypes(&phen_uri, filtered, tx) + }); + + let outcome = merge_streams(&gene_rx, &phen_rx, sink); + + // Producers end when the receivers drop, but join for a clean exit. + let _ = genes.join(); + let _ = phenotypes.join(); + outcome +} + +fn merge_streams( + gene_rx: &Receiver<Item>, + phen_rx: &Receiver<Item>, + sink: &mut dyn DatasetSink, +) -> Result<(), String> { + let mut gene_head = gene_rx.recv().ok(); + let mut phen_head = phen_rx.recv().ok(); + loop { + let gene_label = head_label(&gene_head)?; + let phen_label = head_label(&phen_head)?; + let label = match (gene_label, phen_label) { + (None, None) => break, + (Some(label), None) => label.to_string(), + (None, Some(label)) => label.to_string(), + (Some(gene), Some(phen)) => { + if gene <= phen { + gene.to_string() + } else { + phen.to_string() + } + } + }; + sink.begin_dataset(&label)?; + drain_label(&mut gene_head, gene_rx, &label, sink)?; + drain_label(&mut phen_head, phen_rx, &label, sink)?; + sink.end_dataset(&label)?; + } + Ok(()) +} + +/// Consume every head row whose dataset equals `label`, advancing the +/// channel. Rows for later datasets stay as the new head. +fn drain_label( + head: &mut Option<Item>, + rx: &Receiver<Item>, + label: &str, + sink: &mut dyn DatasetSink, +) -> Result<(), String> { + loop { + match head { + Some(Ok((row_label, _))) if row_label == label => { + let Some(Ok((_, row))) = head.take() else { + unreachable!("head was checked Some(Ok)") + }; + sink.row(row)?; + *head = rx.recv().ok(); + } + Some(Err(error)) => return Err(error.clone()), + _ => return Ok(()), + } } - Ok(out) } diff --git a/src/indexer/run.rs b/src/indexer/run.rs index 6ca9a17..b3248f5 100644 --- a/src/indexer/run.rs +++ b/src/indexer/run.rs @@ -1,17 +1,32 @@ -//! Run orchestration: FULL (bootstrap) and INCREMENTAL (per-document -//! in-place update) over one mechanism — build documents from a crawl, -//! then write them into the index by identity term. +//! Run orchestration: FULL (bootstrap) and INCREMENTAL (per-dataset +//! in-place update) over one mechanism — stream the crawl one dataset at +//! a time, diff each dataset against its shard, and write changed +//! documents by identity term. //! -//! The write path is STREAMING: a cheap identity pre-pass validates -//! `name:dataset` uniqueness before any index is opened, then the crawl -//! is replayed once, building one document at a time and writing it -//! immediately, so the run never holds more than O(1) live `Document` -//! objects. FULL writes into a fresh temp directory and only renames it -//! into place on full success, so an error cannot destroy an existing -//! index; INCREMENTAL opens its write transaction lazily, so a -//! zero-change run writes nothing. - -use std::collections::{BTreeMap, BTreeSet}; +//! Design of this step (Step 2): +//! +//! * **Streaming, dataset-grouped crawl.** [`rows::stream_dataset_groups`] +//! runs the two SQL crawls in producer threads ordered by dataset and +//! hands the run one dataset at a time; no whole-crawl buffer exists. +//! * **Sharded sidecar.** Each dataset has one shard (see [`sidecar`]); +//! the diff loads only the current dataset's shard, so the resident +//! idterm->hash map is O(largest shard), not O(documents). +//! * **Cheap-first startup.** The steady-state check compares the +//! manifest `doc_count` to `db.doc_count()` (O(1)) and recomputes every +//! shard's `content_sha256` from its file; only on mismatch is the +//! expensive identity-term enumeration performed (and the run falls +//! back to FULL). `index_identity_terms` is therefore off the +//! steady-state path. +//! * **Bounded commits.** Writes are committed every [`COMMIT_CHUNK_DOCS`] +//! documents with a per-chunk RSS probe; FULL writes a temp index and +//! swaps it in only on success. +//! +//! Duplicate-identity detection is a separate streaming pass that keeps +//! only the current dataset's identity set (O(largest dataset)); it runs +//! to completion before any index is opened, so a duplicate crawl can +//! never touch an existing index or sidecar. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use xapian_rs::{Database, DbAction, DbBackend, Enquire, Query, WritableDatabase}; @@ -19,9 +34,15 @@ use xapian_rs::{Database, DbAction, DbBackend, Enquire, Query, WritableDatabase} use super::build::{BuiltDocument, build_gene, build_phenotype, gene_idterm, phenotype_idterm}; use super::cache::RdfCaches; use super::livemem::{self, CountedDocument}; -use super::rows::{self, GeneRow, PhenotypeRow}; +use super::rows::{self, CrawlRow, DatasetSink}; use super::schema; -use super::sidecar::Sidecar; +use super::sidecar::{self, Manifest, ResidentShard, ShardEntry}; + +/// Documents written between commits. Stated once; the incremental and +/// FULL writers both commit every `COMMIT_CHUNK_DOCS` (replaced+deleted) +/// documents so a long run never accumulates one unbounded Xapian +/// transaction. +pub const COMMIT_CHUNK_DOCS: usize = 5000; /// What a run did. #[derive(Debug, Default, Clone)] @@ -35,6 +56,24 @@ pub struct Stats { pub duplicate_identity_terms: Vec<String>, /// High-water mark of simultaneously live built documents. pub max_live_documents: usize, + /// Shards written (staged + renamed) this run. + pub shards_written: usize, + /// Shards removed this run (dataset no longer present). + pub shards_deleted: usize, + /// High-water mark of simultaneously resident shard-map entries. + pub max_resident_sidecar_entries: usize, + /// High-water mark of the current dataset's identity-term set. + pub max_resident_dataset_terms: usize, + /// High-water mark of simultaneously open shards (load-all RED rises). + pub peak_open_shards: usize, + /// Cumulative shard-file bytes read. + pub shard_bytes_read: usize, + /// Chunk commits performed. + pub commit_chunks: usize, + /// Peak `VmRSS` (KiB) observed by the per-chunk probe. + pub chunk_rss_peak_kib: u64, + /// How the run classified startup (`incremental` or a FULL reason). + pub startup: String, } /// Inputs for one run. @@ -47,16 +86,27 @@ pub struct RunConfig { pub publish_filter: bool, } -/// Test-only sabotage hooks for the RED controls (gate 5). Never set in -/// a normal run; each value deliberately breaks one guarantee so the -/// gate can be shown to fail. `duplicate` skips the identity-uniqueness -/// error (last-writer-wins, the regression the pre-pass closes); `hash` -/// perturbs every source hash; `delete` drops the delete pass; -/// `commit` cancels instead of committing. +/// Test-only sabotage hooks for the RED controls. Never set in a normal +/// run; each value deliberately breaks one guarantee so the gate can be +/// shown to fail. `duplicate` skips the identity-uniqueness error +/// (last-writer-wins); `hash` perturbs every source hash; `delete` drops +/// every delete; `commit` cancels instead of committing; `no-recovery` +/// trusts the sidecar and skips the crash-recovery work entirely. fn red_mode() -> Option<String> { std::env::var("GN4_INDEXER_RED").ok() } +/// RED: skip the startup shard-integrity check and swallow a corrupt or +/// unreadable shard as empty (the silent-skip regression the check +/// closes). +fn red_skip_shard_check() -> bool { + red_mode().as_deref() == Some("skip-shard-check") +} + +fn index_exists(index_path: &Path) -> bool { + index_path.is_dir() +} + /// The sibling temp directory a FULL build writes before its atomic /// swap into place. fn temp_index_path(index_path: &Path) -> PathBuf { @@ -102,95 +152,39 @@ fn swap_into_place(temp: &Path, index_path: &Path) -> Result<(), String> { } } -/// The cheap identity pre-pass: enumerate every row's identity term -/// WITHOUT building documents, flag duplicate `name:dataset` pairs, and -/// return the full idterm set. Duplicates are an error to report, not -/// last-writer-wins. -fn identity_prepass( - genes: &[GeneRow], - phenotypes: &[PhenotypeRow], -) -> (BTreeSet<String>, Vec<String>) { - let mut seen: BTreeMap<String, usize> = BTreeMap::new(); - let mut duplicates = Vec::new(); - let mut idterms = BTreeSet::new(); - let mut position = 0usize; - for row in genes { - if let Some(idterm) = gene_idterm(row) { - record_idterm(&mut seen, &mut duplicates, &mut idterms, idterm, position); - position += 1; - } - } - for row in phenotypes { - if let Some(idterm) = phenotype_idterm(row) { - record_idterm(&mut seen, &mut duplicates, &mut idterms, idterm, position); - position += 1; - } +/// The identity term of a crawl row, or `None` when it produces no +/// document. Uses the same helpers the document builder uses, so the +/// pre-pass and the builder cannot drift. +fn row_idterm(row: &CrawlRow) -> Option<String> { + match row { + CrawlRow::Gene(gene) => gene_idterm(gene), + CrawlRow::Phenotype(phenotype) => phenotype_idterm(phenotype), } - (idterms, duplicates) } -fn record_idterm( - seen: &mut BTreeMap<String, usize>, - duplicates: &mut Vec<String>, - idterms: &mut BTreeSet<String>, - idterm: String, - position: usize, -) { - if let Some(previous) = seen.get(&idterm) { - duplicates.push(format!( - "{idterm} (first at position {previous}, duplicate at {position})" - )); - } else { - seen.insert(idterm.clone(), position); - idterms.insert(idterm); +/// Build the document for a crawl row. +fn build_row(row: CrawlRow, caches: &RdfCaches) -> Option<BuiltDocument> { + match row { + CrawlRow::Gene(gene) => build_gene(&gene, caches), + CrawlRow::Phenotype(phenotype) => build_phenotype(&phenotype), } } -/// Replay the crawl once, building one document at a time and handing -/// it to `emit`. Documents are produced in the same order (genes then -/// phenotypes) the batch builder used. -fn for_each_document<F>( - genes: &[GeneRow], - phenotypes: &[PhenotypeRow], - caches: &RdfCaches, - red_hash: bool, - mut emit: F, -) -> Result<(), String> -where - F: FnMut(BuiltDocument) -> Result<(), String>, -{ - for row in genes { - if let Some(mut document) = build_gene(row, caches) { - if let Some(nonce) = red_nonce(red_hash) { - document.source_hash.push_str(&nonce); - } - emit(document)?; - } - } - for row in phenotypes { - if let Some(mut document) = build_phenotype(row) { - if let Some(nonce) = red_nonce(red_hash) { - document.source_hash.push_str(&nonce); - } - emit(document)?; - } - } - Ok(()) -} - fn red_nonce(red_hash: bool) -> Option<String> { if !red_hash { return None; } - // Include the process id so every RUN produces different hashes: - // the point of the RED control is that a re-run over unchanged - // data still rewrites (i.e. the idempotence gate can fail). + // Include a per-run counter so every RUN produces different hashes: + // the point of the RED control is that a re-run over unchanged data + // still rewrites (i.e. the idempotence gate can fail). static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst); Some(format!("-red{}-{n}", std::process::id())) } /// Enumerate the identity (`Q`-prefixed) terms the index actually holds. +/// This is the expensive O(docs) enumeration; it is only reached on the +/// startup-mismatch path, never in the steady state. fn index_identity_terms(db: &WritableDatabase) -> BTreeSet<String> { let read: Database = Database::from(db); let mut enquire = Enquire::new(&read); @@ -210,40 +204,327 @@ fn index_identity_terms(db: &WritableDatabase) -> BTreeSet<String> { terms } -/// Run a FULL (bootstrap) build: a fresh temp index written document by -/// document through `replace_document_by_term`, swapped into place only -/// on full success. +// --------------------------------------------------------------------------- +// Duplicate-identity pre-pass (one streaming pass, O(largest dataset)). +// --------------------------------------------------------------------------- + +struct DuplicateSink { + set: BTreeSet<String>, + duplicates: Vec<String>, + max_terms: usize, +} + +impl DuplicateSink { + fn new() -> Self { + DuplicateSink { + set: BTreeSet::new(), + duplicates: Vec::new(), + max_terms: 0, + } + } +} + +impl DatasetSink for DuplicateSink { + fn begin_dataset(&mut self, _label: &str) -> Result<(), String> { + self.set.clear(); + Ok(()) + } + + fn row(&mut self, row: CrawlRow) -> Result<(), String> { + if let Some(idterm) = row_idterm(&row) { + if !self.set.insert(idterm.clone()) { + self.duplicates.push(idterm); + } + self.max_terms = self.max_terms.max(self.set.len()); + } + Ok(()) + } + + fn end_dataset(&mut self, _label: &str) -> Result<(), String> { + Ok(()) + } +} + +/// Run the duplicate-identity pre-pass. Returns the duplicate terms +/// (empty when the crawl is unique). The per-dataset set is bounded by +/// the largest dataset, and the pass finishes before any index write. +fn duplicate_prepass(sql_uri: &str, filtered: bool) -> Result<(Vec<String>, usize), String> { + let mut sink = DuplicateSink::new(); + rows::stream_dataset_groups(sql_uri, filtered, &mut sink)?; + Ok((sink.duplicates, sink.max_terms)) +} + +// --------------------------------------------------------------------------- +// Chunked-commit pacer with a per-chunk memory probe. +// --------------------------------------------------------------------------- + +/// Reads this process's `VmRSS` (KiB) from `/proc/self/status`. +fn read_vm_rss_kib() -> u64 { + let text = std::fs::read_to_string("/proc/self/status").unwrap_or_default(); + for line in text.lines() { + if line.starts_with("VmRSS:") { + if let Some(value) = line.split_whitespace().nth(1) { + return value.parse().unwrap_or(0); + } + } + } + 0 +} + +struct CommitPacer { + chunk: usize, + open: bool, + writes_since: usize, + chunks: usize, + rss_peak_kib: u64, + /// Optional file each chunk's probe is appended to (evidence). + log_path: Option<PathBuf>, + /// Park after this many chunk commits (crash-gate injection point). + halt_after: Option<usize>, + /// RED: cancel instead of committing. + red_commit: bool, +} + +impl CommitPacer { + fn new(red_commit: bool) -> Self { + let log_path = std::env::var("GN4_INDEXER_COMMIT_LOG") + .ok() + .map(PathBuf::from); + let halt_after = std::env::var("GN4_INDEXER_HALT_AFTER_COMMITS") + .ok() + .and_then(|value| value.parse().ok()); + // The production chunk size is COMMIT_CHUNK_DOCS; the crash gate + // overrides it downward so a small mix spans several chunks. + let chunk = std::env::var("GN4_INDEXER_COMMIT_CHUNK") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(COMMIT_CHUNK_DOCS); + CommitPacer { + chunk, + open: false, + writes_since: 0, + chunks: 0, + rss_peak_kib: 0, + log_path, + halt_after, + red_commit, + } + } + + /// Record one write, opening a transaction lazily and committing when + /// the chunk fills. + fn write(&mut self, db: &mut WritableDatabase) -> Result<(), String> { + if !self.open { + db.begin_transaction(None); + self.open = true; + } + self.writes_since += 1; + if self.writes_since >= self.chunk { + self.flush(db)?; + } + Ok(()) + } + + /// Commit the open chunk (or cancel it under the RED), probe memory, + /// and optionally park for the crash gate. + fn flush(&mut self, db: &mut WritableDatabase) -> Result<(), String> { + if self.red_commit { + db.cancel_transaction(); + } else { + db.commit_transaction(); + } + self.open = false; + self.writes_since = 0; + self.chunks += 1; + let rss = read_vm_rss_kib(); + self.rss_peak_kib = self.rss_peak_kib.max(rss); + if let Some(path) = &self.log_path { + use std::io::Write; + if let Ok(mut file) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + { + let _ = writeln!(file, "chunk={} rss_kib={}", self.chunks, rss); + } + } + if let Some(limit) = self.halt_after { + if self.chunks == limit { + // Park forever: the crash gate kills -9 at this point, so + // the process dies with the chunk committed and no + // destructors, flushes or sidecar writes having run. + eprintln!( + "gn-indexer: halted after {} chunk commits (kill -9 target)", + self.chunks + ); + loop { + std::thread::sleep(std::time::Duration::from_secs(3600)); + } + } + } + Ok(()) + } + + /// Commit the final partial chunk. + fn finish(&mut self, db: &mut WritableDatabase) -> Result<(), String> { + if self.open { + self.flush(db)?; + } + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Sidecar finalize (temp shards first, manifest last). +// --------------------------------------------------------------------------- + +struct SidecarBuild { + index_path: PathBuf, + shard_dir: PathBuf, + stage_dir: PathBuf, + manifest: Manifest, + staged: Vec<(PathBuf, ShardEntry)>, +} + +impl SidecarBuild { + fn new(index_path: &Path, mode: &str, rif_hash: String, wiki_hash: String) -> Self { + let stage_dir = PathBuf::from(format!( + "{}.tmp.{}", + sidecar::shard_dir(index_path).display(), + std::process::id() + )); + SidecarBuild { + index_path: index_path.to_path_buf(), + shard_dir: sidecar::shard_dir(index_path), + stage_dir, + manifest: Manifest { + version: sidecar::SIDECAR_VERSION, + mode: mode.to_string(), + doc_count: 0, + rif_hash, + wiki_hash, + shards: BTreeMap::new(), + }, + staged: Vec::new(), + } + } + + /// Stage a shard's new content (temp + fsync) and record its manifest + /// entry. The rename happens in [`finalize`]. + fn stage(&mut self, label: &str, map: &BTreeMap<String, String>) -> Result<(), String> { + let (temp, entry) = sidecar::stage_shard(&self.stage_dir, label, map)?; + self.manifest + .shards + .insert(label.to_string(), entry.clone()); + self.staged.push((temp, entry)); + Ok(()) + } + + /// Keep an unchanged shard's existing (recomputed) entry. + fn keep(&mut self, label: &str, entry: ShardEntry) { + self.manifest.shards.insert(label.to_string(), entry); + } + + /// Drop a dataset whose rows are all gone. + fn remove(&mut self, label: &str) { + self.manifest.shards.remove(label); + } + + fn shard_count(&self) -> usize { + self.manifest.shards.len() + } + + /// Sum of the (kept + staged) shard counts: the total document count + /// the finalized manifest will describe. + fn total_count(&self) -> usize { + self.manifest.shards.values().map(|e| e.count).sum() + } + + /// Rename every staged shard into place, fsync the shard directory, + /// prune shard files the manifest no longer references, and write the + /// **manifest last**. + fn finalize(mut self) -> Result<(), String> { + self.manifest.doc_count = self.manifest.shards.values().map(|e| e.count).sum(); + std::fs::create_dir_all(&self.shard_dir) + .map_err(|e| format!("create shard dir {}: {e}", self.shard_dir.display()))?; + for (temp, entry) in &self.staged { + let target = self.shard_dir.join(&entry.file); + std::fs::rename(temp, &target) + .map_err(|e| format!("rename shard {}: {e}", target.display()))?; + } + sidecar::fsync_dir(&self.shard_dir)?; + + let keep: BTreeSet<String> = self + .manifest + .shards + .values() + .map(|e| e.file.clone()) + .collect(); + if let Ok(entries) = std::fs::read_dir(&self.shard_dir) { + for entry in entries.flatten() { + let name = entry.file_name().to_string_lossy().to_string(); + if name.starts_with('.') { + continue; + } + if !keep.contains(&name) { + let _ = std::fs::remove_file(entry.path()); + } + } + } + let _ = std::fs::remove_dir_all(&self.stage_dir); + self.manifest.store(&self.index_path) + } +} + +// --------------------------------------------------------------------------- +// FULL +// --------------------------------------------------------------------------- + +/// Run a FULL (bootstrap) build: duplicate pre-pass, then a fresh temp +/// index written dataset by dataset through chunked commits, swapped into +/// place only on full success, then a fresh sharded sidecar. pub fn run_full(config: &RunConfig) -> Result<Stats, String> { + run_full_with_reason(config, None) +} + +fn run_full_with_reason(config: &RunConfig, reason: Option<String>) -> Result<Stats, String> { let caches = RdfCaches::load(&config.rdf_cache)?; - let genes = rows::crawl_genes(&config.sql_uri)?; - let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?; + // Duplicate validation happens BEFORE any index is opened or written. + let (duplicates, max_dataset_terms) = + duplicate_prepass(&config.sql_uri, config.publish_filter)?; let red = red_mode(); - - // Identity validation happens BEFORE any index is opened or - // written, so a duplicate crawl cannot touch an existing index. - let (_idterms, duplicates) = identity_prepass(&genes, &phenotypes); if !duplicates.is_empty() && red.as_deref() != Some("duplicate") { return Err(format!( - "duplicate identity terms in crawl: {}", - duplicates.join(", ") + "duplicate identity terms in crawl: {}{}", + duplicates.join(", "), + match &reason { + Some(r) => format!(" (startup: {r})"), + None => String::new(), + } )); } livemem::reset(); + sidecar::mem::reset(); + sidecar::cleanup_temp(&config.index_path)?; let retain = livemem::red_retain(); + let temp = temp_index_path(&config.index_path); - let outcome = stream_full( - &temp, - &genes, - &phenotypes, - &caches, - red.as_deref() == Some("hash"), - retain, + if temp.exists() { + let _ = std::fs::remove_dir_all(&temp); + } + let mut build = SidecarBuild::new( + &config.index_path, + "full", + caches.rif.hash.clone(), + caches.wiki.hash.clone(), ); - let (index_doc_count, hashes, built) = match outcome { + let outcome = stream_full(&temp, config, &caches, &mut build, retain); + let (index_doc_count, built, pacer_chunks, pacer_rss_peak) = match outcome { Ok(value) => value, Err(error) => { let _ = std::fs::remove_dir_all(&temp); + let _ = std::fs::remove_dir_all(&build.stage_dir); return Err(error); } }; @@ -252,96 +533,195 @@ pub fn run_full(config: &RunConfig) -> Result<Stats, String> { return Err(error); } - // Sidecar strictly after the index commit (and the swap). - let sidecar = Sidecar { - version: 1, - mode: "full".to_string(), - doc_count: built, - rif_hash: caches.rif.hash.clone(), - wiki_hash: caches.wiki.hash.clone(), - documents: hashes, - }; - sidecar.store(&config.index_path)?; + // Sidecar strictly after the index commit (and the swap): changed + // shards first, then the manifest last. + let shard_count = build.shard_count(); + let stage_dir = build.stage_dir.clone(); + if let Err(error) = build.finalize() { + let _ = std::fs::remove_dir_all(&stage_dir); + return Err(error); + } - Ok(Stats { + let mut stats = Stats { mode: "full".to_string(), built, replaced: built, + deleted: 0, + unchanged: 0, index_doc_count, duplicate_identity_terms: Vec::new(), max_live_documents: livemem::max_live(), - ..Default::default() - }) + shards_written: shard_count, + shards_deleted: 0, + max_resident_sidecar_entries: sidecar::mem::peak_resident_entries(), + max_resident_dataset_terms: max_dataset_terms, + peak_open_shards: sidecar::mem::peak_open_shards(), + shard_bytes_read: sidecar::mem::bytes_read(), + commit_chunks: pacer_chunks, + chunk_rss_peak_kib: pacer_rss_peak, + startup: match reason { + Some(reason) => format!("full ({reason})"), + None => "full".to_string(), + }, + }; + stats.mode = "full".to_string(); + Ok(stats) +} + +struct FullSink<'a> { + db: WritableDatabase, + caches: &'a RdfCaches, + red_hash: bool, + retain: bool, + kept: Vec<CountedDocument<BuiltDocument>>, + pacer: CommitPacer, + build: &'a mut SidecarBuild, + shard: BTreeMap<String, String>, + built: usize, +} + +impl<'a> DatasetSink for FullSink<'a> { + fn begin_dataset(&mut self, _label: &str) -> Result<(), String> { + self.shard.clear(); + Ok(()) + } + + fn row(&mut self, row: CrawlRow) -> Result<(), String> { + let Some(mut document) = build_row(row, self.caches) else { + return Ok(()); + }; + if let Some(nonce) = red_nonce(self.red_hash) { + document.source_hash.push_str(&nonce); + } + let counted = CountedDocument::new(document); + let doc = counted.get(); + self.db.replace_document_by_term(&doc.idterm, &doc.document); + self.shard + .insert(doc.idterm.clone(), doc.source_hash.clone()); + self.built += 1; + if self.retain { + self.kept.push(counted); + } + self.pacer.write(&mut self.db)?; + Ok(()) + } + + fn end_dataset(&mut self, label: &str) -> Result<(), String> { + if !self.shard.is_empty() { + self.build.stage(label, &self.shard)?; + } + self.shard.clear(); + Ok(()) + } } -/// Stream a fresh FULL build into `temp`: open once, one transaction, -/// write each document as it is built. Returns the committed document -/// count, the idterm→source-hash map for the sidecar, and the built -/// count. fn stream_full( temp: &Path, - genes: &[GeneRow], - phenotypes: &[PhenotypeRow], + config: &RunConfig, caches: &RdfCaches, - red_hash: bool, + build: &mut SidecarBuild, retain: bool, -) -> Result<(u32, BTreeMap<String, String>, usize), String> { - let mut db = WritableDatabase::open( +) -> Result<(u32, usize, usize, u64), String> { + // One writer handle, opened into the fresh temp directory; the + // transaction is opened lazily by the commit pacer. + let db = WritableDatabase::open( temp, Some(DbAction::CreateOrOverwrite), Some(DbBackend::Glass), None, None, ); - db.begin_transaction(None); - - let mut hashes: BTreeMap<String, String> = BTreeMap::new(); - let mut kept: Vec<CountedDocument<BuiltDocument>> = Vec::new(); - let mut count = 0usize; - { - let emit = |document: BuiltDocument| -> Result<(), String> { - let counted = CountedDocument::new(document); - let idterm = counted.get().idterm.clone(); - hashes.insert(idterm.clone(), counted.get().source_hash.clone()); - let _ = db.replace_document_by_term(&idterm, &counted.get().document); - count += 1; - if retain { - kept.push(counted); - } - Ok(()) - }; - if let Err(error) = for_each_document(genes, phenotypes, caches, red_hash, emit) { - db.cancel_transaction(); - return Err(error); + let mut sink = FullSink { + db, + caches, + red_hash: red_mode().as_deref() == Some("hash"), + retain, + kept: Vec::new(), + pacer: CommitPacer::new(red_mode().as_deref() == Some("commit")), + build, + shard: BTreeMap::new(), + built: 0, + }; + let result = rows::stream_dataset_groups(&config.sql_uri, config.publish_filter, &mut sink); + if let Err(error) = result { + if sink.pacer.open { + sink.db.cancel_transaction(); } + return Err(error); } - - commit(&mut db, red_mode().as_deref() == Some("commit"))?; - let doc_count = db.doc_count(); - drop(db); - drop(kept); - Ok((doc_count, hashes, count)) + sink.pacer.finish(&mut sink.db)?; + let doc_count = sink.db.doc_count(); + let built = sink.built; + let chunks = sink.pacer.chunks; + let rss_peak = sink.pacer.rss_peak_kib; + Ok((doc_count, built, chunks, rss_peak)) } -/// Run an INCREMENTAL pass: crawl, hash, diff against the sidecar, -/// replace only changed documents, delete vanished identity terms, and -/// write nothing when nothing changed. +// --------------------------------------------------------------------------- +// INCREMENTAL +// --------------------------------------------------------------------------- + +/// Run an INCREMENTAL pass: cheap-first startup check, then a dataset- +/// grouped streaming diff that replaces changed documents, deletes +/// vanished ones, and writes nothing when nothing changed. pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { - let Some(previous) = Sidecar::load(&config.index_path) else { - // Missing or unreadable sidecar means FULL. - return run_full(config); + let Some(manifest) = Manifest::load(&config.index_path) else { + // Missing, unreadable, or old-version sidecar means FULL. + return run_full_with_reason(config, Some("no/old sidecar".to_string())); }; - if previous.version != 1 { - return run_full(config); + if !index_exists(&config.index_path) { + return run_full_with_reason(config, Some("index missing".to_string())); } - let caches = RdfCaches::load(&config.rdf_cache)?; - let genes = rows::crawl_genes(&config.sql_uri)?; - let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?; + sidecar::cleanup_temp(&config.index_path)?; + + // Cheap-first startup (O(1)): the manifest doc count against the + // index's own doc count. + let probe = WritableDatabase::open( + &config.index_path, + Some(DbAction::CreateOrOpen), + Some(DbBackend::Glass), + None, + None, + ); + let index_doc_count = probe.doc_count(); let red = red_mode(); + let no_recovery = red.as_deref() == Some("no-recovery"); + if !no_recovery && index_doc_count as usize != manifest.doc_count { + let reason = format!( + "doc_count index={} manifest={}", + index_doc_count, manifest.doc_count + ); + // Expensive path only here: enumerate index identity terms to + // characterise the key-set mismatch, then rebuild. + let index_terms = index_identity_terms(&probe); + eprintln!( + "gn-indexer: startup mismatch ({reason}); index identity terms = {}; falling back to FULL", + index_terms.len() + ); + drop(probe); + return run_full_with_reason(config, Some(reason)); + } + drop(probe); - // Identity pre-pass, before any write. - let (idterms, duplicates) = identity_prepass(&genes, &phenotypes); + // Verify every shard (bit-rot) and confirm it parses, before trusting + // the manifest. Any mismatch routes to FULL. Each shard is loaded and + // dropped, so the peak stays O(largest shard). + if !no_recovery && !red_skip_shard_check() { + let dir = sidecar::shard_dir(&config.index_path); + for (label, entry) in &manifest.shards { + if let Err(error) = ResidentShard::load(&dir, label, entry) { + let reason = format!("shard {label}: {error}"); + eprintln!("gn-indexer: startup mismatch ({reason}); falling back to FULL"); + return run_full_with_reason(config, Some(reason)); + } + } + } + + let caches = RdfCaches::load(&config.rdf_cache)?; + // Duplicate validation before any write. + let (duplicates, max_dataset_terms) = + duplicate_prepass(&config.sql_uri, config.publish_filter)?; if !duplicates.is_empty() && red.as_deref() != Some("duplicate") { return Err(format!( "duplicate identity terms in crawl: {}", @@ -350,9 +730,10 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { } livemem::reset(); + sidecar::mem::reset(); let retain = livemem::red_retain(); - let mut db = WritableDatabase::open( + let db = WritableDatabase::open( &config.index_path, Some(DbAction::CreateOrOpen), Some(DbBackend::Glass), @@ -360,136 +741,292 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { None, ); - // Startup mismatch detection: the sidecar's key set must equal the - // index's actual identity terms, else the sidecar is behind the - // index (or the index was written elsewhere) and is not trusted; do - // a full compare against the index's own terms. - let index_terms = index_identity_terms(&db); - let sidecar_terms: BTreeSet<String> = previous.documents.keys().cloned().collect(); - let mismatched = index_terms != sidecar_terms; - - let mut hashes: BTreeMap<String, String> = BTreeMap::new(); - let mut unchanged = 0usize; - let mut replaced = 0usize; - let mut open_transaction = false; - let mut kept: Vec<CountedDocument<BuiltDocument>> = Vec::new(); - { - let previous_documents = &previous.documents; - let emit = |document: BuiltDocument| -> Result<(), String> { - let counted = CountedDocument::new(document); - let doc = counted.get(); - hashes.insert(doc.idterm.clone(), doc.source_hash.clone()); - let known = if mismatched { - None - } else { - previous_documents.get(&doc.idterm) - }; - match known { - Some(hash) if *hash == doc.source_hash => unchanged += 1, - _ => { - if !open_transaction { - db.begin_transaction(None); - open_transaction = true; - } - let _ = db.replace_document_by_term(&doc.idterm, &doc.document); - replaced += 1; - } - } - if retain { - kept.push(counted); - } - Ok(()) - }; - if let Err(error) = for_each_document( - &genes, - &phenotypes, - &caches, - red.as_deref() == Some("hash"), - emit, - ) { - if open_transaction { - db.cancel_transaction(); - } - return Err(error); - } + let shard_dir = sidecar::shard_dir(&config.index_path); + let by_label: HashMap<String, ShardEntry> = manifest + .shards + .iter() + .map(|(label, entry)| (label.clone(), entry.clone())) + .collect(); + + let mut build = SidecarBuild::new( + &config.index_path, + "incremental", + caches.rif.hash.clone(), + caches.wiki.hash.clone(), + ); + // Carry forward every existing shard entry; changed datasets replace + // their entry, removed datasets are dropped below. + for (label, entry) in &manifest.shards { + build.keep(label, entry.clone()); } - // Existence truth: the sidecar's key set (or, on mismatch, the - // index's own terms). Anything there but not in the crawl is gone. - // Deletes need the full crawl, so they are applied at the end. - let mut to_delete: BTreeSet<String> = if mismatched { - index_terms - } else { - sidecar_terms + let mut sink = IncrementalSink { + db, + caches: &caches, + red_hash: red.as_deref() == Some("hash"), + red_delete: red.as_deref() == Some("delete"), + no_recovery, + retain, + kept: Vec::new(), + pacer: CommitPacer::new(red.as_deref() == Some("commit")), + build: &mut build, + shard_dir, + by_label, + seen: BTreeSet::new(), + previous: None, + previous_label: String::new(), + dataset_terms: BTreeSet::new(), + dataset_map: BTreeMap::new(), + retainer: sidecar::ResidentRetainer::new(), + replaced: 0, + deleted: 0, + unchanged: 0, + max_dataset_terms: 0, + new_shards: 0, + removed_shards: 0, }; - if red.as_deref() == Some("delete") { - to_delete.clear(); - } else { - for term in &idterms { - to_delete.remove(term); + + let result = rows::stream_dataset_groups(&config.sql_uri, config.publish_filter, &mut sink); + if let Err(error) = result { + if sink.pacer.open { + sink.db.cancel_transaction(); } + return Err(error); } - let mut deleted = 0usize; - if !to_delete.is_empty() { - if !open_transaction { - db.begin_transaction(None); - open_transaction = true; - } - for term in &to_delete { - db.delete_document_by_term(term); - deleted += 1; - } + // Datasets in the manifest that the crawl never produced: every one + // of their documents is gone. + let leftovers: Vec<String> = sink + .by_label + .keys() + .filter(|label| !sink.seen.contains(*label)) + .cloned() + .collect(); + for label in leftovers { + sink.delete_dataset(&label)?; + sink.removed_shards += 1; } - let mut stats = Stats { + let nothing_changed = sink.replaced == 0 && sink.deleted == 0; + sink.pacer.finish(&mut sink.db)?; + let doc_count = sink.db.doc_count(); + let replaced = sink.replaced; + let deleted = sink.deleted; + let unchanged = sink.unchanged; + let max_resident = sidecar::mem::peak_resident_entries(); + let peak_open = sidecar::mem::peak_open_shards(); + let bytes_read = sidecar::mem::bytes_read(); + let shards_written = sink.new_shards; + let shards_deleted = sink.removed_shards; + let sink_dataset_terms = sink.max_dataset_terms; + let commit_chunks = sink.pacer.chunks; + let chunk_rss_peak_kib = sink.pacer.rss_peak_kib; + let max_live = livemem::max_live(); + drop(sink); + + // The finalized manifest's shard counts are the crawl's document + // count (kept + staged, minus removed datasets). + let built = build.total_count(); + + // Sidecar after the commit; skipped when nothing changed so an + // idempotent run touches neither the index nor the sidecar. + if !nothing_changed { + build.finalize()?; + } else { + let _ = std::fs::remove_dir_all(&build.stage_dir); + } + + Ok(Stats { mode: "incremental".to_string(), - built: hashes.len(), - unchanged, + built, replaced, deleted, - max_live_documents: livemem::max_live(), - ..Default::default() - }; + unchanged, + index_doc_count: doc_count, + duplicate_identity_terms: Vec::new(), + max_live_documents: max_live, + shards_written, + shards_deleted, + max_resident_sidecar_entries: max_resident, + max_resident_dataset_terms: max_dataset_terms.max(sink_dataset_terms), + peak_open_shards: peak_open, + shard_bytes_read: bytes_read, + commit_chunks, + chunk_rss_peak_kib, + startup: "incremental".to_string(), + }) +} - let nothing_changed = replaced == 0 && deleted == 0; - if open_transaction { - // A zero-change run opens no transaction and writes nothing. - commit(&mut db, red.as_deref() == Some("commit"))?; +struct IncrementalSink<'a> { + db: WritableDatabase, + caches: &'a RdfCaches, + red_hash: bool, + red_delete: bool, + no_recovery: bool, + retain: bool, + kept: Vec<CountedDocument<BuiltDocument>>, + pacer: CommitPacer, + build: &'a mut SidecarBuild, + shard_dir: PathBuf, + by_label: HashMap<String, ShardEntry>, + seen: BTreeSet<String>, + previous: Option<ResidentShard>, + previous_label: String, + dataset_terms: BTreeSet<String>, + dataset_map: BTreeMap<String, String>, + retainer: sidecar::ResidentRetainer, + replaced: usize, + deleted: usize, + unchanged: usize, + max_dataset_terms: usize, + new_shards: usize, + removed_shards: usize, +} + +impl<'a> IncrementalSink<'a> { + /// Delete every document of a dataset (all its rows vanished) and + /// drop its shard. + fn delete_dataset(&mut self, label: &str) -> Result<(), String> { + if self.no_recovery { + // RED: trust the sidecar, skip the recovery work entirely. + return Ok(()); + } + if !self.red_delete { + let entry = self.by_label.get(label).cloned(); + if let Some(entry) = entry { + let shard = ResidentShard::load(&self.shard_dir, label, &entry) + .map_err(|e| format!("shard {label}: {e}"))?; + for term in shard.map().keys() { + self.db.delete_document_by_term(term); + self.deleted += 1; + self.pacer.write(&mut self.db)?; + } + } + } + self.build.remove(label); + Ok(()) } - stats.index_doc_count = db.doc_count(); +} - // Sidecar after the commit; skipped when nothing changed so an - // idempotent run touches neither the index nor the sidecar. - if !nothing_changed { - let sidecar = Sidecar { - version: 1, - mode: "incremental".to_string(), - doc_count: hashes.len(), - rif_hash: caches.rif.hash.clone(), - wiki_hash: caches.wiki.hash.clone(), - documents: hashes, +impl<'a> DatasetSink for IncrementalSink<'a> { + fn begin_dataset(&mut self, label: &str) -> Result<(), String> { + self.seen.insert(label.to_string()); + self.dataset_terms.clear(); + self.dataset_map.clear(); + self.previous = None; + self.previous_label = label.to_string(); + if let Some(entry) = self.by_label.get(label).cloned() { + // Verify-before-parse: the shard file's hash is re-checked + // here (it was checked at startup too). A mismatch would mean + // the file changed under us; treat it as a hard error. + match ResidentShard::load(&self.shard_dir, label, &entry) { + Ok(shard) => { + if sidecar::red_load_all() { + // Load-all RED: hold every shard resident so the peak + // resident counter reflects O(docs), not O(largest shard). + self.retainer.retain(shard); + } else { + self.previous = Some(shard); + } + } + Err(error) => { + if red_skip_shard_check() { + // RED: swallow the corruption as an empty shard, so + // the dataset is silently treated as new/empty. + eprintln!("gn-indexer: [RED skip-shard-check] ignoring {label}: {error}"); + self.previous = None; + } else { + return Err(format!("shard {label}: {error}")); + } + } + } + } + Ok(()) + } + + fn row(&mut self, row: CrawlRow) -> Result<(), String> { + let Some(idterm) = row_idterm(&row) else { + return Ok(()); + }; + self.dataset_terms.insert(idterm.clone()); + self.max_dataset_terms = self.max_dataset_terms.max(self.dataset_terms.len()); + let Some(mut document) = build_row(row, self.caches) else { + return Ok(()); }; - sidecar.store(&config.index_path)?; + if let Some(nonce) = red_nonce(self.red_hash) { + document.source_hash.push_str(&nonce); + } + let counted = CountedDocument::new(document); + let doc = counted.get(); + self.dataset_map + .insert(doc.idterm.clone(), doc.source_hash.clone()); + let known = if self.no_recovery { + // RED: pretend the source hash is unchanged, so no rewrite + // ever happens and a crash cannot be recovered. + Some(doc.source_hash.clone()) + } else { + self.previous + .as_ref() + .and_then(|shard| shard.map().get(&doc.idterm).cloned()) + }; + match known { + Some(hash) if hash == doc.source_hash => self.unchanged += 1, + _ => { + self.db.replace_document_by_term(&doc.idterm, &doc.document); + self.replaced += 1; + self.pacer.write(&mut self.db)?; + } + } + if self.retain { + self.kept.push(counted); + } + Ok(()) } - let _ = kept; - Ok(stats) -} -fn commit(db: &mut WritableDatabase, skip: bool) -> Result<(), String> { - if skip { - // RED control: writes are made but never committed, so they are - // invisible to readers. - db.cancel_transaction(); - return Ok(()); + fn end_dataset(&mut self, label: &str) -> Result<(), String> { + // Deletes: identity terms in the previous shard that the crawl no + // longer produces, computed from this dataset's key set. + if !self.red_delete && !self.no_recovery { + if let Some(shard) = &self.previous { + let vanished: Vec<String> = shard + .map() + .keys() + .filter(|term| !self.dataset_terms.contains(*term)) + .cloned() + .collect(); + for term in vanished { + self.db.delete_document_by_term(&term); + self.deleted += 1; + self.pacer.write(&mut self.db)?; + } + } + } + + // Decide whether the shard content changed. + let unchanged_shard = match &self.previous { + Some(shard) => shard.map() == &self.dataset_map, + None => self.dataset_map.is_empty(), + }; + if !unchanged_shard && !self.dataset_map.is_empty() { + self.build.stage(label, &self.dataset_map)?; + self.new_shards += 1; + } else if unchanged_shard { + if let Some(entry) = self.by_label.get(label).cloned() { + self.build.keep(label, entry); + } + } else if self.dataset_map.is_empty() { + // The dataset still appears in the crawl but produced no + // documents: treat it as gone. + self.build.remove(label); + } + self.previous = None; + Ok(()) } - db.commit_transaction(); - Ok(()) } /// Where the sidecar lives for an index path. pub fn sidecar_location(index_path: &Path) -> PathBuf { - super::sidecar::sidecar_path(index_path) + sidecar::sidecar_path(index_path) } /// The document type of an identity term (for reporting). diff --git a/src/indexer/sidecar.rs b/src/indexer/sidecar.rs index 8dcc5a3..394fe50 100644 --- a/src/indexer/sidecar.rs +++ b/src/indexer/sidecar.rs @@ -1,23 +1,71 @@ -//! The incremental sidecar: per-document source hashes stored next to -//! the index (not inside it), written atomically and strictly after the -//! index commit. +//! The incremental sidecar, sharded by dataset (format version 2). //! -//! The sidecar's key set is the existence truth for the incremental -//! path: a document absent from a crawl is deleted by its identity term -//! even if the index still holds it. Write order is index commit first, -//! sidecar rename after, so a crash between the two leaves a sidecar -//! that is behind the index — the startup key-set check detects exactly -//! that and falls back to a full compare. +//! A small **manifest** at `<index>.sidecar.json` (version 2) holds the +//! run metadata and one entry per dataset shard: +//! +//! ```text +//! { version, mode, doc_count, rif_hash, wiki_hash, +//! shards: { "<dataset>": { file, count, content_sha256 } } } +//! ``` +//! +//! Each shard is a JSON map `identity-term -> source-hash` at +//! `<index>.sidecar.d/<file>`, where the filename is the dataset label +//! sanitized for the filesystem plus a short digest of the raw label (so +//! two labels that sanitize to the same string still get distinct files). +//! +//! Why sharding: the per-dataset shard is the unit the incremental diff +//! loads, so the resident idterm->hash map is O(largest shard), not +//! O(documents). The manifest is sharded so a crawl can rewrite one +//! dataset without touching the rest. +//! +//! Integrity and atomicity: +//! +//! * a shard's `content_sha256` is **recomputed from the shard file** at +//! startup (bit-rot detection) — it is never blindly carried forward; +//! * a shard is **verified before it is parsed**, and a hash mismatch or +//! an unparseable shard routes to the full-rebuild path — a corrupt +//! shard is never swallowed as an empty one (which would silently drop +//! its documents); +//! * changed shards are written temp + fsync + rename + directory fsync, +//! and the manifest is written the same way **last**, so a crash leaves +//! the previous, self-consistent sidecar in place (stray temp files are +//! ignored and cleaned). +//! +//! An old version-1 monolithic sidecar (one `documents` map, no +//! `shards`) does not deserialize as a manifest, so it triggers FULL. use std::collections::BTreeMap; +use std::fs::{File, OpenOptions}; +use std::io::Write; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; use serde::{Deserialize, Serialize}; -/// The on-disk sidecar. -#[derive(Serialize, Deserialize, Default, PartialEq)] -pub struct Sidecar { - /// Format version; a mismatch triggers a full rebuild. +use super::sha256::{Sha256, hex_digest}; + +/// Sidecar format version. A manifest with any other version is ignored +/// and the run falls back to FULL (so a version-1 monolithic file, which +/// additionally fails to deserialize, is never mis-parsed). +pub const SIDECAR_VERSION: u32 = 2; + +/// The shard directory suffix: `<index>.sidecar.d`. +const SHARD_DIR_SUFFIX: &str = ".sidecar.d"; + +/// One shard's manifest entry. +#[derive(Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct ShardEntry { + /// Filename inside the shard directory. + pub file: String, + /// Document count of the shard. + pub count: usize, + /// SHA-256 of the shard file's bytes. + pub content_sha256: String, +} + +/// The on-disk manifest. +#[derive(Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct Manifest { pub version: u32, /// How the sidecar was last written (`full` or `incremental`). pub mode: String, @@ -27,11 +75,11 @@ pub struct Sidecar { pub rif_hash: String, /// Monolithic wiki cache hash at the time of writing. pub wiki_hash: String, - /// Identity term -> source hash. - pub documents: BTreeMap<String, String>, + /// Dataset label -> shard entry. + pub shards: BTreeMap<String, ShardEntry>, } -/// The sidecar path for an index directory (`<index>.sidecar.json`). +/// The manifest path for an index directory (`<index>.sidecar.json`). pub fn sidecar_path(index_path: &Path) -> PathBuf { PathBuf::from(format!( "{}.sidecar.json", @@ -39,20 +87,366 @@ pub fn sidecar_path(index_path: &Path) -> PathBuf { )) } -impl Sidecar { - pub fn load(index_path: &Path) -> Option<Sidecar> { +/// The shard directory for an index directory (`<index>.sidecar.d`). +pub fn shard_dir(index_path: &Path) -> PathBuf { + PathBuf::from(format!( + "{}{}", + index_path.to_string_lossy().trim_end_matches('/'), + SHARD_DIR_SUFFIX + )) +} + +/// Sanitize a dataset label into a filesystem-safe stem: every byte +/// outside `[A-Za-z0-9._-]` becomes `_`, leading dots are stripped, and +/// an empty result becomes `dataset`. +pub fn sanitize_label(label: &str) -> String { + let mut out = String::with_capacity(label.len()); + for ch in label.chars() { + if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' || ch == '.' { + out.push(ch); + } else { + out.push('_'); + } + } + while out.starts_with('.') { + out.remove(0); + } + if out.is_empty() { + out.push_str("dataset"); + } + out +} + +/// The shard filename for a dataset label: the sanitized stem plus a +/// short digest of the raw label, so distinct labels cannot collide. +pub fn shard_filename(label: &str) -> String { + let stem = sanitize_label(label); + let digest = hex_digest(label.as_bytes()); + format!("{}-{}.json", stem, &digest[..8]) +} + +impl Manifest { + /// Load and validate the manifest. `None` when the file is missing, + /// unreadable, fails to deserialize (an old version-1 file, or any + /// other shape), or carries a version other than [`SIDECAR_VERSION`]. + pub fn load(index_path: &Path) -> Option<Manifest> { let text = std::fs::read_to_string(sidecar_path(index_path)).ok()?; - serde_json::from_str(&text).ok() + let manifest: Manifest = serde_json::from_str(&text).ok()?; + if manifest.version != SIDECAR_VERSION { + return None; + } + Some(manifest) } - /// Atomically replace the sidecar: write a sibling temp file, sync, - /// then rename over the target. + /// Atomically replace the manifest (temp + fsync + rename + dir + /// fsync). Callers write changed shards first; the manifest is last. pub fn store(&self, index_path: &Path) -> Result<(), String> { let path = sidecar_path(index_path); - let temp = PathBuf::from(format!("{}.tmp.{}", path.display(), std::process::id())); - let json = serde_json::to_string(self).map_err(|e| format!("sidecar encode: {e}"))?; - std::fs::write(&temp, json).map_err(|e| format!("sidecar write: {e}"))?; - std::fs::rename(&temp, &path).map_err(|e| format!("sidecar rename: {e}"))?; - Ok(()) + let json = serde_json::to_string(self).map_err(|e| format!("manifest encode: {e}"))?; + atomic_write(&path, json.as_bytes()) } } + +/// Write `bytes` to `path` atomically: sibling temp file, fsync the file, +/// rename over the target, then fsync the containing directory. +pub fn atomic_write(path: &Path, bytes: &[u8]) -> Result<(), String> { + let temp = PathBuf::from(format!("{}.tmp.{}", path.display(), std::process::id())); + { + let mut file = + File::create(&temp).map_err(|e| format!("create {}: {e}", temp.display()))?; + file.write_all(bytes) + .map_err(|e| format!("write {}: {e}", temp.display()))?; + file.sync_all() + .map_err(|e| format!("fsync {}: {e}", temp.display()))?; + } + std::fs::rename(&temp, path).map_err(|e| format!("rename {}: {e}", path.display()))?; + fsync_dir(path.parent().unwrap_or_else(|| Path::new("."))) +} + +/// fsync a directory so a rename into it is durable. +pub fn fsync_dir(dir: &Path) -> Result<(), String> { + let handle = File::open(dir).map_err(|e| format!("open dir {}: {e}", dir.display()))?; + handle + .sync_all() + .map_err(|e| format!("fsync dir {}: {e}", dir.display())) +} + +/// Stage a changed shard: serialize `map`, write it to a temp file in +/// `dir`, fsync it, and return the temp path plus the manifest entry it +/// will become after [`finalize_shard`]. The temp is **not** renamed yet, +/// so a crash mid-run leaves the previous sidecar untouched. +pub fn stage_shard( + dir: &Path, + label: &str, + map: &BTreeMap<String, String>, +) -> Result<(PathBuf, ShardEntry), String> { + std::fs::create_dir_all(dir).map_err(|e| format!("create shard dir {}: {e}", dir.display()))?; + let file = shard_filename(label); + let temp = dir.join(format!(".tmp-{}-{}", std::process::id(), file)); + let json = serde_json::to_vec(map).map_err(|e| format!("shard encode: {e}"))?; + { + let mut handle = + File::create(&temp).map_err(|e| format!("create shard {}: {e}", temp.display()))?; + handle + .write_all(&json) + .map_err(|e| format!("write shard {}: {e}", temp.display()))?; + handle + .sync_all() + .map_err(|e| format!("fsync shard {}: {e}", temp.display()))?; + } + let entry = ShardEntry { + file, + count: map.len(), + content_sha256: hex_digest(&json), + }; + Ok((temp, entry)) +} + +/// Rename a staged shard temp into place and fsync the shard directory. +pub fn finalize_shard(temp: &Path, dir: &Path, entry: &ShardEntry) -> Result<(), String> { + let target = dir.join(&entry.file); + std::fs::rename(temp, &target) + .map_err(|e| format!("rename shard {}: {e}", target.display()))?; + fsync_dir(dir) +} + +/// Why a shard was rejected at verification time. +#[derive(Debug)] +pub enum ShardError { + /// The shard file could not be read. + Read(String), + /// The shard file's bytes hash to something other than its manifest + /// `content_sha256`. + HashMismatch { expected: String, actual: String }, + /// The shard file hashed correctly but did not parse. Treated exactly + /// like a hash mismatch so a corrupt shard can never be read as empty. + Unparseable(String), +} + +impl std::fmt::Display for ShardError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ShardError::Read(e) => write!(f, "read: {e}"), + ShardError::HashMismatch { expected, actual } => { + write!( + f, + "content_sha256 mismatch (manifest {expected}, file {actual})" + ) + } + ShardError::Unparseable(e) => write!(f, "unparseable: {e}"), + } + } +} + +/// Read a shard file's bytes and check them against `entry.content_sha256` +/// **without parsing**. This is the startup bit-rot pass: an unchanged +/// shard's hash is recomputed from the file, never trusted blindly. +pub fn verify_shard_bytes(dir: &Path, entry: &ShardEntry) -> Result<Vec<u8>, ShardError> { + let path = dir.join(&entry.file); + let bytes = + std::fs::read(&path).map_err(|e| ShardError::Read(format!("{}: {e}", path.display())))?; + mem::add_bytes_read(bytes.len() as u64); + let actual = hex_digest(&bytes); + if actual != entry.content_sha256 { + return Err(ShardError::HashMismatch { + expected: entry.content_sha256.clone(), + actual, + }); + } + Ok(bytes) +} + +/// Verify a shard and then parse it. The parse is only attempted after +/// the hash matches, and a parse failure is surfaced as a rejection +/// (routing to full) rather than as an empty shard. +pub fn read_verified_shard( + dir: &Path, + entry: &ShardEntry, +) -> Result<BTreeMap<String, String>, ShardError> { + let bytes = verify_shard_bytes(dir, entry)?; + let map: BTreeMap<String, String> = serde_json::from_slice(&bytes) + .map_err(|e| ShardError::Unparseable(format!("{}: {e}", entry.file)))?; + Ok(map) +} + +/// A shard map whose entries are accounted as resident until dropped. +/// The high-water mark proves the diff loads O(largest shard), not all +/// shards: a RED that holds every shard alive makes the peak O(docs). +pub struct ResidentShard { + /// Dataset label this shard belongs to. + pub label: String, + map: BTreeMap<String, String>, +} + +impl ResidentShard { + pub fn load(dir: &Path, label: &str, entry: &ShardEntry) -> Result<ResidentShard, ShardError> { + let map = read_verified_shard(dir, entry)?; + mem::note_load(map.len()); + Ok(ResidentShard { + label: label.to_string(), + map, + }) + } + + pub fn map(&self) -> &BTreeMap<String, String> { + &self.map + } +} + +impl Drop for ResidentShard { + fn drop(&mut self) { + mem::note_drop(self.map.len()); + } +} + +/// Process-global observability counters for the sharded sidecar. They +/// exist so "load-all" cannot quietly pass: the run asserts +/// `max_resident_entries` is O(largest shard). +pub mod mem { + use super::{AtomicUsize, Ordering}; + + static RESIDENT: AtomicUsize = AtomicUsize::new(0); + static PEAK_RESIDENT: AtomicUsize = AtomicUsize::new(0); + static BYTES_READ: AtomicUsize = AtomicUsize::new(0); + static OPEN_SHARDS: AtomicUsize = AtomicUsize::new(0); + static PEAK_OPEN: AtomicUsize = AtomicUsize::new(0); + + /// Reset all counters before a run. + pub fn reset() { + RESIDENT.store(0, Ordering::SeqCst); + PEAK_RESIDENT.store(0, Ordering::SeqCst); + BYTES_READ.store(0, Ordering::SeqCst); + OPEN_SHARDS.store(0, Ordering::SeqCst); + PEAK_OPEN.store(0, Ordering::SeqCst); + } + + pub(super) fn note_load(entries: usize) { + let resident = RESIDENT.fetch_add(entries, Ordering::SeqCst) + entries; + PEAK_RESIDENT.fetch_max(resident, Ordering::SeqCst); + let open = OPEN_SHARDS.fetch_add(1, Ordering::SeqCst) + 1; + PEAK_OPEN.fetch_max(open, Ordering::SeqCst); + } + + pub(super) fn note_drop(entries: usize) { + RESIDENT.fetch_sub(entries, Ordering::SeqCst); + OPEN_SHARDS.fetch_sub(1, Ordering::SeqCst); + } + + /// Count bytes read from shard files, whether or not parsed. + pub fn add_bytes_read(bytes: u64) { + BYTES_READ.fetch_add(bytes as usize, Ordering::SeqCst); + } + + /// High-water mark of simultaneously resident shard-map entries. + pub fn peak_resident_entries() -> usize { + PEAK_RESIDENT.load(Ordering::SeqCst) + } + + /// High-water mark of simultaneously open shards. + pub fn peak_open_shards() -> usize { + PEAK_OPEN.load(Ordering::SeqCst) + } + + /// Cumulative shard-file bytes read. + pub fn bytes_read() -> usize { + BYTES_READ.load(Ordering::SeqCst) + } +} + +/// Keep an already-loaded shard resident (the "load-all" RED). Holding +/// every shard alive drives the peak-resident counter to O(docs). +pub struct ResidentRetainer { + held: Vec<ResidentShard>, +} + +impl ResidentRetainer { + pub fn new() -> Self { + ResidentRetainer { held: Vec::new() } + } + + pub fn retain(&mut self, shard: ResidentShard) { + self.held.push(shard); + } +} + +impl Default for ResidentRetainer { + fn default() -> Self { + Self::new() + } +} + +/// Whether to hold every loaded shard alive (the load-all RED control). +pub fn red_load_all() -> bool { + std::env::var("GN4_INDEXER_RED_SIDECAR").as_deref() == Ok("load-all") +} + +/// Remove stray temp files a previous crash may have left: staged shard +/// temps and stage directories, manifest temps, and half-written index +/// temp/backup directories. They are never read. +pub fn cleanup_temp(index_path: &Path) -> Result<(), String> { + let dir = shard_dir(index_path); + if let Ok(entries) = std::fs::read_dir(&dir) { + for entry in entries.flatten() { + let name = entry.file_name(); + if name.to_string_lossy().starts_with(".tmp-") { + let _ = std::fs::remove_file(entry.path()); + } + } + } + let parent = index_path.parent().unwrap_or_else(|| Path::new(".")); + let manifest = sidecar_path(index_path); + let stem = manifest + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + let shard_stem = format!( + "{}.tmp.", + dir.file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default() + ); + let index_stem = index_path + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + if let Ok(entries) = std::fs::read_dir(parent) { + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with(&format!("{stem}.tmp.")) || name.starts_with(&shard_stem) { + let path = entry.path(); + if path.is_dir() { + let _ = std::fs::remove_dir_all(&path); + } else { + let _ = std::fs::remove_file(&path); + } + } + // Left-over FULL index temp dirs and backups from a crash. + if name.starts_with(&format!(".{index_stem}.tmp.")) + || name.starts_with(&format!(".{index_stem}.old.")) + { + let _ = std::fs::remove_dir_all(entry.path()); + } + } + } + Ok(()) +} + +/// A SHA-256 helper mirroring the sidecar's hashing of a shard file, for +/// callers that want to (re)compute a manifest hash. +pub fn shard_content_hash(map: &BTreeMap<String, String>) -> String { + let json = serde_json::to_vec(map).unwrap_or_default(); + let mut hasher = Sha256::new(); + hasher.update(&json); + hex_digest(&json) +} + +/// Open a temp file for staging without rewriting an existing one. +#[allow(dead_code)] +fn open_new(path: &Path) -> Result<File, String> { + OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|e| format!("create {}: {e}", path.display())) +} |
