about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorAlexander_Kabui <alexanderkabui@proton.me>2026-10-13 15:11:33 +0300
committerAlexander_Kabui <alexanderkabui@proton.me>2026-10-14 07:27:02 +0300
commita6003ffbfe40aadaa1d9548bdc3294987b97d304 (patch)
treec1bf7ef12eba17ac178090ae48fddb807e23acd7 /src
parenta188b75157516c5e00dd130d9ca8dfbd6cf36679 (diff)
downloadgenenetwork4-a6003ffbfe40aadaa1d9548bdc3294987b97d304.tar.gz
* src/indexer/run.rs (write path): stream documents, not buffer all.
Building and writing each document as the crawl yields it is leaner than
collecting the whole set in a Vec, so live documents stay O(1) and peak
memory stops scaling with the corpus.
* src/indexer/livemem.rs: the live-document counter that proves it.
* src/indexer/build.rs, src/indexer/mod.rs, src/bin/gn-indexer.rs: adapt
to the streamed build and write signatures.
* Cargo.lock: repin xapian-rs 97fd343 -> 70c5eac (the write-API set).
Diffstat (limited to 'src')
-rw-r--r--src/bin/gn-indexer.rs1
-rw-r--r--src/indexer/build.rs31
-rw-r--r--src/indexer/livemem.rs58
-rw-r--r--src/indexer/mod.rs1
-rw-r--r--src/indexer/run.rs381
5 files changed, 368 insertions, 104 deletions
diff --git a/src/bin/gn-indexer.rs b/src/bin/gn-indexer.rs
index 3b70945..d81990e 100644
--- a/src/bin/gn-indexer.rs
+++ b/src/bin/gn-indexer.rs
@@ -81,6 +81,7 @@ fn main() -> ExitCode {
             println!("deleted={}", stats.deleted);
             println!("unchanged={}", stats.unchanged);
             println!("index_doc_count={}", stats.index_doc_count);
+            println!("max_live_documents={}", stats.max_live_documents);
             ExitCode::SUCCESS
         }
         Err(error) => {
diff --git a/src/indexer/build.rs b/src/indexer/build.rs
index e72b0a7..13074f8 100644
--- a/src/indexer/build.rs
+++ b/src/indexer/build.rs
@@ -83,10 +83,33 @@ impl Canon {
     }
 }
 
+/// The stable identity term of a gene row, or `None` when name or
+/// dataset is absent (the crawl then produces no document).
+///
+/// This is the cheap identity pre-pass: it computes exactly the
+/// `name:dataset` term `build_gene` would write, without building a
+/// document, so identity uniqueness can be validated before any write.
+pub fn gene_idterm(row: &GeneRow) -> Option<String> {
+    let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?);
+    Some(schema::identity_term(schema::DOCTYPE_GENE, name, dataset))
+}
+
+/// The stable identity term of a phenotype row, or `None` when name or
+/// dataset is absent. The integer name is stringified exactly as the
+/// document builder does.
+pub fn phenotype_idterm(row: &PhenotypeRow) -> Option<String> {
+    let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?);
+    Some(schema::identity_term(
+        schema::DOCTYPE_PHENOTYPE,
+        &name.to_string(),
+        dataset,
+    ))
+}
+
 /// Build a gene document. `None` when name or dataset is absent (the
 /// indexer then writes no document at all).
 pub fn build_gene(row: &GeneRow, caches: &RdfCaches) -> Option<BuiltDocument> {
-    let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?);
+    let idterm = gene_idterm(row)?;
 
     let mut doc = Document::default();
     {
@@ -184,7 +207,6 @@ pub fn build_gene(row: &GeneRow, caches: &RdfCaches) -> Option<BuiltDocument> {
     push_opt_f64(&mut data, "geno_mb", &row.geno_mb);
     doc.set_data(data.to_string());
 
-    let idterm = schema::identity_term(schema::DOCTYPE_GENE, name, dataset);
     doc.add_boolean_term(schema::type_term(schema::DOCTYPE_GENE));
     doc.add_boolean_term(&idterm);
 
@@ -221,10 +243,10 @@ pub fn build_gene(row: &GeneRow, caches: &RdfCaches) -> Option<BuiltDocument> {
 
 /// Build a phenotype document.
 pub fn build_phenotype(row: &PhenotypeRow) -> Option<BuiltDocument> {
-    let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?);
     // Python converts the integer name to str for the document (and for
     // the identity term's identifier).
-    let name_text = name.to_string();
+    let name_text = row.name.as_ref()?.to_string();
+    let idterm = phenotype_idterm(row)?;
 
     // The raw comma-separated authors string is indexed before storage.
     let authors_raw = row.authors.clone();
@@ -320,7 +342,6 @@ pub fn build_phenotype(row: &PhenotypeRow) -> Option<BuiltDocument> {
     push_opt_f64(&mut data, "geno_mb", &row.geno_mb);
     doc.set_data(data.to_string());
 
-    let idterm = schema::identity_term(schema::DOCTYPE_PHENOTYPE, &name_text, dataset);
     doc.add_boolean_term(schema::type_term(schema::DOCTYPE_PHENOTYPE));
     doc.add_boolean_term(&idterm);
 
diff --git a/src/indexer/livemem.rs b/src/indexer/livemem.rs
new file mode 100644
index 0000000..8d898cf
--- /dev/null
+++ b/src/indexer/livemem.rs
@@ -0,0 +1,58 @@
+//! Live-document accounting for the streaming write path.
+//!
+//! The memory objective of the streaming refactor is that the number
+//! of `Document` objects simultaneously alive in the run orchestrator
+//! is O(1), not O(crawl size). [`CountedDocument`] wraps one built
+//! document; constructing it increments a process-global live counter
+//! and records the high-water mark, dropping it decrements. The peak
+//! is reported on stdout as `max_live_documents` when
+//! `GN4_INDEXER_LIVE_COUNTER=1`.
+//!
+//! The RED control (`GN4_INDEXER_LIVE_RED=retain`) makes the caller
+//! retain every wrapped document instead of releasing it after the
+//! write, so the counter must rise to O(docs) and the flatness
+//! assertion must fail.
+
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+static LIVE: AtomicUsize = AtomicUsize::new(0);
+static PEAK: AtomicUsize = AtomicUsize::new(0);
+
+/// A built document whose lifetime is tracked by the counter.
+pub struct CountedDocument<T> {
+    inner: T,
+}
+
+impl<T> CountedDocument<T> {
+    pub fn new(inner: T) -> Self {
+        let live = LIVE.fetch_add(1, Ordering::SeqCst) + 1;
+        PEAK.fetch_max(live, Ordering::SeqCst);
+        Self { inner }
+    }
+
+    pub fn get(&self) -> &T {
+        &self.inner
+    }
+}
+
+impl<T> Drop for CountedDocument<T> {
+    fn drop(&mut self) {
+        LIVE.fetch_sub(1, Ordering::SeqCst);
+    }
+}
+
+/// Reset the live and peak counters before a run.
+pub fn reset() {
+    LIVE.store(0, Ordering::SeqCst);
+    PEAK.store(0, Ordering::SeqCst);
+}
+
+/// The high-water mark of simultaneously live documents this run.
+pub fn max_live() -> usize {
+    PEAK.load(Ordering::SeqCst)
+}
+
+/// Whether the run should retain every document (the RED control).
+pub fn red_retain() -> bool {
+    std::env::var("GN4_INDEXER_LIVE_RED").as_deref() == Ok("retain")
+}
diff --git a/src/indexer/mod.rs b/src/indexer/mod.rs
index 7486feb..7972b54 100644
--- a/src/indexer/mod.rs
+++ b/src/indexer/mod.rs
@@ -22,6 +22,7 @@
 
 pub mod build;
 pub mod cache;
+pub mod livemem;
 pub mod pyjson;
 pub mod rows;
 pub mod run;
diff --git a/src/indexer/run.rs b/src/indexer/run.rs
index 5d26981..6ca9a17 100644
--- a/src/indexer/run.rs
+++ b/src/indexer/run.rs
@@ -1,14 +1,24 @@
 //! Run orchestration: FULL (bootstrap) and INCREMENTAL (per-document
 //! in-place update) over one mechanism — build documents from a crawl,
 //! then write them into the index by identity term.
+//!
+//! The write path is STREAMING: a cheap identity pre-pass validates
+//! `name:dataset` uniqueness before any index is opened, then the crawl
+//! is replayed once, building one document at a time and writing it
+//! immediately, so the run never holds more than O(1) live `Document`
+//! objects. FULL writes into a fresh temp directory and only renames it
+//! into place on full success, so an error cannot destroy an existing
+//! index; INCREMENTAL opens its write transaction lazily, so a
+//! zero-change run writes nothing.
 
 use std::collections::{BTreeMap, BTreeSet};
 use std::path::{Path, PathBuf};
 
 use xapian_rs::{Database, DbAction, DbBackend, Enquire, Query, WritableDatabase};
 
-use super::build::{BuiltDocument, build_gene, build_phenotype};
+use super::build::{BuiltDocument, build_gene, build_phenotype, gene_idterm, phenotype_idterm};
 use super::cache::RdfCaches;
+use super::livemem::{self, CountedDocument};
 use super::rows::{self, GeneRow, PhenotypeRow};
 use super::schema;
 use super::sidecar::Sidecar;
@@ -23,6 +33,8 @@ pub struct Stats {
     pub unchanged: usize,
     pub index_doc_count: u32,
     pub duplicate_identity_terms: Vec<String>,
+    /// High-water mark of simultaneously live built documents.
+    pub max_live_documents: usize,
 }
 
 /// Inputs for one run.
@@ -37,60 +49,133 @@ pub struct RunConfig {
 
 /// Test-only sabotage hooks for the RED controls (gate 5). Never set in
 /// a normal run; each value deliberately breaks one guarantee so the
-/// gate can be shown to fail.
+/// gate can be shown to fail. `duplicate` skips the identity-uniqueness
+/// error (last-writer-wins, the regression the pre-pass closes); `hash`
+/// perturbs every source hash; `delete` drops the delete pass;
+/// `commit` cancels instead of committing.
 fn red_mode() -> Option<String> {
     std::env::var("GN4_INDEXER_RED").ok()
 }
 
-/// Build every document for a crawl, asserting identity-term
-/// uniqueness (a duplicate `name:dataset` pair is a reported error, not
-/// last-writer-wins).
-fn build_all(
+/// The sibling temp directory a FULL build writes before its atomic
+/// swap into place.
+fn temp_index_path(index_path: &Path) -> PathBuf {
+    let parent = index_path.parent().unwrap_or_else(|| Path::new("."));
+    let name = index_path
+        .file_name()
+        .map(|n| n.to_string_lossy().to_string())
+        .unwrap_or_else(|| "index".to_string());
+    parent.join(format!(".{name}.tmp.{}", std::process::id()))
+}
+
+/// Atomically replace `index_path` with the freshly built `temp` index
+/// directory. The pre-existing index is moved aside first and restored
+/// if the swap fails, so it is never left destroyed by a failed build.
+fn swap_into_place(temp: &Path, index_path: &Path) -> Result<(), String> {
+    let parent = index_path.parent().unwrap_or_else(|| Path::new("."));
+    let backup = parent.join(format!(
+        ".{}.old.{}",
+        index_path
+            .file_name()
+            .map(|n| n.to_string_lossy().to_string())
+            .unwrap_or_else(|| "index".to_string()),
+        std::process::id()
+    ));
+    let had_existing = index_path.exists();
+    if had_existing {
+        std::fs::rename(index_path, &backup)
+            .map_err(|e| format!("index swap (move aside): {e}"))?;
+    }
+    match std::fs::rename(temp, index_path) {
+        Ok(()) => {
+            if had_existing {
+                let _ = std::fs::remove_dir_all(&backup);
+            }
+            Ok(())
+        }
+        Err(e) => {
+            if had_existing {
+                let _ = std::fs::rename(&backup, index_path);
+            }
+            Err(format!("index swap (rename into place): {e}"))
+        }
+    }
+}
+
+/// The cheap identity pre-pass: enumerate every row's identity term
+/// WITHOUT building documents, flag duplicate `name:dataset` pairs, and
+/// return the full idterm set. Duplicates are an error to report, not
+/// last-writer-wins.
+fn identity_prepass(
     genes: &[GeneRow],
     phenotypes: &[PhenotypeRow],
-    caches: &RdfCaches,
-    red_hash: bool,
-) -> Result<(Vec<BuiltDocument>, Vec<String>), String> {
-    let mut built = Vec::with_capacity(genes.len() + phenotypes.len());
+) -> (BTreeSet<String>, Vec<String>) {
     let mut seen: BTreeMap<String, usize> = BTreeMap::new();
     let mut duplicates = Vec::new();
+    let mut idterms = BTreeSet::new();
+    let mut position = 0usize;
     for row in genes {
-        if let Some(mut document) = build_gene(row, caches) {
-            if let Some(nonce) = red_nonce(red_hash) {
-                document.source_hash.push_str(&nonce);
-            }
-            record(&mut seen, &mut duplicates, &mut built, document);
+        if let Some(idterm) = gene_idterm(row) {
+            record_idterm(&mut seen, &mut duplicates, &mut idterms, idterm, position);
+            position += 1;
         }
     }
     for row in phenotypes {
-        if let Some(mut document) = build_phenotype(row) {
-            if let Some(nonce) = red_nonce(red_hash) {
-                document.source_hash.push_str(&nonce);
-            }
-            record(&mut seen, &mut duplicates, &mut built, document);
+        if let Some(idterm) = phenotype_idterm(row) {
+            record_idterm(&mut seen, &mut duplicates, &mut idterms, idterm, position);
+            position += 1;
         }
     }
-    Ok((built, duplicates))
+    (idterms, duplicates)
 }
 
-/// Record one built document, flagging a duplicate identity term
-/// instead of silently overwriting.
-fn record(
+fn record_idterm(
     seen: &mut BTreeMap<String, usize>,
     duplicates: &mut Vec<String>,
-    built: &mut Vec<BuiltDocument>,
-    document: BuiltDocument,
+    idterms: &mut BTreeSet<String>,
+    idterm: String,
+    position: usize,
 ) {
-    if let Some(previous) = seen.get(&document.idterm) {
+    if let Some(previous) = seen.get(&idterm) {
         duplicates.push(format!(
-            "{} (first at position {previous}, duplicate at {})",
-            document.idterm,
-            built.len()
+            "{idterm} (first at position {previous}, duplicate at {position})"
         ));
     } else {
-        seen.insert(document.idterm.clone(), built.len());
+        seen.insert(idterm.clone(), position);
+        idterms.insert(idterm);
+    }
+}
+
+/// Replay the crawl once, building one document at a time and handing
+/// it to `emit`. Documents are produced in the same order (genes then
+/// phenotypes) the batch builder used.
+fn for_each_document<F>(
+    genes: &[GeneRow],
+    phenotypes: &[PhenotypeRow],
+    caches: &RdfCaches,
+    red_hash: bool,
+    mut emit: F,
+) -> Result<(), String>
+where
+    F: FnMut(BuiltDocument) -> Result<(), String>,
+{
+    for row in genes {
+        if let Some(mut document) = build_gene(row, caches) {
+            if let Some(nonce) = red_nonce(red_hash) {
+                document.source_hash.push_str(&nonce);
+            }
+            emit(document)?;
+        }
+    }
+    for row in phenotypes {
+        if let Some(mut document) = build_phenotype(row) {
+            if let Some(nonce) = red_nonce(red_hash) {
+                document.source_hash.push_str(&nonce);
+            }
+            emit(document)?;
+        }
     }
-    built.push(document);
+    Ok(())
 }
 
 fn red_nonce(red_hash: bool) -> Option<String> {
@@ -125,58 +210,117 @@ fn index_identity_terms(db: &WritableDatabase) -> BTreeSet<String> {
     terms
 }
 
-/// Run a FULL (bootstrap) build: a fresh index written document by
-/// document through `replace_document_by_term`.
+/// Run a FULL (bootstrap) build: a fresh temp index written document by
+/// document through `replace_document_by_term`, swapped into place only
+/// on full success.
 pub fn run_full(config: &RunConfig) -> Result<Stats, String> {
     let caches = RdfCaches::load(&config.rdf_cache)?;
     let genes = rows::crawl_genes(&config.sql_uri)?;
     let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?;
     let red = red_mode();
-    let (built, duplicates) =
-        build_all(&genes, &phenotypes, &caches, red.as_deref() == Some("hash"))?;
 
-    let mut stats = Stats {
-        mode: "full".to_string(),
-        built: built.len(),
-        replaced: built.len(),
-        duplicate_identity_terms: duplicates,
-        ..Default::default()
-    };
-    if !stats.duplicate_identity_terms.is_empty() {
+    // Identity validation happens BEFORE any index is opened or
+    // written, so a duplicate crawl cannot touch an existing index.
+    let (_idterms, duplicates) = identity_prepass(&genes, &phenotypes);
+    if !duplicates.is_empty() && red.as_deref() != Some("duplicate") {
         return Err(format!(
             "duplicate identity terms in crawl: {}",
-            stats.duplicate_identity_terms.join(", ")
+            duplicates.join(", ")
         ));
     }
 
-    let mut db = WritableDatabase::open(
-        &config.index_path,
-        Some(DbAction::CreateOrOverwrite),
-        Some(DbBackend::Glass),
-        None,
-        None,
+    livemem::reset();
+    let retain = livemem::red_retain();
+    let temp = temp_index_path(&config.index_path);
+    let outcome = stream_full(
+        &temp,
+        &genes,
+        &phenotypes,
+        &caches,
+        red.as_deref() == Some("hash"),
+        retain,
     );
-    db.begin_transaction(None);
-    for document in &built {
-        let _ = db.replace_document_by_term(&document.idterm, &document.document);
+    let (index_doc_count, hashes, built) = match outcome {
+        Ok(value) => value,
+        Err(error) => {
+            let _ = std::fs::remove_dir_all(&temp);
+            return Err(error);
+        }
+    };
+    if let Err(error) = swap_into_place(&temp, &config.index_path) {
+        let _ = std::fs::remove_dir_all(&temp);
+        return Err(error);
     }
-    commit(&mut db, red.as_deref() == Some("commit"))?;
-    stats.index_doc_count = db.doc_count();
 
-    // Sidecar strictly after the index commit.
+    // Sidecar strictly after the index commit (and the swap).
     let sidecar = Sidecar {
         version: 1,
         mode: "full".to_string(),
-        doc_count: built.len(),
+        doc_count: built,
         rif_hash: caches.rif.hash.clone(),
         wiki_hash: caches.wiki.hash.clone(),
-        documents: built
-            .iter()
-            .map(|d| (d.idterm.clone(), d.source_hash.clone()))
-            .collect(),
+        documents: hashes,
     };
     sidecar.store(&config.index_path)?;
-    Ok(stats)
+
+    Ok(Stats {
+        mode: "full".to_string(),
+        built,
+        replaced: built,
+        index_doc_count,
+        duplicate_identity_terms: Vec::new(),
+        max_live_documents: livemem::max_live(),
+        ..Default::default()
+    })
+}
+
+/// Stream a fresh FULL build into `temp`: open once, one transaction,
+/// write each document as it is built. Returns the committed document
+/// count, the idterm→source-hash map for the sidecar, and the built
+/// count.
+fn stream_full(
+    temp: &Path,
+    genes: &[GeneRow],
+    phenotypes: &[PhenotypeRow],
+    caches: &RdfCaches,
+    red_hash: bool,
+    retain: bool,
+) -> Result<(u32, BTreeMap<String, String>, usize), String> {
+    let mut db = WritableDatabase::open(
+        temp,
+        Some(DbAction::CreateOrOverwrite),
+        Some(DbBackend::Glass),
+        None,
+        None,
+    );
+    db.begin_transaction(None);
+
+    let mut hashes: BTreeMap<String, String> = BTreeMap::new();
+    let mut kept: Vec<CountedDocument<BuiltDocument>> = Vec::new();
+    let mut count = 0usize;
+    {
+        let emit = |document: BuiltDocument| -> Result<(), String> {
+            let counted = CountedDocument::new(document);
+            let idterm = counted.get().idterm.clone();
+            hashes.insert(idterm.clone(), counted.get().source_hash.clone());
+            let _ = db.replace_document_by_term(&idterm, &counted.get().document);
+            count += 1;
+            if retain {
+                kept.push(counted);
+            }
+            Ok(())
+        };
+        if let Err(error) = for_each_document(genes, phenotypes, caches, red_hash, emit) {
+            db.cancel_transaction();
+            return Err(error);
+        }
+    }
+
+    commit(&mut db, red_mode().as_deref() == Some("commit"))?;
+    let doc_count = db.doc_count();
+    drop(db);
+    drop(kept);
+    Ok((doc_count, hashes, count))
 }
 
 /// Run an INCREMENTAL pass: crawl, hash, diff against the sidecar,
@@ -195,15 +339,19 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> {
     let genes = rows::crawl_genes(&config.sql_uri)?;
     let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?;
     let red = red_mode();
-    let (built, duplicates) =
-        build_all(&genes, &phenotypes, &caches, red.as_deref() == Some("hash"))?;
-    if !duplicates.is_empty() {
+
+    // Identity pre-pass, before any write.
+    let (idterms, duplicates) = identity_prepass(&genes, &phenotypes);
+    if !duplicates.is_empty() && red.as_deref() != Some("duplicate") {
         return Err(format!(
             "duplicate identity terms in crawl: {}",
             duplicates.join(", ")
         ));
     }
 
+    livemem::reset();
+    let retain = livemem::red_retain();
+
     let mut db = WritableDatabase::open(
         &config.index_path,
         Some(DbAction::CreateOrOpen),
@@ -220,56 +368,93 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> {
     let sidecar_terms: BTreeSet<String> = previous.documents.keys().cloned().collect();
     let mismatched = index_terms != sidecar_terms;
 
-    let built_by_term: BTreeMap<&str, &BuiltDocument> =
-        built.iter().map(|d| (d.idterm.as_str(), d)).collect();
-
-    let mut to_write: Vec<&BuiltDocument> = Vec::new();
+    let mut hashes: BTreeMap<String, String> = BTreeMap::new();
     let mut unchanged = 0usize;
-    for document in &built {
-        let known = if mismatched {
-            None
-        } else {
-            previous.documents.get(&document.idterm)
+    let mut replaced = 0usize;
+    let mut open_transaction = false;
+    let mut kept: Vec<CountedDocument<BuiltDocument>> = Vec::new();
+    {
+        let previous_documents = &previous.documents;
+        let emit = |document: BuiltDocument| -> Result<(), String> {
+            let counted = CountedDocument::new(document);
+            let doc = counted.get();
+            hashes.insert(doc.idterm.clone(), doc.source_hash.clone());
+            let known = if mismatched {
+                None
+            } else {
+                previous_documents.get(&doc.idterm)
+            };
+            match known {
+                Some(hash) if *hash == doc.source_hash => unchanged += 1,
+                _ => {
+                    if !open_transaction {
+                        db.begin_transaction(None);
+                        open_transaction = true;
+                    }
+                    let _ = db.replace_document_by_term(&doc.idterm, &doc.document);
+                    replaced += 1;
+                }
+            }
+            if retain {
+                kept.push(counted);
+            }
+            Ok(())
         };
-        match known {
-            Some(hash) if *hash == document.source_hash => unchanged += 1,
-            _ => to_write.push(document),
+        if let Err(error) = for_each_document(
+            &genes,
+            &phenotypes,
+            &caches,
+            red.as_deref() == Some("hash"),
+            emit,
+        ) {
+            if open_transaction {
+                db.cancel_transaction();
+            }
+            return Err(error);
         }
     }
 
     // Existence truth: the sidecar's key set (or, on mismatch, the
     // index's own terms). Anything there but not in the crawl is gone.
+    // Deletes need the full crawl, so they are applied at the end.
     let mut to_delete: BTreeSet<String> = if mismatched {
-        index_terms.clone()
+        index_terms
     } else {
-        sidecar_terms.clone()
+        sidecar_terms
     };
     if red.as_deref() == Some("delete") {
         to_delete.clear();
     } else {
-        for term in built_by_term.keys() {
-            to_delete.remove(*term);
+        for term in &idterms {
+            to_delete.remove(term);
+        }
+    }
+
+    let mut deleted = 0usize;
+    if !to_delete.is_empty() {
+        if !open_transaction {
+            db.begin_transaction(None);
+            open_transaction = true;
+        }
+        for term in &to_delete {
+            db.delete_document_by_term(term);
+            deleted += 1;
         }
     }
 
     let mut stats = Stats {
         mode: "incremental".to_string(),
-        built: built.len(),
+        built: hashes.len(),
         unchanged,
+        replaced,
+        deleted,
+        max_live_documents: livemem::max_live(),
         ..Default::default()
     };
 
-    let nothing_changed = to_write.is_empty() && to_delete.is_empty();
-    if !nothing_changed {
-        db.begin_transaction(None);
-        for document in &to_write {
-            let _ = db.replace_document_by_term(&document.idterm, &document.document);
-            stats.replaced += 1;
-        }
-        for term in &to_delete {
-            db.delete_document_by_term(term);
-            stats.deleted += 1;
-        }
+    let nothing_changed = replaced == 0 && deleted == 0;
+    if open_transaction {
+        // A zero-change run opens no transaction and writes nothing.
         commit(&mut db, red.as_deref() == Some("commit"))?;
     }
     stats.index_doc_count = db.doc_count();
@@ -280,16 +465,14 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> {
         let sidecar = Sidecar {
             version: 1,
             mode: "incremental".to_string(),
-            doc_count: built.len(),
+            doc_count: hashes.len(),
             rif_hash: caches.rif.hash.clone(),
             wiki_hash: caches.wiki.hash.clone(),
-            documents: built
-                .iter()
-                .map(|d| (d.idterm.clone(), d.source_hash.clone()))
-                .collect(),
+            documents: hashes,
         };
         sidecar.store(&config.index_path)?;
     }
+    let _ = kept;
     Ok(stats)
 }