about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorAlexander_Kabui <alexanderkabui@proton.me>2026-10-13 15:11:33 +0300
committerAlexander_Kabui <alexanderkabui@proton.me>2026-10-14 07:27:03 +0300
commit53c3f26ec2fdf209876a8a8eab561acc6fd755d4 (patch)
tree25fc2be0d09e37da759be90fc3590f1496afaa89 /src
parenta6003ffbfe40aadaa1d9548bdc3294987b97d304 (diff)
downloadgenenetwork4-53c3f26ec2fdf209876a8a8eab561acc6fd755d4.tar.gz
* src/indexer/sidecar.rs (sharding): one sidecar shard per dataset.
A monolithic sidecar rewrote and re-read the whole identity map each run;
sharding it by dataset under <index>.sidecar.json plus <index>.sidecar.d/
lets a run load and rewrite only the shards it touches and keeps resident
memory at O(largest dataset).
* src/indexer/run.rs: crawl the SQL sources one dataset at a time through a
sink and commit every 5000 documents instead of buffering the whole crawl;
a bounded-channel variant was tried and reverted (several times slower).
* src/indexer/rows.rs: order and group the crawl streams by dataset.
* src/bin/gn-indexer.rs: follow the sharded sink wiring.
Diffstat (limited to 'src')
-rw-r--r--src/bin/gn-indexer.rs15
-rw-r--r--src/indexer/rows.rs284
-rw-r--r--src/indexer/run.rs1103
-rw-r--r--src/indexer/sidecar.rs446
4 files changed, 1477 insertions, 371 deletions
diff --git a/src/bin/gn-indexer.rs b/src/bin/gn-indexer.rs
index d81990e..224edb8 100644
--- a/src/bin/gn-indexer.rs
+++ b/src/bin/gn-indexer.rs
@@ -82,6 +82,21 @@ fn main() -> ExitCode {
             println!("unchanged={}", stats.unchanged);
             println!("index_doc_count={}", stats.index_doc_count);
             println!("max_live_documents={}", stats.max_live_documents);
+            println!("shards_written={}", stats.shards_written);
+            println!("shards_deleted={}", stats.shards_deleted);
+            println!(
+                "max_resident_sidecar_entries={}",
+                stats.max_resident_sidecar_entries
+            );
+            println!(
+                "max_resident_dataset_terms={}",
+                stats.max_resident_dataset_terms
+            );
+            println!("peak_open_shards={}", stats.peak_open_shards);
+            println!("shard_bytes_read={}", stats.shard_bytes_read);
+            println!("commit_chunks={}", stats.commit_chunks);
+            println!("chunk_rss_peak_kib={}", stats.chunk_rss_peak_kib);
+            println!("startup={}", stats.startup);
             ExitCode::SUCCESS
         }
         Err(error) => {
diff --git a/src/indexer/rows.rs b/src/indexer/rows.rs
index b8669e1..69b94cc 100644
--- a/src/indexer/rows.rs
+++ b/src/indexer/rows.rs
@@ -1,5 +1,5 @@
 //! The crawl: the SQL queries and typed row structs the document
-//! builder consumes.
+//! builder consumes, streamed and grouped by dataset.
 //!
 //! The queries and joins are the indexer's own (`genes_query` and
 //! `phenotypes_query` in index-genenetwork), rendered by its
@@ -12,6 +12,18 @@
 //! PublishFreezes (e.g. BXDPublish and GITrMetPublish) each index every
 //! trait under their own dataset label. This cross-product is INTENTIONAL
 //! reference semantics and is reproduced, not "cleaned up".
+//!
+//! # Streaming, dataset-grouped crawl
+//!
+//! [`stream_dataset_groups`] never materialises the crawl. Each query is
+//! executed in its own producer thread with `ORDER BY dataset`; the merge
+//! consumes the two sorted streams and hands the caller **one dataset at
+//! a time** through a [`DatasetSink`], whose `begin_dataset` / `row` /
+//! `end_dataset` callbacks see rows one at a time. The only per-dataset
+//! state the caller keeps is bounded by the largest dataset, not by the
+//! whole crawl.
+
+use std::sync::mpsc::{Receiver, Sender};
 
 use mysql::prelude::Queryable;
 use mysql::{Conn, Opts, OptsBuilder, Row};
@@ -74,6 +86,34 @@ pub struct PhenotypeRow {
     pub geno_mb: Option<f64>,
 }
 
+/// One crawled row, tagged with its document type. The dataset label is
+/// the row's dataset column — never parsed back out of the identity
+/// term, whose `name` part may itself contain a colon.
+pub enum CrawlRow {
+    Gene(GeneRow),
+    Phenotype(PhenotypeRow),
+}
+
+impl CrawlRow {
+    /// The row's dataset label (the shard key).
+    pub fn dataset(&self) -> Option<&str> {
+        match self {
+            CrawlRow::Gene(row) => row.dataset.as_deref(),
+            CrawlRow::Phenotype(row) => row.dataset.as_deref(),
+        }
+    }
+}
+
+/// Receive one dataset group at a time from [`stream_dataset_groups`].
+pub trait DatasetSink {
+    /// Called once when a dataset group starts, before any of its rows.
+    fn begin_dataset(&mut self, label: &str) -> Result<(), String>;
+    /// Called for each row of the current dataset, in arrival order.
+    fn row(&mut self, row: CrawlRow) -> Result<(), String>;
+    /// Called once after the last row of the dataset.
+    fn end_dataset(&mut self, label: &str) -> Result<(), String>;
+}
+
 fn connect(sql_uri: &str) -> Result<Conn, String> {
     let opts = Opts::from_url(sql_uri).map_err(|e| format!("bad SQL_URI: {e}"))?;
     Conn::new(OptsBuilder::from_opts(opts)).map_err(|e| format!("cannot connect: {e}"))
@@ -122,7 +162,8 @@ fn genes_sql() -> String {
          INNER JOIN ProbeSetXRef ON ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id \
          INNER JOIN ProbeSet ON ProbeSet.Id = ProbeSetXRef.ProbeSetId \
          LEFT JOIN Geno ON ProbeSetXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id \
-         WHERE {GENE_PUBLIC_FILTER}"
+         WHERE {GENE_PUBLIC_FILTER} \
+         ORDER BY dataset"
     )
 }
 
@@ -155,75 +196,194 @@ fn phenotypes_sql(filtered: bool) -> String {
          INNER JOIN Phenotype ON PublishXRef.PhenotypeId = Phenotype.Id \
          INNER JOIN Publication ON PublishXRef.PublicationId = Publication.Id \
          LEFT JOIN Geno ON PublishXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id\
-         {where_clause}"
+         {where_clause} \
+         ORDER BY dataset"
     )
 }
 
-/// Crawl every public, non-confidential expression record.
-pub fn crawl_genes(sql_uri: &str) -> Result<Vec<GeneRow>, String> {
+type Item = Result<(String, CrawlRow), String>;
+
+/// Producer thread for the gene crawl: stream rows ordered by dataset and
+/// send each as `(dataset, CrawlRow)`. Rows with no dataset produce no
+/// document and are skipped.
+fn produce_genes(sql_uri: &str, tx: &Sender<Item>) -> Result<(), String> {
     let mut conn = connect(sql_uri)?;
-    let rows: Vec<Row> = conn
-        .query(genes_sql())
+    let result = conn
+        .query_iter(genes_sql())
         .map_err(|e| format!("gene crawl failed: {e}"))?;
-    let mut out = Vec::with_capacity(rows.len());
-    for row in &rows {
-        out.push(GeneRow {
-            name: text(row, "name")?,
-            symbol: text(row, "symbol")?,
-            description: text(row, "description")?,
-            chr: text(row, "chr")?,
-            mb: float(row, "mb")?,
-            alias: text(row, "alias")?,
-            genbankid: text(row, "genbankid")?,
-            unigeneid: text(row, "unigeneid")?,
-            probe_target_description: text(row, "probe_target_description")?,
-            dataset: text(row, "dataset")?,
-            dataset_fullname: text(row, "dataset_fullname")?,
-            species: text(row, "species")?,
-            group: text(row, "group")?,
-            tissue: text(row, "tissue")?,
-            mean: float(row, "mean")?,
-            lrs: float(row, "lrs")?,
-            additive: float(row, "additive")?,
-            geno_chr: text(row, "geno_chr")?,
-            geno_mb: float(row, "geno_mb")?,
-        });
+    for row in result {
+        let row = row.map_err(|e| format!("gene crawl failed: {e}"))?;
+        let Some(dataset) = text(&row, "dataset")? else {
+            continue;
+        };
+        let row = GeneRow {
+            name: text(&row, "name")?,
+            symbol: text(&row, "symbol")?,
+            description: text(&row, "description")?,
+            chr: text(&row, "chr")?,
+            mb: float(&row, "mb")?,
+            alias: text(&row, "alias")?,
+            genbankid: text(&row, "genbankid")?,
+            unigeneid: text(&row, "unigeneid")?,
+            probe_target_description: text(&row, "probe_target_description")?,
+            dataset: Some(dataset.clone()),
+            dataset_fullname: text(&row, "dataset_fullname")?,
+            species: text(&row, "species")?,
+            group: text(&row, "group")?,
+            tissue: text(&row, "tissue")?,
+            mean: float(&row, "mean")?,
+            lrs: float(&row, "lrs")?,
+            additive: float(&row, "additive")?,
+            geno_chr: text(&row, "geno_chr")?,
+            geno_mb: float(&row, "geno_mb")?,
+        };
+        if tx.send(Ok((dataset, CrawlRow::Gene(row)))).is_err() {
+            return Ok(());
+        }
     }
-    Ok(out)
+    Ok(())
 }
 
-/// Crawl published phenotype records. `filtered=false` drops the
-/// public/confidentiality predicate: that is the leak control, used
-/// only by the gate-4 RED run.
-pub fn crawl_phenotypes(sql_uri: &str, filtered: bool) -> Result<Vec<PhenotypeRow>, String> {
+/// Producer thread for the phenotype crawl.
+fn produce_phenotypes(sql_uri: &str, filtered: bool, tx: &Sender<Item>) -> Result<(), String> {
     let mut conn = connect(sql_uri)?;
-    let rows: Vec<Row> = conn
-        .query(phenotypes_sql(filtered))
+    let result = conn
+        .query_iter(phenotypes_sql(filtered))
         .map_err(|e| format!("phenotype crawl failed: {e}"))?;
-    let mut out = Vec::with_capacity(rows.len());
-    for row in &rows {
-        out.push(PhenotypeRow {
-            species: text(row, "species")?,
-            group: text(row, "group")?,
-            dataset: text(row, "dataset")?,
-            dataset_fullname: text(row, "dataset_fullname")?,
-            name: integer(row, "name")?,
-            abbreviation: text(row, "abbreviation")?,
-            description: text(row, "description")?,
-            lab_code: text(row, "Lab_code")?,
-            abstract_: text(row, "Abstract")?,
-            title: text(row, "Title")?,
-            authors: text(row, "authors")?,
-            year: integer(row, "year")?,
-            pubmed_id: integer(row, "pubmed_id")?,
-            lrs: float(row, "lrs")?,
-            additive: float(row, "additive")?,
-            inbredsetcode: text(row, "inbredsetcode")?,
-            mean: float(row, "mean")?,
-            nsamples: integer(row, "nsamples")?,
-            geno_chr: text(row, "geno_chr")?,
-            geno_mb: float(row, "geno_mb")?,
-        });
+    for row in result {
+        let row = row.map_err(|e| format!("phenotype crawl failed: {e}"))?;
+        let Some(dataset) = text(&row, "dataset")? else {
+            continue;
+        };
+        let row = PhenotypeRow {
+            species: text(&row, "species")?,
+            group: text(&row, "group")?,
+            dataset: Some(dataset.clone()),
+            dataset_fullname: text(&row, "dataset_fullname")?,
+            name: integer(&row, "name")?,
+            abbreviation: text(&row, "abbreviation")?,
+            description: text(&row, "description")?,
+            lab_code: text(&row, "Lab_code")?,
+            abstract_: text(&row, "Abstract")?,
+            title: text(&row, "Title")?,
+            authors: text(&row, "authors")?,
+            year: integer(&row, "year")?,
+            pubmed_id: integer(&row, "pubmed_id")?,
+            lrs: float(&row, "lrs")?,
+            additive: float(&row, "additive")?,
+            inbredsetcode: text(&row, "inbredsetcode")?,
+            mean: float(&row, "mean")?,
+            nsamples: integer(&row, "nsamples")?,
+            geno_chr: text(&row, "geno_chr")?,
+            geno_mb: float(&row, "geno_mb")?,
+        };
+        if tx.send(Ok((dataset, CrawlRow::Phenotype(row)))).is_err() {
+            return Ok(());
+        }
+    }
+    Ok(())
+}
+
+fn spawn_producer<F>(tx: Sender<Item>, body: F) -> std::thread::JoinHandle<()>
+where
+    F: FnOnce(&Sender<Item>) -> Result<(), String> + Send + 'static,
+{
+    std::thread::spawn(move || {
+        if let Err(error) = body(&tx) {
+            let _ = tx.send(Err(error));
+        }
+    })
+}
+
+/// The dataset label at the head of a channel, propagating a producer
+/// error. `Ok(None)` means the channel is exhausted.
+fn head_label(head: &Option<Item>) -> Result<Option<&str>, String> {
+    match head {
+        None => Ok(None),
+        Some(Ok((label, _))) => Ok(Some(label.as_str())),
+        Some(Err(error)) => Err(error.clone()),
+    }
+}
+
+/// Stream the crawl as dataset groups. Two producer threads (genes and
+/// phenotypes) each return rows ordered by dataset; this merge consumes
+/// them in dataset order and invokes `sink` once per dataset. No whole-
+/// crawl buffer exists: at most one pending row per source is held.
+pub fn stream_dataset_groups(
+    sql_uri: &str,
+    filtered: bool,
+    sink: &mut dyn DatasetSink,
+) -> Result<(), String> {
+    // Unbounded channels: the producers stream the two SQL result sets in
+    // their own threads and the merge consumes them; this is the fast
+    // path (a bounded channel was tried and made the scaled run several
+    // times slower, so it was reverted).
+    let (gene_tx, gene_rx) = std::sync::mpsc::channel::<Item>();
+    let (phen_tx, phen_rx) = std::sync::mpsc::channel::<Item>();
+    let gene_uri = sql_uri.to_string();
+    let phen_uri = sql_uri.to_string();
+    let genes = spawn_producer(gene_tx, move |tx| produce_genes(&gene_uri, tx));
+    let phenotypes = spawn_producer(phen_tx, move |tx| {
+        produce_phenotypes(&phen_uri, filtered, tx)
+    });
+
+    let outcome = merge_streams(&gene_rx, &phen_rx, sink);
+
+    // Producers end when the receivers drop, but join for a clean exit.
+    let _ = genes.join();
+    let _ = phenotypes.join();
+    outcome
+}
+
+fn merge_streams(
+    gene_rx: &Receiver<Item>,
+    phen_rx: &Receiver<Item>,
+    sink: &mut dyn DatasetSink,
+) -> Result<(), String> {
+    let mut gene_head = gene_rx.recv().ok();
+    let mut phen_head = phen_rx.recv().ok();
+    loop {
+        let gene_label = head_label(&gene_head)?;
+        let phen_label = head_label(&phen_head)?;
+        let label = match (gene_label, phen_label) {
+            (None, None) => break,
+            (Some(label), None) => label.to_string(),
+            (None, Some(label)) => label.to_string(),
+            (Some(gene), Some(phen)) => {
+                if gene <= phen {
+                    gene.to_string()
+                } else {
+                    phen.to_string()
+                }
+            }
+        };
+        sink.begin_dataset(&label)?;
+        drain_label(&mut gene_head, gene_rx, &label, sink)?;
+        drain_label(&mut phen_head, phen_rx, &label, sink)?;
+        sink.end_dataset(&label)?;
+    }
+    Ok(())
+}
+
+/// Consume every head row whose dataset equals `label`, advancing the
+/// channel. Rows for later datasets stay as the new head.
+fn drain_label(
+    head: &mut Option<Item>,
+    rx: &Receiver<Item>,
+    label: &str,
+    sink: &mut dyn DatasetSink,
+) -> Result<(), String> {
+    loop {
+        match head {
+            Some(Ok((row_label, _))) if row_label == label => {
+                let Some(Ok((_, row))) = head.take() else {
+                    unreachable!("head was checked Some(Ok)")
+                };
+                sink.row(row)?;
+                *head = rx.recv().ok();
+            }
+            Some(Err(error)) => return Err(error.clone()),
+            _ => return Ok(()),
+        }
     }
-    Ok(out)
 }
diff --git a/src/indexer/run.rs b/src/indexer/run.rs
index 6ca9a17..b3248f5 100644
--- a/src/indexer/run.rs
+++ b/src/indexer/run.rs
@@ -1,17 +1,32 @@
-//! Run orchestration: FULL (bootstrap) and INCREMENTAL (per-document
-//! in-place update) over one mechanism — build documents from a crawl,
-//! then write them into the index by identity term.
+//! Run orchestration: FULL (bootstrap) and INCREMENTAL (per-dataset
+//! in-place update) over one mechanism — stream the crawl one dataset at
+//! a time, diff each dataset against its shard, and write changed
+//! documents by identity term.
 //!
-//! The write path is STREAMING: a cheap identity pre-pass validates
-//! `name:dataset` uniqueness before any index is opened, then the crawl
-//! is replayed once, building one document at a time and writing it
-//! immediately, so the run never holds more than O(1) live `Document`
-//! objects. FULL writes into a fresh temp directory and only renames it
-//! into place on full success, so an error cannot destroy an existing
-//! index; INCREMENTAL opens its write transaction lazily, so a
-//! zero-change run writes nothing.
-
-use std::collections::{BTreeMap, BTreeSet};
+//! Design of this step (Step 2):
+//!
+//! * **Streaming, dataset-grouped crawl.** [`rows::stream_dataset_groups`]
+//!   runs the two SQL crawls in producer threads ordered by dataset and
+//!   hands the run one dataset at a time; no whole-crawl buffer exists.
+//! * **Sharded sidecar.** Each dataset has one shard (see [`sidecar`]);
+//!   the diff loads only the current dataset's shard, so the resident
+//!   idterm->hash map is O(largest shard), not O(documents).
+//! * **Cheap-first startup.** The steady-state check compares the
+//!   manifest `doc_count` to `db.doc_count()` (O(1)) and recomputes every
+//!   shard's `content_sha256` from its file; only on mismatch is the
+//!   expensive identity-term enumeration performed (and the run falls
+//!   back to FULL). `index_identity_terms` is therefore off the
+//!   steady-state path.
+//! * **Bounded commits.** Writes are committed every [`COMMIT_CHUNK_DOCS`]
+//!   documents with a per-chunk RSS probe; FULL writes a temp index and
+//!   swaps it in only on success.
+//!
+//! Duplicate-identity detection is a separate streaming pass that keeps
+//! only the current dataset's identity set (O(largest dataset)); it runs
+//! to completion before any index is opened, so a duplicate crawl can
+//! never touch an existing index or sidecar.
+
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::path::{Path, PathBuf};
 
 use xapian_rs::{Database, DbAction, DbBackend, Enquire, Query, WritableDatabase};
@@ -19,9 +34,15 @@ use xapian_rs::{Database, DbAction, DbBackend, Enquire, Query, WritableDatabase}
 use super::build::{BuiltDocument, build_gene, build_phenotype, gene_idterm, phenotype_idterm};
 use super::cache::RdfCaches;
 use super::livemem::{self, CountedDocument};
-use super::rows::{self, GeneRow, PhenotypeRow};
+use super::rows::{self, CrawlRow, DatasetSink};
 use super::schema;
-use super::sidecar::Sidecar;
+use super::sidecar::{self, Manifest, ResidentShard, ShardEntry};
+
+/// Documents written between commits. Stated once; the incremental and
+/// FULL writers both commit every `COMMIT_CHUNK_DOCS` (replaced+deleted)
+/// documents so a long run never accumulates one unbounded Xapian
+/// transaction.
+pub const COMMIT_CHUNK_DOCS: usize = 5000;
 
 /// What a run did.
 #[derive(Debug, Default, Clone)]
@@ -35,6 +56,24 @@ pub struct Stats {
     pub duplicate_identity_terms: Vec<String>,
     /// High-water mark of simultaneously live built documents.
     pub max_live_documents: usize,
+    /// Shards written (staged + renamed) this run.
+    pub shards_written: usize,
+    /// Shards removed this run (dataset no longer present).
+    pub shards_deleted: usize,
+    /// High-water mark of simultaneously resident shard-map entries.
+    pub max_resident_sidecar_entries: usize,
+    /// High-water mark of the current dataset's identity-term set.
+    pub max_resident_dataset_terms: usize,
+    /// High-water mark of simultaneously open shards (load-all RED rises).
+    pub peak_open_shards: usize,
+    /// Cumulative shard-file bytes read.
+    pub shard_bytes_read: usize,
+    /// Chunk commits performed.
+    pub commit_chunks: usize,
+    /// Peak `VmRSS` (KiB) observed by the per-chunk probe.
+    pub chunk_rss_peak_kib: u64,
+    /// How the run classified startup (`incremental` or a FULL reason).
+    pub startup: String,
 }
 
 /// Inputs for one run.
@@ -47,16 +86,27 @@ pub struct RunConfig {
     pub publish_filter: bool,
 }
 
-/// Test-only sabotage hooks for the RED controls (gate 5). Never set in
-/// a normal run; each value deliberately breaks one guarantee so the
-/// gate can be shown to fail. `duplicate` skips the identity-uniqueness
-/// error (last-writer-wins, the regression the pre-pass closes); `hash`
-/// perturbs every source hash; `delete` drops the delete pass;
-/// `commit` cancels instead of committing.
+/// Test-only sabotage hooks for the RED controls. Never set in a normal
+/// run; each value deliberately breaks one guarantee so the gate can be
+/// shown to fail. `duplicate` skips the identity-uniqueness error
+/// (last-writer-wins); `hash` perturbs every source hash; `delete` drops
+/// every delete; `commit` cancels instead of committing; `no-recovery`
+/// trusts the sidecar and skips the crash-recovery work entirely.
 fn red_mode() -> Option<String> {
     std::env::var("GN4_INDEXER_RED").ok()
 }
 
+/// RED: skip the startup shard-integrity check and swallow a corrupt or
+/// unreadable shard as empty (the silent-skip regression the check
+/// closes).
+fn red_skip_shard_check() -> bool {
+    red_mode().as_deref() == Some("skip-shard-check")
+}
+
+fn index_exists(index_path: &Path) -> bool {
+    index_path.is_dir()
+}
+
 /// The sibling temp directory a FULL build writes before its atomic
 /// swap into place.
 fn temp_index_path(index_path: &Path) -> PathBuf {
@@ -102,95 +152,39 @@ fn swap_into_place(temp: &Path, index_path: &Path) -> Result<(), String> {
     }
 }
 
-/// The cheap identity pre-pass: enumerate every row's identity term
-/// WITHOUT building documents, flag duplicate `name:dataset` pairs, and
-/// return the full idterm set. Duplicates are an error to report, not
-/// last-writer-wins.
-fn identity_prepass(
-    genes: &[GeneRow],
-    phenotypes: &[PhenotypeRow],
-) -> (BTreeSet<String>, Vec<String>) {
-    let mut seen: BTreeMap<String, usize> = BTreeMap::new();
-    let mut duplicates = Vec::new();
-    let mut idterms = BTreeSet::new();
-    let mut position = 0usize;
-    for row in genes {
-        if let Some(idterm) = gene_idterm(row) {
-            record_idterm(&mut seen, &mut duplicates, &mut idterms, idterm, position);
-            position += 1;
-        }
-    }
-    for row in phenotypes {
-        if let Some(idterm) = phenotype_idterm(row) {
-            record_idterm(&mut seen, &mut duplicates, &mut idterms, idterm, position);
-            position += 1;
-        }
+/// The identity term of a crawl row, or `None` when it produces no
+/// document. Uses the same helpers the document builder uses, so the
+/// pre-pass and the builder cannot drift.
+fn row_idterm(row: &CrawlRow) -> Option<String> {
+    match row {
+        CrawlRow::Gene(gene) => gene_idterm(gene),
+        CrawlRow::Phenotype(phenotype) => phenotype_idterm(phenotype),
     }
-    (idterms, duplicates)
 }
 
-fn record_idterm(
-    seen: &mut BTreeMap<String, usize>,
-    duplicates: &mut Vec<String>,
-    idterms: &mut BTreeSet<String>,
-    idterm: String,
-    position: usize,
-) {
-    if let Some(previous) = seen.get(&idterm) {
-        duplicates.push(format!(
-            "{idterm} (first at position {previous}, duplicate at {position})"
-        ));
-    } else {
-        seen.insert(idterm.clone(), position);
-        idterms.insert(idterm);
+/// Build the document for a crawl row.
+fn build_row(row: CrawlRow, caches: &RdfCaches) -> Option<BuiltDocument> {
+    match row {
+        CrawlRow::Gene(gene) => build_gene(&gene, caches),
+        CrawlRow::Phenotype(phenotype) => build_phenotype(&phenotype),
     }
 }
 
-/// Replay the crawl once, building one document at a time and handing
-/// it to `emit`. Documents are produced in the same order (genes then
-/// phenotypes) the batch builder used.
-fn for_each_document<F>(
-    genes: &[GeneRow],
-    phenotypes: &[PhenotypeRow],
-    caches: &RdfCaches,
-    red_hash: bool,
-    mut emit: F,
-) -> Result<(), String>
-where
-    F: FnMut(BuiltDocument) -> Result<(), String>,
-{
-    for row in genes {
-        if let Some(mut document) = build_gene(row, caches) {
-            if let Some(nonce) = red_nonce(red_hash) {
-                document.source_hash.push_str(&nonce);
-            }
-            emit(document)?;
-        }
-    }
-    for row in phenotypes {
-        if let Some(mut document) = build_phenotype(row) {
-            if let Some(nonce) = red_nonce(red_hash) {
-                document.source_hash.push_str(&nonce);
-            }
-            emit(document)?;
-        }
-    }
-    Ok(())
-}
-
 fn red_nonce(red_hash: bool) -> Option<String> {
     if !red_hash {
         return None;
     }
-    // Include the process id so every RUN produces different hashes:
-    // the point of the RED control is that a re-run over unchanged
-    // data still rewrites (i.e. the idempotence gate can fail).
+    // Include a per-run counter so every RUN produces different hashes:
+    // the point of the RED control is that a re-run over unchanged data
+    // still rewrites (i.e. the idempotence gate can fail).
     static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
     let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
     Some(format!("-red{}-{n}", std::process::id()))
 }
 
 /// Enumerate the identity (`Q`-prefixed) terms the index actually holds.
+/// This is the expensive O(docs) enumeration; it is only reached on the
+/// startup-mismatch path, never in the steady state.
 fn index_identity_terms(db: &WritableDatabase) -> BTreeSet<String> {
     let read: Database = Database::from(db);
     let mut enquire = Enquire::new(&read);
@@ -210,40 +204,327 @@ fn index_identity_terms(db: &WritableDatabase) -> BTreeSet<String> {
     terms
 }
 
-/// Run a FULL (bootstrap) build: a fresh temp index written document by
-/// document through `replace_document_by_term`, swapped into place only
-/// on full success.
+// ---------------------------------------------------------------------------
+// Duplicate-identity pre-pass (one streaming pass, O(largest dataset)).
+// ---------------------------------------------------------------------------
+
+struct DuplicateSink {
+    set: BTreeSet<String>,
+    duplicates: Vec<String>,
+    max_terms: usize,
+}
+
+impl DuplicateSink {
+    fn new() -> Self {
+        DuplicateSink {
+            set: BTreeSet::new(),
+            duplicates: Vec::new(),
+            max_terms: 0,
+        }
+    }
+}
+
+impl DatasetSink for DuplicateSink {
+    fn begin_dataset(&mut self, _label: &str) -> Result<(), String> {
+        self.set.clear();
+        Ok(())
+    }
+
+    fn row(&mut self, row: CrawlRow) -> Result<(), String> {
+        if let Some(idterm) = row_idterm(&row) {
+            if !self.set.insert(idterm.clone()) {
+                self.duplicates.push(idterm);
+            }
+            self.max_terms = self.max_terms.max(self.set.len());
+        }
+        Ok(())
+    }
+
+    fn end_dataset(&mut self, _label: &str) -> Result<(), String> {
+        Ok(())
+    }
+}
+
+/// Run the duplicate-identity pre-pass. Returns the duplicate terms
+/// (empty when the crawl is unique). The per-dataset set is bounded by
+/// the largest dataset, and the pass finishes before any index write.
+fn duplicate_prepass(sql_uri: &str, filtered: bool) -> Result<(Vec<String>, usize), String> {
+    let mut sink = DuplicateSink::new();
+    rows::stream_dataset_groups(sql_uri, filtered, &mut sink)?;
+    Ok((sink.duplicates, sink.max_terms))
+}
+
+// ---------------------------------------------------------------------------
+// Chunked-commit pacer with a per-chunk memory probe.
+// ---------------------------------------------------------------------------
+
+/// Reads this process's `VmRSS` (KiB) from `/proc/self/status`.
+fn read_vm_rss_kib() -> u64 {
+    let text = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
+    for line in text.lines() {
+        if line.starts_with("VmRSS:") {
+            if let Some(value) = line.split_whitespace().nth(1) {
+                return value.parse().unwrap_or(0);
+            }
+        }
+    }
+    0
+}
+
+struct CommitPacer {
+    chunk: usize,
+    open: bool,
+    writes_since: usize,
+    chunks: usize,
+    rss_peak_kib: u64,
+    /// Optional file each chunk's probe is appended to (evidence).
+    log_path: Option<PathBuf>,
+    /// Park after this many chunk commits (crash-gate injection point).
+    halt_after: Option<usize>,
+    /// RED: cancel instead of committing.
+    red_commit: bool,
+}
+
+impl CommitPacer {
+    fn new(red_commit: bool) -> Self {
+        let log_path = std::env::var("GN4_INDEXER_COMMIT_LOG")
+            .ok()
+            .map(PathBuf::from);
+        let halt_after = std::env::var("GN4_INDEXER_HALT_AFTER_COMMITS")
+            .ok()
+            .and_then(|value| value.parse().ok());
+        // The production chunk size is COMMIT_CHUNK_DOCS; the crash gate
+        // overrides it downward so a small mix spans several chunks.
+        let chunk = std::env::var("GN4_INDEXER_COMMIT_CHUNK")
+            .ok()
+            .and_then(|value| value.parse().ok())
+            .unwrap_or(COMMIT_CHUNK_DOCS);
+        CommitPacer {
+            chunk,
+            open: false,
+            writes_since: 0,
+            chunks: 0,
+            rss_peak_kib: 0,
+            log_path,
+            halt_after,
+            red_commit,
+        }
+    }
+
+    /// Record one write, opening a transaction lazily and committing when
+    /// the chunk fills.
+    fn write(&mut self, db: &mut WritableDatabase) -> Result<(), String> {
+        if !self.open {
+            db.begin_transaction(None);
+            self.open = true;
+        }
+        self.writes_since += 1;
+        if self.writes_since >= self.chunk {
+            self.flush(db)?;
+        }
+        Ok(())
+    }
+
+    /// Commit the open chunk (or cancel it under the RED), probe memory,
+    /// and optionally park for the crash gate.
+    fn flush(&mut self, db: &mut WritableDatabase) -> Result<(), String> {
+        if self.red_commit {
+            db.cancel_transaction();
+        } else {
+            db.commit_transaction();
+        }
+        self.open = false;
+        self.writes_since = 0;
+        self.chunks += 1;
+        let rss = read_vm_rss_kib();
+        self.rss_peak_kib = self.rss_peak_kib.max(rss);
+        if let Some(path) = &self.log_path {
+            use std::io::Write;
+            if let Ok(mut file) = std::fs::OpenOptions::new()
+                .create(true)
+                .append(true)
+                .open(path)
+            {
+                let _ = writeln!(file, "chunk={} rss_kib={}", self.chunks, rss);
+            }
+        }
+        if let Some(limit) = self.halt_after {
+            if self.chunks == limit {
+                // Park forever: the crash gate kills -9 at this point, so
+                // the process dies with the chunk committed and no
+                // destructors, flushes or sidecar writes having run.
+                eprintln!(
+                    "gn-indexer: halted after {} chunk commits (kill -9 target)",
+                    self.chunks
+                );
+                loop {
+                    std::thread::sleep(std::time::Duration::from_secs(3600));
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Commit the final partial chunk.
+    fn finish(&mut self, db: &mut WritableDatabase) -> Result<(), String> {
+        if self.open {
+            self.flush(db)?;
+        }
+        Ok(())
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Sidecar finalize (temp shards first, manifest last).
+// ---------------------------------------------------------------------------
+
+struct SidecarBuild {
+    index_path: PathBuf,
+    shard_dir: PathBuf,
+    stage_dir: PathBuf,
+    manifest: Manifest,
+    staged: Vec<(PathBuf, ShardEntry)>,
+}
+
+impl SidecarBuild {
+    fn new(index_path: &Path, mode: &str, rif_hash: String, wiki_hash: String) -> Self {
+        let stage_dir = PathBuf::from(format!(
+            "{}.tmp.{}",
+            sidecar::shard_dir(index_path).display(),
+            std::process::id()
+        ));
+        SidecarBuild {
+            index_path: index_path.to_path_buf(),
+            shard_dir: sidecar::shard_dir(index_path),
+            stage_dir,
+            manifest: Manifest {
+                version: sidecar::SIDECAR_VERSION,
+                mode: mode.to_string(),
+                doc_count: 0,
+                rif_hash,
+                wiki_hash,
+                shards: BTreeMap::new(),
+            },
+            staged: Vec::new(),
+        }
+    }
+
+    /// Stage a shard's new content (temp + fsync) and record its manifest
+    /// entry. The rename happens in [`finalize`].
+    fn stage(&mut self, label: &str, map: &BTreeMap<String, String>) -> Result<(), String> {
+        let (temp, entry) = sidecar::stage_shard(&self.stage_dir, label, map)?;
+        self.manifest
+            .shards
+            .insert(label.to_string(), entry.clone());
+        self.staged.push((temp, entry));
+        Ok(())
+    }
+
+    /// Keep an unchanged shard's existing (recomputed) entry.
+    fn keep(&mut self, label: &str, entry: ShardEntry) {
+        self.manifest.shards.insert(label.to_string(), entry);
+    }
+
+    /// Drop a dataset whose rows are all gone.
+    fn remove(&mut self, label: &str) {
+        self.manifest.shards.remove(label);
+    }
+
+    fn shard_count(&self) -> usize {
+        self.manifest.shards.len()
+    }
+
+    /// Sum of the (kept + staged) shard counts: the total document count
+    /// the finalized manifest will describe.
+    fn total_count(&self) -> usize {
+        self.manifest.shards.values().map(|e| e.count).sum()
+    }
+
+    /// Rename every staged shard into place, fsync the shard directory,
+    /// prune shard files the manifest no longer references, and write the
+    /// **manifest last**.
+    fn finalize(mut self) -> Result<(), String> {
+        self.manifest.doc_count = self.manifest.shards.values().map(|e| e.count).sum();
+        std::fs::create_dir_all(&self.shard_dir)
+            .map_err(|e| format!("create shard dir {}: {e}", self.shard_dir.display()))?;
+        for (temp, entry) in &self.staged {
+            let target = self.shard_dir.join(&entry.file);
+            std::fs::rename(temp, &target)
+                .map_err(|e| format!("rename shard {}: {e}", target.display()))?;
+        }
+        sidecar::fsync_dir(&self.shard_dir)?;
+
+        let keep: BTreeSet<String> = self
+            .manifest
+            .shards
+            .values()
+            .map(|e| e.file.clone())
+            .collect();
+        if let Ok(entries) = std::fs::read_dir(&self.shard_dir) {
+            for entry in entries.flatten() {
+                let name = entry.file_name().to_string_lossy().to_string();
+                if name.starts_with('.') {
+                    continue;
+                }
+                if !keep.contains(&name) {
+                    let _ = std::fs::remove_file(entry.path());
+                }
+            }
+        }
+        let _ = std::fs::remove_dir_all(&self.stage_dir);
+        self.manifest.store(&self.index_path)
+    }
+}
+
+// ---------------------------------------------------------------------------
+// FULL
+// ---------------------------------------------------------------------------
+
+/// Run a FULL (bootstrap) build: duplicate pre-pass, then a fresh temp
+/// index written dataset by dataset through chunked commits, swapped into
+/// place only on full success, then a fresh sharded sidecar.
 pub fn run_full(config: &RunConfig) -> Result<Stats, String> {
+    run_full_with_reason(config, None)
+}
+
+fn run_full_with_reason(config: &RunConfig, reason: Option<String>) -> Result<Stats, String> {
     let caches = RdfCaches::load(&config.rdf_cache)?;
-    let genes = rows::crawl_genes(&config.sql_uri)?;
-    let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?;
+    // Duplicate validation happens BEFORE any index is opened or written.
+    let (duplicates, max_dataset_terms) =
+        duplicate_prepass(&config.sql_uri, config.publish_filter)?;
     let red = red_mode();
-
-    // Identity validation happens BEFORE any index is opened or
-    // written, so a duplicate crawl cannot touch an existing index.
-    let (_idterms, duplicates) = identity_prepass(&genes, &phenotypes);
     if !duplicates.is_empty() && red.as_deref() != Some("duplicate") {
         return Err(format!(
-            "duplicate identity terms in crawl: {}",
-            duplicates.join(", ")
+            "duplicate identity terms in crawl: {}{}",
+            duplicates.join(", "),
+            match &reason {
+                Some(r) => format!(" (startup: {r})"),
+                None => String::new(),
+            }
         ));
     }
 
     livemem::reset();
+    sidecar::mem::reset();
+    sidecar::cleanup_temp(&config.index_path)?;
     let retain = livemem::red_retain();
+
     let temp = temp_index_path(&config.index_path);
-    let outcome = stream_full(
-        &temp,
-        &genes,
-        &phenotypes,
-        &caches,
-        red.as_deref() == Some("hash"),
-        retain,
+    if temp.exists() {
+        let _ = std::fs::remove_dir_all(&temp);
+    }
+    let mut build = SidecarBuild::new(
+        &config.index_path,
+        "full",
+        caches.rif.hash.clone(),
+        caches.wiki.hash.clone(),
     );
-    let (index_doc_count, hashes, built) = match outcome {
+    let outcome = stream_full(&temp, config, &caches, &mut build, retain);
+    let (index_doc_count, built, pacer_chunks, pacer_rss_peak) = match outcome {
         Ok(value) => value,
         Err(error) => {
             let _ = std::fs::remove_dir_all(&temp);
+            let _ = std::fs::remove_dir_all(&build.stage_dir);
             return Err(error);
         }
     };
@@ -252,96 +533,195 @@ pub fn run_full(config: &RunConfig) -> Result<Stats, String> {
         return Err(error);
     }
 
-    // Sidecar strictly after the index commit (and the swap).
-    let sidecar = Sidecar {
-        version: 1,
-        mode: "full".to_string(),
-        doc_count: built,
-        rif_hash: caches.rif.hash.clone(),
-        wiki_hash: caches.wiki.hash.clone(),
-        documents: hashes,
-    };
-    sidecar.store(&config.index_path)?;
+    // Sidecar strictly after the index commit (and the swap): changed
+    // shards first, then the manifest last.
+    let shard_count = build.shard_count();
+    let stage_dir = build.stage_dir.clone();
+    if let Err(error) = build.finalize() {
+        let _ = std::fs::remove_dir_all(&stage_dir);
+        return Err(error);
+    }
 
-    Ok(Stats {
+    let mut stats = Stats {
         mode: "full".to_string(),
         built,
         replaced: built,
+        deleted: 0,
+        unchanged: 0,
         index_doc_count,
         duplicate_identity_terms: Vec::new(),
         max_live_documents: livemem::max_live(),
-        ..Default::default()
-    })
+        shards_written: shard_count,
+        shards_deleted: 0,
+        max_resident_sidecar_entries: sidecar::mem::peak_resident_entries(),
+        max_resident_dataset_terms: max_dataset_terms,
+        peak_open_shards: sidecar::mem::peak_open_shards(),
+        shard_bytes_read: sidecar::mem::bytes_read(),
+        commit_chunks: pacer_chunks,
+        chunk_rss_peak_kib: pacer_rss_peak,
+        startup: match reason {
+            Some(reason) => format!("full ({reason})"),
+            None => "full".to_string(),
+        },
+    };
+    stats.mode = "full".to_string();
+    Ok(stats)
+}
+
+struct FullSink<'a> {
+    db: WritableDatabase,
+    caches: &'a RdfCaches,
+    red_hash: bool,
+    retain: bool,
+    kept: Vec<CountedDocument<BuiltDocument>>,
+    pacer: CommitPacer,
+    build: &'a mut SidecarBuild,
+    shard: BTreeMap<String, String>,
+    built: usize,
+}
+
+impl<'a> DatasetSink for FullSink<'a> {
+    fn begin_dataset(&mut self, _label: &str) -> Result<(), String> {
+        self.shard.clear();
+        Ok(())
+    }
+
+    fn row(&mut self, row: CrawlRow) -> Result<(), String> {
+        let Some(mut document) = build_row(row, self.caches) else {
+            return Ok(());
+        };
+        if let Some(nonce) = red_nonce(self.red_hash) {
+            document.source_hash.push_str(&nonce);
+        }
+        let counted = CountedDocument::new(document);
+        let doc = counted.get();
+        self.db.replace_document_by_term(&doc.idterm, &doc.document);
+        self.shard
+            .insert(doc.idterm.clone(), doc.source_hash.clone());
+        self.built += 1;
+        if self.retain {
+            self.kept.push(counted);
+        }
+        self.pacer.write(&mut self.db)?;
+        Ok(())
+    }
+
+    fn end_dataset(&mut self, label: &str) -> Result<(), String> {
+        if !self.shard.is_empty() {
+            self.build.stage(label, &self.shard)?;
+        }
+        self.shard.clear();
+        Ok(())
+    }
 }
 
-/// Stream a fresh FULL build into `temp`: open once, one transaction,
-/// write each document as it is built. Returns the committed document
-/// count, the idterm→source-hash map for the sidecar, and the built
-/// count.
 fn stream_full(
     temp: &Path,
-    genes: &[GeneRow],
-    phenotypes: &[PhenotypeRow],
+    config: &RunConfig,
     caches: &RdfCaches,
-    red_hash: bool,
+    build: &mut SidecarBuild,
     retain: bool,
-) -> Result<(u32, BTreeMap<String, String>, usize), String> {
-    let mut db = WritableDatabase::open(
+) -> Result<(u32, usize, usize, u64), String> {
+    // One writer handle, opened into the fresh temp directory; the
+    // transaction is opened lazily by the commit pacer.
+    let db = WritableDatabase::open(
         temp,
         Some(DbAction::CreateOrOverwrite),
         Some(DbBackend::Glass),
         None,
         None,
     );
-    db.begin_transaction(None);
-
-    let mut hashes: BTreeMap<String, String> = BTreeMap::new();
-    let mut kept: Vec<CountedDocument<BuiltDocument>> = Vec::new();
-    let mut count = 0usize;
-    {
-        let emit = |document: BuiltDocument| -> Result<(), String> {
-            let counted = CountedDocument::new(document);
-            let idterm = counted.get().idterm.clone();
-            hashes.insert(idterm.clone(), counted.get().source_hash.clone());
-            let _ = db.replace_document_by_term(&idterm, &counted.get().document);
-            count += 1;
-            if retain {
-                kept.push(counted);
-            }
-            Ok(())
-        };
-        if let Err(error) = for_each_document(genes, phenotypes, caches, red_hash, emit) {
-            db.cancel_transaction();
-            return Err(error);
+    let mut sink = FullSink {
+        db,
+        caches,
+        red_hash: red_mode().as_deref() == Some("hash"),
+        retain,
+        kept: Vec::new(),
+        pacer: CommitPacer::new(red_mode().as_deref() == Some("commit")),
+        build,
+        shard: BTreeMap::new(),
+        built: 0,
+    };
+    let result = rows::stream_dataset_groups(&config.sql_uri, config.publish_filter, &mut sink);
+    if let Err(error) = result {
+        if sink.pacer.open {
+            sink.db.cancel_transaction();
         }
+        return Err(error);
     }
-
-    commit(&mut db, red_mode().as_deref() == Some("commit"))?;
-    let doc_count = db.doc_count();
-    drop(db);
-    drop(kept);
-    Ok((doc_count, hashes, count))
+    sink.pacer.finish(&mut sink.db)?;
+    let doc_count = sink.db.doc_count();
+    let built = sink.built;
+    let chunks = sink.pacer.chunks;
+    let rss_peak = sink.pacer.rss_peak_kib;
+    Ok((doc_count, built, chunks, rss_peak))
 }
 
-/// Run an INCREMENTAL pass: crawl, hash, diff against the sidecar,
-/// replace only changed documents, delete vanished identity terms, and
-/// write nothing when nothing changed.
+// ---------------------------------------------------------------------------
+// INCREMENTAL
+// ---------------------------------------------------------------------------
+
+/// Run an INCREMENTAL pass: cheap-first startup check, then a dataset-
+/// grouped streaming diff that replaces changed documents, deletes
+/// vanished ones, and writes nothing when nothing changed.
 pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> {
-    let Some(previous) = Sidecar::load(&config.index_path) else {
-        // Missing or unreadable sidecar means FULL.
-        return run_full(config);
+    let Some(manifest) = Manifest::load(&config.index_path) else {
+        // Missing, unreadable, or old-version sidecar means FULL.
+        return run_full_with_reason(config, Some("no/old sidecar".to_string()));
     };
-    if previous.version != 1 {
-        return run_full(config);
+    if !index_exists(&config.index_path) {
+        return run_full_with_reason(config, Some("index missing".to_string()));
     }
 
-    let caches = RdfCaches::load(&config.rdf_cache)?;
-    let genes = rows::crawl_genes(&config.sql_uri)?;
-    let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?;
+    sidecar::cleanup_temp(&config.index_path)?;
+
+    // Cheap-first startup (O(1)): the manifest doc count against the
+    // index's own doc count.
+    let probe = WritableDatabase::open(
+        &config.index_path,
+        Some(DbAction::CreateOrOpen),
+        Some(DbBackend::Glass),
+        None,
+        None,
+    );
+    let index_doc_count = probe.doc_count();
     let red = red_mode();
+    let no_recovery = red.as_deref() == Some("no-recovery");
+    if !no_recovery && index_doc_count as usize != manifest.doc_count {
+        let reason = format!(
+            "doc_count index={} manifest={}",
+            index_doc_count, manifest.doc_count
+        );
+        // Expensive path only here: enumerate index identity terms to
+        // characterise the key-set mismatch, then rebuild.
+        let index_terms = index_identity_terms(&probe);
+        eprintln!(
+            "gn-indexer: startup mismatch ({reason}); index identity terms = {}; falling back to FULL",
+            index_terms.len()
+        );
+        drop(probe);
+        return run_full_with_reason(config, Some(reason));
+    }
+    drop(probe);
 
-    // Identity pre-pass, before any write.
-    let (idterms, duplicates) = identity_prepass(&genes, &phenotypes);
+    // Verify every shard (bit-rot) and confirm it parses, before trusting
+    // the manifest. Any mismatch routes to FULL. Each shard is loaded and
+    // dropped, so the peak stays O(largest shard).
+    if !no_recovery && !red_skip_shard_check() {
+        let dir = sidecar::shard_dir(&config.index_path);
+        for (label, entry) in &manifest.shards {
+            if let Err(error) = ResidentShard::load(&dir, label, entry) {
+                let reason = format!("shard {label}: {error}");
+                eprintln!("gn-indexer: startup mismatch ({reason}); falling back to FULL");
+                return run_full_with_reason(config, Some(reason));
+            }
+        }
+    }
+
+    let caches = RdfCaches::load(&config.rdf_cache)?;
+    // Duplicate validation before any write.
+    let (duplicates, max_dataset_terms) =
+        duplicate_prepass(&config.sql_uri, config.publish_filter)?;
     if !duplicates.is_empty() && red.as_deref() != Some("duplicate") {
         return Err(format!(
             "duplicate identity terms in crawl: {}",
@@ -350,9 +730,10 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> {
     }
 
     livemem::reset();
+    sidecar::mem::reset();
     let retain = livemem::red_retain();
 
-    let mut db = WritableDatabase::open(
+    let db = WritableDatabase::open(
         &config.index_path,
         Some(DbAction::CreateOrOpen),
         Some(DbBackend::Glass),
@@ -360,136 +741,292 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> {
         None,
     );
 
-    // Startup mismatch detection: the sidecar's key set must equal the
-    // index's actual identity terms, else the sidecar is behind the
-    // index (or the index was written elsewhere) and is not trusted; do
-    // a full compare against the index's own terms.
-    let index_terms = index_identity_terms(&db);
-    let sidecar_terms: BTreeSet<String> = previous.documents.keys().cloned().collect();
-    let mismatched = index_terms != sidecar_terms;
-
-    let mut hashes: BTreeMap<String, String> = BTreeMap::new();
-    let mut unchanged = 0usize;
-    let mut replaced = 0usize;
-    let mut open_transaction = false;
-    let mut kept: Vec<CountedDocument<BuiltDocument>> = Vec::new();
-    {
-        let previous_documents = &previous.documents;
-        let emit = |document: BuiltDocument| -> Result<(), String> {
-            let counted = CountedDocument::new(document);
-            let doc = counted.get();
-            hashes.insert(doc.idterm.clone(), doc.source_hash.clone());
-            let known = if mismatched {
-                None
-            } else {
-                previous_documents.get(&doc.idterm)
-            };
-            match known {
-                Some(hash) if *hash == doc.source_hash => unchanged += 1,
-                _ => {
-                    if !open_transaction {
-                        db.begin_transaction(None);
-                        open_transaction = true;
-                    }
-                    let _ = db.replace_document_by_term(&doc.idterm, &doc.document);
-                    replaced += 1;
-                }
-            }
-            if retain {
-                kept.push(counted);
-            }
-            Ok(())
-        };
-        if let Err(error) = for_each_document(
-            &genes,
-            &phenotypes,
-            &caches,
-            red.as_deref() == Some("hash"),
-            emit,
-        ) {
-            if open_transaction {
-                db.cancel_transaction();
-            }
-            return Err(error);
-        }
+    let shard_dir = sidecar::shard_dir(&config.index_path);
+    let by_label: HashMap<String, ShardEntry> = manifest
+        .shards
+        .iter()
+        .map(|(label, entry)| (label.clone(), entry.clone()))
+        .collect();
+
+    let mut build = SidecarBuild::new(
+        &config.index_path,
+        "incremental",
+        caches.rif.hash.clone(),
+        caches.wiki.hash.clone(),
+    );
+    // Carry forward every existing shard entry; changed datasets replace
+    // their entry, removed datasets are dropped below.
+    for (label, entry) in &manifest.shards {
+        build.keep(label, entry.clone());
     }
 
-    // Existence truth: the sidecar's key set (or, on mismatch, the
-    // index's own terms). Anything there but not in the crawl is gone.
-    // Deletes need the full crawl, so they are applied at the end.
-    let mut to_delete: BTreeSet<String> = if mismatched {
-        index_terms
-    } else {
-        sidecar_terms
+    let mut sink = IncrementalSink {
+        db,
+        caches: &caches,
+        red_hash: red.as_deref() == Some("hash"),
+        red_delete: red.as_deref() == Some("delete"),
+        no_recovery,
+        retain,
+        kept: Vec::new(),
+        pacer: CommitPacer::new(red.as_deref() == Some("commit")),
+        build: &mut build,
+        shard_dir,
+        by_label,
+        seen: BTreeSet::new(),
+        previous: None,
+        previous_label: String::new(),
+        dataset_terms: BTreeSet::new(),
+        dataset_map: BTreeMap::new(),
+        retainer: sidecar::ResidentRetainer::new(),
+        replaced: 0,
+        deleted: 0,
+        unchanged: 0,
+        max_dataset_terms: 0,
+        new_shards: 0,
+        removed_shards: 0,
     };
-    if red.as_deref() == Some("delete") {
-        to_delete.clear();
-    } else {
-        for term in &idterms {
-            to_delete.remove(term);
+
+    let result = rows::stream_dataset_groups(&config.sql_uri, config.publish_filter, &mut sink);
+    if let Err(error) = result {
+        if sink.pacer.open {
+            sink.db.cancel_transaction();
         }
+        return Err(error);
     }
 
-    let mut deleted = 0usize;
-    if !to_delete.is_empty() {
-        if !open_transaction {
-            db.begin_transaction(None);
-            open_transaction = true;
-        }
-        for term in &to_delete {
-            db.delete_document_by_term(term);
-            deleted += 1;
-        }
+    // Datasets in the manifest that the crawl never produced: every one
+    // of their documents is gone.
+    let leftovers: Vec<String> = sink
+        .by_label
+        .keys()
+        .filter(|label| !sink.seen.contains(*label))
+        .cloned()
+        .collect();
+    for label in leftovers {
+        sink.delete_dataset(&label)?;
+        sink.removed_shards += 1;
     }
 
-    let mut stats = Stats {
+    let nothing_changed = sink.replaced == 0 && sink.deleted == 0;
+    sink.pacer.finish(&mut sink.db)?;
+    let doc_count = sink.db.doc_count();
+    let replaced = sink.replaced;
+    let deleted = sink.deleted;
+    let unchanged = sink.unchanged;
+    let max_resident = sidecar::mem::peak_resident_entries();
+    let peak_open = sidecar::mem::peak_open_shards();
+    let bytes_read = sidecar::mem::bytes_read();
+    let shards_written = sink.new_shards;
+    let shards_deleted = sink.removed_shards;
+    let sink_dataset_terms = sink.max_dataset_terms;
+    let commit_chunks = sink.pacer.chunks;
+    let chunk_rss_peak_kib = sink.pacer.rss_peak_kib;
+    let max_live = livemem::max_live();
+    drop(sink);
+
+    // The finalized manifest's shard counts are the crawl's document
+    // count (kept + staged, minus removed datasets).
+    let built = build.total_count();
+
+    // Sidecar after the commit; skipped when nothing changed so an
+    // idempotent run touches neither the index nor the sidecar.
+    if !nothing_changed {
+        build.finalize()?;
+    } else {
+        let _ = std::fs::remove_dir_all(&build.stage_dir);
+    }
+
+    Ok(Stats {
         mode: "incremental".to_string(),
-        built: hashes.len(),
-        unchanged,
+        built,
         replaced,
         deleted,
-        max_live_documents: livemem::max_live(),
-        ..Default::default()
-    };
+        unchanged,
+        index_doc_count: doc_count,
+        duplicate_identity_terms: Vec::new(),
+        max_live_documents: max_live,
+        shards_written,
+        shards_deleted,
+        max_resident_sidecar_entries: max_resident,
+        max_resident_dataset_terms: max_dataset_terms.max(sink_dataset_terms),
+        peak_open_shards: peak_open,
+        shard_bytes_read: bytes_read,
+        commit_chunks,
+        chunk_rss_peak_kib,
+        startup: "incremental".to_string(),
+    })
+}
 
-    let nothing_changed = replaced == 0 && deleted == 0;
-    if open_transaction {
-        // A zero-change run opens no transaction and writes nothing.
-        commit(&mut db, red.as_deref() == Some("commit"))?;
+struct IncrementalSink<'a> {
+    db: WritableDatabase,
+    caches: &'a RdfCaches,
+    red_hash: bool,
+    red_delete: bool,
+    no_recovery: bool,
+    retain: bool,
+    kept: Vec<CountedDocument<BuiltDocument>>,
+    pacer: CommitPacer,
+    build: &'a mut SidecarBuild,
+    shard_dir: PathBuf,
+    by_label: HashMap<String, ShardEntry>,
+    seen: BTreeSet<String>,
+    previous: Option<ResidentShard>,
+    previous_label: String,
+    dataset_terms: BTreeSet<String>,
+    dataset_map: BTreeMap<String, String>,
+    retainer: sidecar::ResidentRetainer,
+    replaced: usize,
+    deleted: usize,
+    unchanged: usize,
+    max_dataset_terms: usize,
+    new_shards: usize,
+    removed_shards: usize,
+}
+
+impl<'a> IncrementalSink<'a> {
+    /// Delete every document of a dataset (all its rows vanished) and
+    /// drop its shard.
+    fn delete_dataset(&mut self, label: &str) -> Result<(), String> {
+        if self.no_recovery {
+            // RED: trust the sidecar, skip the recovery work entirely.
+            return Ok(());
+        }
+        if !self.red_delete {
+            let entry = self.by_label.get(label).cloned();
+            if let Some(entry) = entry {
+                let shard = ResidentShard::load(&self.shard_dir, label, &entry)
+                    .map_err(|e| format!("shard {label}: {e}"))?;
+                for term in shard.map().keys() {
+                    self.db.delete_document_by_term(term);
+                    self.deleted += 1;
+                    self.pacer.write(&mut self.db)?;
+                }
+            }
+        }
+        self.build.remove(label);
+        Ok(())
     }
-    stats.index_doc_count = db.doc_count();
+}
 
-    // Sidecar after the commit; skipped when nothing changed so an
-    // idempotent run touches neither the index nor the sidecar.
-    if !nothing_changed {
-        let sidecar = Sidecar {
-            version: 1,
-            mode: "incremental".to_string(),
-            doc_count: hashes.len(),
-            rif_hash: caches.rif.hash.clone(),
-            wiki_hash: caches.wiki.hash.clone(),
-            documents: hashes,
+impl<'a> DatasetSink for IncrementalSink<'a> {
+    fn begin_dataset(&mut self, label: &str) -> Result<(), String> {
+        self.seen.insert(label.to_string());
+        self.dataset_terms.clear();
+        self.dataset_map.clear();
+        self.previous = None;
+        self.previous_label = label.to_string();
+        if let Some(entry) = self.by_label.get(label).cloned() {
+            // Verify-before-parse: the shard file's hash is re-checked
+            // here (it was checked at startup too). A mismatch would mean
+            // the file changed under us; treat it as a hard error.
+            match ResidentShard::load(&self.shard_dir, label, &entry) {
+                Ok(shard) => {
+                    if sidecar::red_load_all() {
+                        // Load-all RED: hold every shard resident so the peak
+                        // resident counter reflects O(docs), not O(largest shard).
+                        self.retainer.retain(shard);
+                    } else {
+                        self.previous = Some(shard);
+                    }
+                }
+                Err(error) => {
+                    if red_skip_shard_check() {
+                        // RED: swallow the corruption as an empty shard, so
+                        // the dataset is silently treated as new/empty.
+                        eprintln!("gn-indexer: [RED skip-shard-check] ignoring {label}: {error}");
+                        self.previous = None;
+                    } else {
+                        return Err(format!("shard {label}: {error}"));
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn row(&mut self, row: CrawlRow) -> Result<(), String> {
+        let Some(idterm) = row_idterm(&row) else {
+            return Ok(());
+        };
+        self.dataset_terms.insert(idterm.clone());
+        self.max_dataset_terms = self.max_dataset_terms.max(self.dataset_terms.len());
+        let Some(mut document) = build_row(row, self.caches) else {
+            return Ok(());
         };
-        sidecar.store(&config.index_path)?;
+        if let Some(nonce) = red_nonce(self.red_hash) {
+            document.source_hash.push_str(&nonce);
+        }
+        let counted = CountedDocument::new(document);
+        let doc = counted.get();
+        self.dataset_map
+            .insert(doc.idterm.clone(), doc.source_hash.clone());
+        let known = if self.no_recovery {
+            // RED: pretend the source hash is unchanged, so no rewrite
+            // ever happens and a crash cannot be recovered.
+            Some(doc.source_hash.clone())
+        } else {
+            self.previous
+                .as_ref()
+                .and_then(|shard| shard.map().get(&doc.idterm).cloned())
+        };
+        match known {
+            Some(hash) if hash == doc.source_hash => self.unchanged += 1,
+            _ => {
+                self.db.replace_document_by_term(&doc.idterm, &doc.document);
+                self.replaced += 1;
+                self.pacer.write(&mut self.db)?;
+            }
+        }
+        if self.retain {
+            self.kept.push(counted);
+        }
+        Ok(())
     }
-    let _ = kept;
-    Ok(stats)
-}
 
-fn commit(db: &mut WritableDatabase, skip: bool) -> Result<(), String> {
-    if skip {
-        // RED control: writes are made but never committed, so they are
-        // invisible to readers.
-        db.cancel_transaction();
-        return Ok(());
+    fn end_dataset(&mut self, label: &str) -> Result<(), String> {
+        // Deletes: identity terms in the previous shard that the crawl no
+        // longer produces, computed from this dataset's key set.
+        if !self.red_delete && !self.no_recovery {
+            if let Some(shard) = &self.previous {
+                let vanished: Vec<String> = shard
+                    .map()
+                    .keys()
+                    .filter(|term| !self.dataset_terms.contains(*term))
+                    .cloned()
+                    .collect();
+                for term in vanished {
+                    self.db.delete_document_by_term(&term);
+                    self.deleted += 1;
+                    self.pacer.write(&mut self.db)?;
+                }
+            }
+        }
+
+        // Decide whether the shard content changed.
+        let unchanged_shard = match &self.previous {
+            Some(shard) => shard.map() == &self.dataset_map,
+            None => self.dataset_map.is_empty(),
+        };
+        if !unchanged_shard && !self.dataset_map.is_empty() {
+            self.build.stage(label, &self.dataset_map)?;
+            self.new_shards += 1;
+        } else if unchanged_shard {
+            if let Some(entry) = self.by_label.get(label).cloned() {
+                self.build.keep(label, entry);
+            }
+        } else if self.dataset_map.is_empty() {
+            // The dataset still appears in the crawl but produced no
+            // documents: treat it as gone.
+            self.build.remove(label);
+        }
+        self.previous = None;
+        Ok(())
     }
-    db.commit_transaction();
-    Ok(())
 }
 
 /// Where the sidecar lives for an index path.
 pub fn sidecar_location(index_path: &Path) -> PathBuf {
-    super::sidecar::sidecar_path(index_path)
+    sidecar::sidecar_path(index_path)
 }
 
 /// The document type of an identity term (for reporting).
diff --git a/src/indexer/sidecar.rs b/src/indexer/sidecar.rs
index 8dcc5a3..394fe50 100644
--- a/src/indexer/sidecar.rs
+++ b/src/indexer/sidecar.rs
@@ -1,23 +1,71 @@
-//! The incremental sidecar: per-document source hashes stored next to
-//! the index (not inside it), written atomically and strictly after the
-//! index commit.
+//! The incremental sidecar, sharded by dataset (format version 2).
 //!
-//! The sidecar's key set is the existence truth for the incremental
-//! path: a document absent from a crawl is deleted by its identity term
-//! even if the index still holds it. Write order is index commit first,
-//! sidecar rename after, so a crash between the two leaves a sidecar
-//! that is behind the index — the startup key-set check detects exactly
-//! that and falls back to a full compare.
+//! A small **manifest** at `<index>.sidecar.json` (version 2) holds the
+//! run metadata and one entry per dataset shard:
+//!
+//! ```text
+//! { version, mode, doc_count, rif_hash, wiki_hash,
+//!   shards: { "<dataset>": { file, count, content_sha256 } } }
+//! ```
+//!
+//! Each shard is a JSON map `identity-term -> source-hash` at
+//! `<index>.sidecar.d/<file>`, where the filename is the dataset label
+//! sanitized for the filesystem plus a short digest of the raw label (so
+//! two labels that sanitize to the same string still get distinct files).
+//!
+//! Why sharding: the per-dataset shard is the unit the incremental diff
+//! loads, so the resident idterm->hash map is O(largest shard), not
+//! O(documents). The manifest is sharded so a crawl can rewrite one
+//! dataset without touching the rest.
+//!
+//! Integrity and atomicity:
+//!
+//! * a shard's `content_sha256` is **recomputed from the shard file** at
+//!   startup (bit-rot detection) — it is never blindly carried forward;
+//! * a shard is **verified before it is parsed**, and a hash mismatch or
+//!   an unparseable shard routes to the full-rebuild path — a corrupt
+//!   shard is never swallowed as an empty one (which would silently drop
+//!   its documents);
+//! * changed shards are written temp + fsync + rename + directory fsync,
+//!   and the manifest is written the same way **last**, so a crash leaves
+//!   the previous, self-consistent sidecar in place (stray temp files are
+//!   ignored and cleaned).
+//!
+//! An old version-1 monolithic sidecar (one `documents` map, no
+//! `shards`) does not deserialize as a manifest, so it triggers FULL.
 
 use std::collections::BTreeMap;
+use std::fs::{File, OpenOptions};
+use std::io::Write;
 use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicUsize, Ordering};
 
 use serde::{Deserialize, Serialize};
 
-/// The on-disk sidecar.
-#[derive(Serialize, Deserialize, Default, PartialEq)]
-pub struct Sidecar {
-    /// Format version; a mismatch triggers a full rebuild.
+use super::sha256::{Sha256, hex_digest};
+
+/// Sidecar format version. A manifest with any other version is ignored
+/// and the run falls back to FULL (so a version-1 monolithic file, which
+/// additionally fails to deserialize, is never mis-parsed).
+pub const SIDECAR_VERSION: u32 = 2;
+
+/// The shard directory suffix: `<index>.sidecar.d`.
+const SHARD_DIR_SUFFIX: &str = ".sidecar.d";
+
+/// One shard's manifest entry.
+#[derive(Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub struct ShardEntry {
+    /// Filename inside the shard directory.
+    pub file: String,
+    /// Document count of the shard.
+    pub count: usize,
+    /// SHA-256 of the shard file's bytes.
+    pub content_sha256: String,
+}
+
+/// The on-disk manifest.
+#[derive(Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub struct Manifest {
     pub version: u32,
     /// How the sidecar was last written (`full` or `incremental`).
     pub mode: String,
@@ -27,11 +75,11 @@ pub struct Sidecar {
     pub rif_hash: String,
     /// Monolithic wiki cache hash at the time of writing.
     pub wiki_hash: String,
-    /// Identity term -> source hash.
-    pub documents: BTreeMap<String, String>,
+    /// Dataset label -> shard entry.
+    pub shards: BTreeMap<String, ShardEntry>,
 }
 
-/// The sidecar path for an index directory (`<index>.sidecar.json`).
+/// The manifest path for an index directory (`<index>.sidecar.json`).
 pub fn sidecar_path(index_path: &Path) -> PathBuf {
     PathBuf::from(format!(
         "{}.sidecar.json",
@@ -39,20 +87,366 @@ pub fn sidecar_path(index_path: &Path) -> PathBuf {
     ))
 }
 
-impl Sidecar {
-    pub fn load(index_path: &Path) -> Option<Sidecar> {
+/// The shard directory for an index directory (`<index>.sidecar.d`).
+pub fn shard_dir(index_path: &Path) -> PathBuf {
+    PathBuf::from(format!(
+        "{}{}",
+        index_path.to_string_lossy().trim_end_matches('/'),
+        SHARD_DIR_SUFFIX
+    ))
+}
+
+/// Sanitize a dataset label into a filesystem-safe stem: every byte
+/// outside `[A-Za-z0-9._-]` becomes `_`, leading dots are stripped, and
+/// an empty result becomes `dataset`.
+pub fn sanitize_label(label: &str) -> String {
+    let mut out = String::with_capacity(label.len());
+    for ch in label.chars() {
+        if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' || ch == '.' {
+            out.push(ch);
+        } else {
+            out.push('_');
+        }
+    }
+    while out.starts_with('.') {
+        out.remove(0);
+    }
+    if out.is_empty() {
+        out.push_str("dataset");
+    }
+    out
+}
+
+/// The shard filename for a dataset label: the sanitized stem plus a
+/// short digest of the raw label, so distinct labels cannot collide.
+pub fn shard_filename(label: &str) -> String {
+    let stem = sanitize_label(label);
+    let digest = hex_digest(label.as_bytes());
+    format!("{}-{}.json", stem, &digest[..8])
+}
+
+impl Manifest {
+    /// Load and validate the manifest. `None` when the file is missing,
+    /// unreadable, fails to deserialize (an old version-1 file, or any
+    /// other shape), or carries a version other than [`SIDECAR_VERSION`].
+    pub fn load(index_path: &Path) -> Option<Manifest> {
         let text = std::fs::read_to_string(sidecar_path(index_path)).ok()?;
-        serde_json::from_str(&text).ok()
+        let manifest: Manifest = serde_json::from_str(&text).ok()?;
+        if manifest.version != SIDECAR_VERSION {
+            return None;
+        }
+        Some(manifest)
     }
 
-    /// Atomically replace the sidecar: write a sibling temp file, sync,
-    /// then rename over the target.
+    /// Atomically replace the manifest (temp + fsync + rename + dir
+    /// fsync). Callers write changed shards first; the manifest is last.
     pub fn store(&self, index_path: &Path) -> Result<(), String> {
         let path = sidecar_path(index_path);
-        let temp = PathBuf::from(format!("{}.tmp.{}", path.display(), std::process::id()));
-        let json = serde_json::to_string(self).map_err(|e| format!("sidecar encode: {e}"))?;
-        std::fs::write(&temp, json).map_err(|e| format!("sidecar write: {e}"))?;
-        std::fs::rename(&temp, &path).map_err(|e| format!("sidecar rename: {e}"))?;
-        Ok(())
+        let json = serde_json::to_string(self).map_err(|e| format!("manifest encode: {e}"))?;
+        atomic_write(&path, json.as_bytes())
     }
 }
+
+/// Write `bytes` to `path` atomically: sibling temp file, fsync the file,
+/// rename over the target, then fsync the containing directory.
+pub fn atomic_write(path: &Path, bytes: &[u8]) -> Result<(), String> {
+    let temp = PathBuf::from(format!("{}.tmp.{}", path.display(), std::process::id()));
+    {
+        let mut file =
+            File::create(&temp).map_err(|e| format!("create {}: {e}", temp.display()))?;
+        file.write_all(bytes)
+            .map_err(|e| format!("write {}: {e}", temp.display()))?;
+        file.sync_all()
+            .map_err(|e| format!("fsync {}: {e}", temp.display()))?;
+    }
+    std::fs::rename(&temp, path).map_err(|e| format!("rename {}: {e}", path.display()))?;
+    fsync_dir(path.parent().unwrap_or_else(|| Path::new(".")))
+}
+
+/// fsync a directory so a rename into it is durable.
+pub fn fsync_dir(dir: &Path) -> Result<(), String> {
+    let handle = File::open(dir).map_err(|e| format!("open dir {}: {e}", dir.display()))?;
+    handle
+        .sync_all()
+        .map_err(|e| format!("fsync dir {}: {e}", dir.display()))
+}
+
+/// Stage a changed shard: serialize `map`, write it to a temp file in
+/// `dir`, fsync it, and return the temp path plus the manifest entry it
+/// will become after [`finalize_shard`]. The temp is **not** renamed yet,
+/// so a crash mid-run leaves the previous sidecar untouched.
+pub fn stage_shard(
+    dir: &Path,
+    label: &str,
+    map: &BTreeMap<String, String>,
+) -> Result<(PathBuf, ShardEntry), String> {
+    std::fs::create_dir_all(dir).map_err(|e| format!("create shard dir {}: {e}", dir.display()))?;
+    let file = shard_filename(label);
+    let temp = dir.join(format!(".tmp-{}-{}", std::process::id(), file));
+    let json = serde_json::to_vec(map).map_err(|e| format!("shard encode: {e}"))?;
+    {
+        let mut handle =
+            File::create(&temp).map_err(|e| format!("create shard {}: {e}", temp.display()))?;
+        handle
+            .write_all(&json)
+            .map_err(|e| format!("write shard {}: {e}", temp.display()))?;
+        handle
+            .sync_all()
+            .map_err(|e| format!("fsync shard {}: {e}", temp.display()))?;
+    }
+    let entry = ShardEntry {
+        file,
+        count: map.len(),
+        content_sha256: hex_digest(&json),
+    };
+    Ok((temp, entry))
+}
+
+/// Rename a staged shard temp into place and fsync the shard directory.
+pub fn finalize_shard(temp: &Path, dir: &Path, entry: &ShardEntry) -> Result<(), String> {
+    let target = dir.join(&entry.file);
+    std::fs::rename(temp, &target)
+        .map_err(|e| format!("rename shard {}: {e}", target.display()))?;
+    fsync_dir(dir)
+}
+
+/// Why a shard was rejected at verification time.
+#[derive(Debug)]
+pub enum ShardError {
+    /// The shard file could not be read.
+    Read(String),
+    /// The shard file's bytes hash to something other than its manifest
+    /// `content_sha256`.
+    HashMismatch { expected: String, actual: String },
+    /// The shard file hashed correctly but did not parse. Treated exactly
+    /// like a hash mismatch so a corrupt shard can never be read as empty.
+    Unparseable(String),
+}
+
+impl std::fmt::Display for ShardError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            ShardError::Read(e) => write!(f, "read: {e}"),
+            ShardError::HashMismatch { expected, actual } => {
+                write!(
+                    f,
+                    "content_sha256 mismatch (manifest {expected}, file {actual})"
+                )
+            }
+            ShardError::Unparseable(e) => write!(f, "unparseable: {e}"),
+        }
+    }
+}
+
+/// Read a shard file's bytes and check them against `entry.content_sha256`
+/// **without parsing**. This is the startup bit-rot pass: an unchanged
+/// shard's hash is recomputed from the file, never trusted blindly.
+pub fn verify_shard_bytes(dir: &Path, entry: &ShardEntry) -> Result<Vec<u8>, ShardError> {
+    let path = dir.join(&entry.file);
+    let bytes =
+        std::fs::read(&path).map_err(|e| ShardError::Read(format!("{}: {e}", path.display())))?;
+    mem::add_bytes_read(bytes.len() as u64);
+    let actual = hex_digest(&bytes);
+    if actual != entry.content_sha256 {
+        return Err(ShardError::HashMismatch {
+            expected: entry.content_sha256.clone(),
+            actual,
+        });
+    }
+    Ok(bytes)
+}
+
+/// Verify a shard and then parse it. The parse is only attempted after
+/// the hash matches, and a parse failure is surfaced as a rejection
+/// (routing to full) rather than as an empty shard.
+pub fn read_verified_shard(
+    dir: &Path,
+    entry: &ShardEntry,
+) -> Result<BTreeMap<String, String>, ShardError> {
+    let bytes = verify_shard_bytes(dir, entry)?;
+    let map: BTreeMap<String, String> = serde_json::from_slice(&bytes)
+        .map_err(|e| ShardError::Unparseable(format!("{}: {e}", entry.file)))?;
+    Ok(map)
+}
+
+/// A shard map whose entries are accounted as resident until dropped.
+/// The high-water mark proves the diff loads O(largest shard), not all
+/// shards: a RED that holds every shard alive makes the peak O(docs).
+pub struct ResidentShard {
+    /// Dataset label this shard belongs to.
+    pub label: String,
+    map: BTreeMap<String, String>,
+}
+
+impl ResidentShard {
+    pub fn load(dir: &Path, label: &str, entry: &ShardEntry) -> Result<ResidentShard, ShardError> {
+        let map = read_verified_shard(dir, entry)?;
+        mem::note_load(map.len());
+        Ok(ResidentShard {
+            label: label.to_string(),
+            map,
+        })
+    }
+
+    pub fn map(&self) -> &BTreeMap<String, String> {
+        &self.map
+    }
+}
+
+impl Drop for ResidentShard {
+    fn drop(&mut self) {
+        mem::note_drop(self.map.len());
+    }
+}
+
+/// Process-global observability counters for the sharded sidecar. They
+/// exist so "load-all" cannot quietly pass: the run asserts
+/// `max_resident_entries` is O(largest shard).
+pub mod mem {
+    use super::{AtomicUsize, Ordering};
+
+    static RESIDENT: AtomicUsize = AtomicUsize::new(0);
+    static PEAK_RESIDENT: AtomicUsize = AtomicUsize::new(0);
+    static BYTES_READ: AtomicUsize = AtomicUsize::new(0);
+    static OPEN_SHARDS: AtomicUsize = AtomicUsize::new(0);
+    static PEAK_OPEN: AtomicUsize = AtomicUsize::new(0);
+
+    /// Reset all counters before a run.
+    pub fn reset() {
+        RESIDENT.store(0, Ordering::SeqCst);
+        PEAK_RESIDENT.store(0, Ordering::SeqCst);
+        BYTES_READ.store(0, Ordering::SeqCst);
+        OPEN_SHARDS.store(0, Ordering::SeqCst);
+        PEAK_OPEN.store(0, Ordering::SeqCst);
+    }
+
+    pub(super) fn note_load(entries: usize) {
+        let resident = RESIDENT.fetch_add(entries, Ordering::SeqCst) + entries;
+        PEAK_RESIDENT.fetch_max(resident, Ordering::SeqCst);
+        let open = OPEN_SHARDS.fetch_add(1, Ordering::SeqCst) + 1;
+        PEAK_OPEN.fetch_max(open, Ordering::SeqCst);
+    }
+
+    pub(super) fn note_drop(entries: usize) {
+        RESIDENT.fetch_sub(entries, Ordering::SeqCst);
+        OPEN_SHARDS.fetch_sub(1, Ordering::SeqCst);
+    }
+
+    /// Count bytes read from shard files, whether or not parsed.
+    pub fn add_bytes_read(bytes: u64) {
+        BYTES_READ.fetch_add(bytes as usize, Ordering::SeqCst);
+    }
+
+    /// High-water mark of simultaneously resident shard-map entries.
+    pub fn peak_resident_entries() -> usize {
+        PEAK_RESIDENT.load(Ordering::SeqCst)
+    }
+
+    /// High-water mark of simultaneously open shards.
+    pub fn peak_open_shards() -> usize {
+        PEAK_OPEN.load(Ordering::SeqCst)
+    }
+
+    /// Cumulative shard-file bytes read.
+    pub fn bytes_read() -> usize {
+        BYTES_READ.load(Ordering::SeqCst)
+    }
+}
+
+/// Keep an already-loaded shard resident (the "load-all" RED). Holding
+/// every shard alive drives the peak-resident counter to O(docs).
+pub struct ResidentRetainer {
+    held: Vec<ResidentShard>,
+}
+
+impl ResidentRetainer {
+    pub fn new() -> Self {
+        ResidentRetainer { held: Vec::new() }
+    }
+
+    pub fn retain(&mut self, shard: ResidentShard) {
+        self.held.push(shard);
+    }
+}
+
+impl Default for ResidentRetainer {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Whether to hold every loaded shard alive (the load-all RED control).
+pub fn red_load_all() -> bool {
+    std::env::var("GN4_INDEXER_RED_SIDECAR").as_deref() == Ok("load-all")
+}
+
+/// Remove stray temp files a previous crash may have left: staged shard
+/// temps and stage directories, manifest temps, and half-written index
+/// temp/backup directories. They are never read.
+pub fn cleanup_temp(index_path: &Path) -> Result<(), String> {
+    let dir = shard_dir(index_path);
+    if let Ok(entries) = std::fs::read_dir(&dir) {
+        for entry in entries.flatten() {
+            let name = entry.file_name();
+            if name.to_string_lossy().starts_with(".tmp-") {
+                let _ = std::fs::remove_file(entry.path());
+            }
+        }
+    }
+    let parent = index_path.parent().unwrap_or_else(|| Path::new("."));
+    let manifest = sidecar_path(index_path);
+    let stem = manifest
+        .file_name()
+        .map(|n| n.to_string_lossy().to_string())
+        .unwrap_or_default();
+    let shard_stem = format!(
+        "{}.tmp.",
+        dir.file_name()
+            .map(|n| n.to_string_lossy().to_string())
+            .unwrap_or_default()
+    );
+    let index_stem = index_path
+        .file_name()
+        .map(|n| n.to_string_lossy().to_string())
+        .unwrap_or_default();
+    if let Ok(entries) = std::fs::read_dir(parent) {
+        for entry in entries.flatten() {
+            let name = entry.file_name();
+            let name = name.to_string_lossy();
+            if name.starts_with(&format!("{stem}.tmp.")) || name.starts_with(&shard_stem) {
+                let path = entry.path();
+                if path.is_dir() {
+                    let _ = std::fs::remove_dir_all(&path);
+                } else {
+                    let _ = std::fs::remove_file(&path);
+                }
+            }
+            // Left-over FULL index temp dirs and backups from a crash.
+            if name.starts_with(&format!(".{index_stem}.tmp."))
+                || name.starts_with(&format!(".{index_stem}.old."))
+            {
+                let _ = std::fs::remove_dir_all(entry.path());
+            }
+        }
+    }
+    Ok(())
+}
+
+/// A SHA-256 helper mirroring the sidecar's hashing of a shard file, for
+/// callers that want to (re)compute a manifest hash.
+pub fn shard_content_hash(map: &BTreeMap<String, String>) -> String {
+    let json = serde_json::to_vec(map).unwrap_or_default();
+    let mut hasher = Sha256::new();
+    hasher.update(&json);
+    hex_digest(&json)
+}
+
+/// Open a temp file for staging without rewriting an existing one.
+#[allow(dead_code)]
+fn open_new(path: &Path) -> Result<File, String> {
+    OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(path)
+        .map_err(|e| format!("create {}: {e}", path.display()))
+}