about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-10-12 19:08:38 +0200
committerPjotr Prins <pjotr.public01@thebird.nl>2026-10-12 19:08:38 +0200
commit8db38664938c4db10573efc0be04b3d12b249f41 (patch)
treea587d6ceb1d72485096b175555572822aadddb85 /src
parent9cc1631324216d564aca4448cbe71f49dbf2a4af (diff)
parentad3bb1e37e5feb22d3a8eb9e99e072f613fa186c (diff)
downloadgenenetwork4-8db38664938c4db10573efc0be04b3d12b249f41.tar.gz
Merge branch 'main' into gemma
Diffstat (limited to 'src')
-rw-r--r--src/bin/gn-indexer.rs91
-rw-r--r--src/indexer/build.rs372
-rw-r--r--src/indexer/cache.rs241
-rw-r--r--src/indexer/mod.rs32
-rw-r--r--src/indexer/pyjson.rs307
-rw-r--r--src/indexer/rows.rs229
-rw-r--r--src/indexer/run.rs319
-rw-r--r--src/indexer/schema.rs96
-rw-r--r--src/indexer/sha256.rs196
-rw-r--r--src/indexer/sidecar.rs58
-rw-r--r--src/lib.rs1
11 files changed, 1942 insertions, 0 deletions
diff --git a/src/bin/gn-indexer.rs b/src/bin/gn-indexer.rs
new file mode 100644
index 0000000..3b70945
--- /dev/null
+++ b/src/bin/gn-indexer.rs
@@ -0,0 +1,91 @@
+//! `gn-indexer`: the GeneNetwork4 native search-index builder.
+//!
+//! Usage:
+//!   gn-indexer full        [--index PATH] [--rdf-cache PATH] [--no-publish-filter]
+//!   gn-indexer incremental [--index PATH] [--rdf-cache PATH] [--no-publish-filter]
+//!
+//! The target index comes from `--index`, else `XAPIAN_DB_PATH` (the
+//! same key the search engine reads), and is written IN PLACE. The RDF
+//! caches come from `--rdf-cache`, else `GN4_RDF_CACHE`, else
+//! `/tmp/gn4-rdf-cache.json`. `SQL_URI` comes from the crate's normal
+//! layered configuration.
+//!
+//! Test-only RED controls for the incremental gates are read from
+//! `GN4_INDEXER_RED` (values `hash`, `delete`, `commit`); they sabotage
+//! one guarantee each and are never set in a normal run.
+
+use std::path::PathBuf;
+use std::process::ExitCode;
+
+use genenetwork4::config::Config;
+use genenetwork4::indexer::{RunConfig, run_full, run_incremental};
+
+fn usage() -> ExitCode {
+    eprintln!(
+        "usage: gn-indexer <full|incremental> [--index PATH] [--rdf-cache PATH] \
+         [--no-publish-filter]"
+    );
+    ExitCode::from(2)
+}
+
+fn main() -> ExitCode {
+    let mut args = std::env::args().skip(1);
+    let Some(mode) = args.next() else {
+        return usage();
+    };
+    let mut index_arg: Option<PathBuf> = None;
+    let mut rdf_arg: Option<PathBuf> = None;
+    let mut publish_filter = true;
+    while let Some(arg) = args.next() {
+        match arg.as_str() {
+            "--index" => index_arg = args.next().map(PathBuf::from),
+            "--rdf-cache" => rdf_arg = args.next().map(PathBuf::from),
+            "--no-publish-filter" => publish_filter = false,
+            other => {
+                eprintln!("unknown argument: {other}");
+                return usage();
+            }
+        }
+    }
+
+    let config = Config::load();
+    let index_path = index_arg.unwrap_or_else(|| PathBuf::from(&config.xapian_db_path));
+    if index_path.as_os_str().is_empty() {
+        eprintln!("no index path: pass --index or set XAPIAN_DB_PATH");
+        return ExitCode::from(1);
+    }
+    let rdf_cache = rdf_arg.unwrap_or_else(|| {
+        PathBuf::from(
+            std::env::var("GN4_RDF_CACHE").unwrap_or_else(|_| "/tmp/gn4-rdf-cache.json".into()),
+        )
+    });
+
+    let run_config = RunConfig {
+        sql_uri: config.sql_uri,
+        index_path,
+        rdf_cache,
+        publish_filter,
+    };
+
+    let result = match mode.as_str() {
+        "full" => run_full(&run_config),
+        "incremental" => run_incremental(&run_config),
+        _ => return usage(),
+    };
+
+    match result {
+        Ok(stats) => {
+            println!("mode={}", stats.mode);
+            println!("built={}", stats.built);
+            println!("replaced={}", stats.replaced);
+            println!("deleted={}", stats.deleted);
+            println!("unchanged={}", stats.unchanged);
+            println!("index_doc_count={}", stats.index_doc_count);
+            ExitCode::SUCCESS
+        }
+        Err(error) => {
+            eprintln!("gn-indexer: {error}");
+            ExitCode::from(1)
+        }
+    }
+}
diff --git a/src/indexer/build.rs b/src/indexer/build.rs
new file mode 100644
index 0000000..e72b0a7
--- /dev/null
+++ b/src/indexer/build.rs
@@ -0,0 +1,372 @@
+//! Document construction, mirroring `index_genes` / `index_phenotypes`
+//! in `~/project/genenetwork3/scripts/index-genenetwork` call for call.
+//!
+//! The whole point of this module is *not* to improve on the indexer:
+//! the TermGenerator configuration (English stemmer, default stemming
+//! strategy — never set, `STOP_ALL` with an empty `SimpleStopper`),
+//! the order of the `index_text` / `index_text_without_positions`
+//! calls, the prefixes, the value slots, the popped-before-storage
+//! fields and the identity term are reproduced exactly, because the
+//! twin gate compares term sets, wdf, positions, slots and docdata
+//! byte for byte.
+
+use xapian_rs::{Document, SimpleStopper, Stem, StopStrategy, TermGenerator};
+
+use super::cache::RdfCaches;
+use super::pyjson::{JsonObject, JsonValue};
+use super::rows::{GeneRow, PhenotypeRow};
+use super::schema::{self, prefix, slot};
+use super::sha256::hex_digest;
+
+/// A document ready to write, its identity term and its source hash.
+pub struct BuiltDocument {
+    pub idterm: String,
+    pub document: Document,
+    pub source_hash: String,
+}
+
+/// The writer's TermGenerator, configured exactly as the indexer.
+fn writer_termgen() -> TermGenerator {
+    let mut termgen = TermGenerator::default();
+    termgen.set_stemmer(Stem::for_language("en"));
+    termgen.set_stopper_strategy(StopStrategy::All);
+    termgen.set_stopper(SimpleStopper::new());
+    termgen
+}
+
+/// Canonical encoder for a document's complete source, used for the
+/// incremental change hash. Deterministic and delimiter-safe.
+#[derive(Default)]
+struct Canon {
+    buffer: Vec<u8>,
+}
+
+impl Canon {
+    fn field(&mut self, key: &str, value: &str) {
+        self.buffer.extend_from_slice(key.as_bytes());
+        self.buffer.push(b'=');
+        self.buffer
+            .extend_from_slice(value.len().to_string().as_bytes());
+        self.buffer.push(b':');
+        self.buffer.extend_from_slice(value.as_bytes());
+        self.buffer.push(b'\n');
+    }
+
+    fn opt_str(&mut self, key: &str, value: &Option<String>) {
+        match value {
+            Some(text) => self.field(key, text),
+            None => {
+                self.buffer.extend_from_slice(key.as_bytes());
+                self.buffer.extend_from_slice(b"=N\n");
+            }
+        }
+    }
+
+    fn opt_f64(&mut self, key: &str, value: &Option<f64>) {
+        match value {
+            Some(number) => self.field(key, &format!("{:016x}", number.to_bits())),
+            None => {
+                self.buffer.extend_from_slice(key.as_bytes());
+                self.buffer.extend_from_slice(b"=N\n");
+            }
+        }
+    }
+
+    fn opt_i64(&mut self, key: &str, value: &Option<i64>) {
+        match value {
+            Some(number) => self.field(key, &number.to_string()),
+            None => {
+                self.buffer.extend_from_slice(key.as_bytes());
+                self.buffer.extend_from_slice(b"=N\n");
+            }
+        }
+    }
+}
+
+/// Build a gene document. `None` when name or dataset is absent (the
+/// indexer then writes no document at all).
+pub fn build_gene(row: &GeneRow, caches: &RdfCaches) -> Option<BuiltDocument> {
+    let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?);
+
+    let mut doc = Document::default();
+    {
+        let mut termgen = writer_termgen();
+        termgen.set_document(&doc);
+
+        // Values (slots 0,1,2,3,4).
+        if let Some(mean) = row.mean {
+            doc.set_value(slot::MEAN, mean);
+        }
+        if let Some(lrs) = row.lrs {
+            doc.set_value(slot::PEAK, lrs);
+        }
+        if let Some(mb) = row.mb {
+            doc.set_value(slot::POSITION, mb);
+        }
+        if let Some(geno_mb) = row.geno_mb {
+            doc.set_value(slot::PEAK_MB, geno_mb);
+        }
+        if let Some(additive) = row.additive {
+            doc.set_value(slot::ADDITIVE, additive);
+        }
+
+        // Free text WITH positions.
+        for text in [&row.description, &row.tissue, &row.dataset] {
+            if let Some(text) = text {
+                termgen.index_text::<&str>(text, None, None);
+                termgen.increase_termpos(None);
+            }
+        }
+        if let Some(text) = &row.probe_target_description {
+            termgen.index_text::<&str>(text, None, None);
+            termgen.increase_termpos(None);
+        }
+        // Free text WITHOUT positions, unprefixed.
+        for text in [&row.name, &row.symbol, &row.species, &row.group] {
+            if let Some(text) = text {
+                termgen.index_text_without_positions::<&str>(text, None, None);
+            }
+        }
+        for text in [&row.alias, &row.genbankid, &row.unigeneid] {
+            if let Some(text) = text {
+                termgen.index_text_without_positions::<&str>(text, None, None);
+            }
+        }
+        // Prefixed: WITH positions for XI/XDS, without for the rest.
+        if let Some(text) = &row.species {
+            termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::SPECIES));
+        }
+        if let Some(text) = &row.group {
+            termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::GROUP));
+        }
+        if let Some(text) = &row.tissue {
+            termgen.index_text::<&str>(text, Some(0u32), Some(prefix::TISSUE));
+        }
+        if let Some(text) = &row.dataset {
+            termgen.index_text::<&str>(text, Some(0u32), Some(prefix::DATASET));
+        }
+        if let Some(text) = &row.symbol {
+            termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::SYMBOL));
+        }
+        if let Some(text) = &row.chr {
+            termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::CHR));
+        }
+        if let Some(text) = &row.geno_chr {
+            termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::PEAK_CHR));
+        }
+        // RDF dictionaries (XRF / XWK), keyed on (species, symbol).
+        if let (Some(species), Some(symbol)) = (&row.species, &row.symbol) {
+            if let Some(text) = caches.rif.get(species, symbol) {
+                termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::RIF));
+            }
+            if let Some(text) = caches.wiki.get(species, symbol) {
+                termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::WIKI));
+            }
+        }
+    }
+
+    // Stored data: insertion order minus the popped fields.
+    let mut data = JsonObject::new();
+    push_opt_str(&mut data, "name", &row.name);
+    push_opt_str(&mut data, "symbol", &row.symbol);
+    push_opt_str(&mut data, "description", &row.description);
+    push_opt_str(&mut data, "chr", &row.chr);
+    push_opt_f64(&mut data, "mb", &row.mb);
+    push_opt_str(&mut data, "dataset", &row.dataset);
+    push_opt_str(&mut data, "dataset_fullname", &row.dataset_fullname);
+    push_opt_str(&mut data, "species", &row.species);
+    push_opt_str(&mut data, "group", &row.group);
+    push_opt_str(&mut data, "tissue", &row.tissue);
+    push_opt_f64(&mut data, "mean", &row.mean);
+    push_opt_f64(&mut data, "lrs", &row.lrs);
+    push_opt_f64(&mut data, "additive", &row.additive);
+    push_opt_str(&mut data, "geno_chr", &row.geno_chr);
+    push_opt_f64(&mut data, "geno_mb", &row.geno_mb);
+    doc.set_data(data.to_string());
+
+    let idterm = schema::identity_term(schema::DOCTYPE_GENE, name, dataset);
+    doc.add_boolean_term(schema::type_term(schema::DOCTYPE_GENE));
+    doc.add_boolean_term(&idterm);
+
+    // Source hash: full row + the monolithic cache hashes.
+    let mut canon = Canon::default();
+    canon.opt_str("name", &row.name);
+    canon.opt_str("symbol", &row.symbol);
+    canon.opt_str("description", &row.description);
+    canon.opt_str("chr", &row.chr);
+    canon.opt_f64("mb", &row.mb);
+    canon.opt_str("alias", &row.alias);
+    canon.opt_str("genbankid", &row.genbankid);
+    canon.opt_str("unigeneid", &row.unigeneid);
+    canon.opt_str("probe_target_description", &row.probe_target_description);
+    canon.opt_str("dataset", &row.dataset);
+    canon.opt_str("dataset_fullname", &row.dataset_fullname);
+    canon.opt_str("species", &row.species);
+    canon.opt_str("group", &row.group);
+    canon.opt_str("tissue", &row.tissue);
+    canon.opt_f64("mean", &row.mean);
+    canon.opt_f64("lrs", &row.lrs);
+    canon.opt_f64("additive", &row.additive);
+    canon.opt_str("geno_chr", &row.geno_chr);
+    canon.opt_f64("geno_mb", &row.geno_mb);
+    canon.field("rif-cache", &caches.rif.hash);
+    canon.field("wiki-cache", &caches.wiki.hash);
+
+    Some(BuiltDocument {
+        idterm,
+        document: doc,
+        source_hash: hex_digest(&canon.buffer),
+    })
+}
+
+/// Build a phenotype document.
+pub fn build_phenotype(row: &PhenotypeRow) -> Option<BuiltDocument> {
+    let (name, dataset) = (row.name.as_ref()?, row.dataset.as_ref()?);
+    // Python converts the integer name to str for the document (and for
+    // the identity term's identifier).
+    let name_text = name.to_string();
+
+    // The raw comma-separated authors string is indexed before storage.
+    let authors_raw = row.authors.clone();
+    let authors_list: Option<Vec<String>> = authors_raw
+        .as_ref()
+        .map(|text| text.split(',').map(|a| a.trim().to_string()).collect());
+
+    let mut doc = Document::default();
+    {
+        let mut termgen = writer_termgen();
+        termgen.set_document(&doc);
+
+        // Values (slots 0,1,3,4,5,6 — no position slot).
+        if let Some(mean) = row.mean {
+            doc.set_value(slot::MEAN, mean);
+        }
+        if let Some(lrs) = row.lrs {
+            doc.set_value(slot::PEAK, lrs);
+        }
+        if let Some(geno_mb) = row.geno_mb {
+            doc.set_value(slot::PEAK_MB, geno_mb);
+        }
+        if let Some(additive) = row.additive {
+            doc.set_value(slot::ADDITIVE, additive);
+        }
+        if let Some(year) = row.year {
+            doc.set_value(slot::YEAR, year as f64);
+        }
+        if let Some(nsamples) = row.nsamples {
+            doc.set_value(slot::NSAMPLES, nsamples as f64);
+        }
+
+        // Free text WITH positions.
+        for text in [&row.description, &authors_raw, &row.dataset] {
+            if let Some(text) = text {
+                termgen.index_text::<&str>(text, None, None);
+                termgen.increase_termpos(None);
+            }
+        }
+        for text in [&row.abstract_, &row.title] {
+            if let Some(text) = text {
+                termgen.index_text::<&str>(text, None, None);
+                termgen.increase_termpos(None);
+            }
+        }
+        // Free text WITHOUT positions, unprefixed.
+        for text in [&row.species, &row.group, &row.inbredsetcode] {
+            if let Some(text) = text {
+                termgen.index_text_without_positions::<&str>(text, None, None);
+            }
+        }
+        for text in [&row.abbreviation, &row.lab_code] {
+            if let Some(text) = text {
+                termgen.index_text_without_positions::<&str>(text, None, None);
+            }
+        }
+        // Prefixed: WITH positions for A/XDS.
+        if let Some(text) = &row.species {
+            termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::SPECIES));
+        }
+        if let Some(text) = &row.group {
+            termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::GROUP));
+        }
+        if let Some(text) = &authors_raw {
+            termgen.index_text::<&str>(text, Some(0u32), Some(prefix::AUTHOR));
+        }
+        if let Some(text) = &row.geno_chr {
+            termgen.index_text_without_positions::<&str>(text, Some(0u32), Some(prefix::PEAK_CHR));
+        }
+        if let Some(text) = &row.dataset {
+            termgen.index_text::<&str>(text, Some(0u32), Some(prefix::DATASET));
+        }
+    }
+
+    let mut data = JsonObject::new();
+    push_opt_str(&mut data, "species", &row.species);
+    push_opt_str(&mut data, "group", &row.group);
+    push_opt_str(&mut data, "dataset", &row.dataset);
+    push_opt_str(&mut data, "dataset_fullname", &row.dataset_fullname);
+    data.push("name", JsonValue::Str(name_text.clone()));
+    push_opt_str(&mut data, "description", &row.description);
+    if let Some(authors) = &authors_list {
+        data.push("authors", JsonValue::StrList(authors.clone()));
+    }
+    push_opt_i64(&mut data, "year", &row.year);
+    push_opt_i64(&mut data, "pubmed_id", &row.pubmed_id);
+    push_opt_f64(&mut data, "lrs", &row.lrs);
+    push_opt_f64(&mut data, "additive", &row.additive);
+    push_opt_str(&mut data, "inbredsetcode", &row.inbredsetcode);
+    push_opt_f64(&mut data, "mean", &row.mean);
+    push_opt_i64(&mut data, "nsamples", &row.nsamples);
+    push_opt_str(&mut data, "geno_chr", &row.geno_chr);
+    push_opt_f64(&mut data, "geno_mb", &row.geno_mb);
+    doc.set_data(data.to_string());
+
+    let idterm = schema::identity_term(schema::DOCTYPE_PHENOTYPE, &name_text, dataset);
+    doc.add_boolean_term(schema::type_term(schema::DOCTYPE_PHENOTYPE));
+    doc.add_boolean_term(&idterm);
+
+    let mut canon = Canon::default();
+    canon.opt_str("species", &row.species);
+    canon.opt_str("group", &row.group);
+    canon.opt_str("dataset", &row.dataset);
+    canon.opt_str("dataset_fullname", &row.dataset_fullname);
+    canon.opt_i64("name", &row.name);
+    canon.opt_str("abbreviation", &row.abbreviation);
+    canon.opt_str("description", &row.description);
+    canon.opt_str("Lab_code", &row.lab_code);
+    canon.opt_str("Abstract", &row.abstract_);
+    canon.opt_str("Title", &row.title);
+    canon.opt_str("authors", &row.authors);
+    canon.opt_i64("year", &row.year);
+    canon.opt_i64("pubmed_id", &row.pubmed_id);
+    canon.opt_f64("lrs", &row.lrs);
+    canon.opt_f64("additive", &row.additive);
+    canon.opt_str("inbredsetcode", &row.inbredsetcode);
+    canon.opt_f64("mean", &row.mean);
+    canon.opt_i64("nsamples", &row.nsamples);
+    canon.opt_str("geno_chr", &row.geno_chr);
+    canon.opt_f64("geno_mb", &row.geno_mb);
+
+    Some(BuiltDocument {
+        idterm,
+        document: doc,
+        source_hash: hex_digest(&canon.buffer),
+    })
+}
+
+fn push_opt_str(object: &mut JsonObject, key: &str, value: &Option<String>) {
+    if let Some(text) = value {
+        object.push(key, JsonValue::Str(text.clone()));
+    }
+}
+
+fn push_opt_f64(object: &mut JsonObject, key: &str, value: &Option<f64>) {
+    if let Some(number) = value {
+        object.push(key, JsonValue::Float(*number));
+    }
+}
+
+fn push_opt_i64(object: &mut JsonObject, key: &str, value: &Option<i64>) {
+    if let Some(number) = value {
+        object.push(key, JsonValue::Int(*number));
+    }
+}
diff --git a/src/indexer/cache.rs b/src/indexer/cache.rs
new file mode 100644
index 0000000..0145e43
--- /dev/null
+++ b/src/indexer/cache.rs
@@ -0,0 +1,241 @@
+//! The RDF-derived dictionaries (RIF and wiki), reproducing the
+//! indexer's `build_rdf_cache(..., remove_common_words=True)`.
+//!
+//! The indexer fetches this text over SPARQL; the lane replays it from
+//! the authoritative cache file written during the reference build
+//! (`~/spikes/index-self-build/rdf-cache.json`, produced by the
+//! batched patched script — the cache format is defined by
+//! `index-genenetwork-batched`, not by live SPARQL).
+//!
+//! Each cache is treated as ONE monolithic hashed input: a change to
+//! either dictionary re-derives every gene document (the design's
+//! "cache as one input", not per-symbol attribution).
+
+use std::collections::HashMap;
+use std::path::Path;
+
+use serde_json::Value;
+
+use super::sha256::hex_digest;
+
+/// Species batching order, verbatim from `index-genenetwork-batched`
+/// (`ALL_SPECIES`). Dictionary values are keyed on
+/// `(speciesName, symbolName)` and the word counter's first-seen order
+/// follows this order.
+pub const ALL_SPECIES: [&str; 14] = [
+    "arabidopsis",
+    "barley",
+    "bat",
+    "drosophila",
+    "human",
+    "medaka",
+    "monkey",
+    "mouse",
+    "poplar",
+    "rat",
+    "soybean",
+    "tomato",
+    "macaca",
+    "Nematode",
+];
+
+/// One derived dictionary plus its monolithic content hash.
+pub struct Dictionary {
+    entries: HashMap<(String, String), String>,
+    pub hash: String,
+}
+
+impl Dictionary {
+    /// The dictionary text for `(species, symbol)`, if any.
+    pub fn get(&self, species: &str, symbol: &str) -> Option<&str> {
+        self.entries
+            .get(&(species.to_string(), symbol.to_string()))
+            .map(String::as_str)
+    }
+}
+
+/// Both RDF dictionaries (RIF, wiki) plus their content hashes.
+pub struct RdfCaches {
+    pub rif: Dictionary,
+    pub wiki: Dictionary,
+}
+
+impl RdfCaches {
+    /// Load and derive both dictionaries from the cache file.
+    pub fn load(path: &Path) -> Result<RdfCaches, String> {
+        let text = std::fs::read_to_string(path)
+            .map_err(|e| format!("cannot read RDF cache {}: {e}", path.display()))?;
+        let store: Value = serde_json::from_str(&text)
+            .map_err(|e| format!("RDF cache {} is not valid JSON: {e}", path.display()))?;
+        Ok(RdfCaches {
+            rif: derive(&store, "rif")?,
+            wiki: derive(&store, "wiki")?,
+        })
+    }
+}
+
+/// Indexer word tokenisation: Python `re.compile(r"\w+")` with a
+/// `str` subject (Unicode word characters: alphanumerics plus `_`).
+/// Callers join the runs with single spaces.
+fn tokenize_words(text: &str) -> Vec<&str> {
+    let mut tokens = Vec::new();
+    let mut start: Option<usize> = None;
+    for (index, ch) in text.char_indices() {
+        let is_word = ch.is_alphanumeric() || ch == '_';
+        match (is_word, start) {
+            (true, None) => start = Some(index),
+            (false, Some(begin)) => {
+                tokens.push(&text[begin..index]);
+                start = None;
+            }
+            _ => {}
+        }
+    }
+    if let Some(begin) = start {
+        tokens.push(&text[begin..]);
+    }
+    tokens
+}
+
+/// Derive one dictionary from the parsed cache store.
+fn derive(store: &Value, kind: &str) -> Result<Dictionary, String> {
+    let mut cache: HashMap<(String, String), String> = HashMap::new();
+    // Word counts with first-seen insertion order (Python Counter).
+    let mut counts: HashMap<String, u64> = HashMap::new();
+    let mut order: Vec<String> = Vec::new();
+    let kind_store = store
+        .get(kind)
+        .and_then(Value::as_object)
+        .ok_or_else(|| format!("RDF cache has no `{kind}` section"))?;
+    for species in ALL_SPECIES {
+        let Some(bindings) = kind_store.get(species).and_then(Value::as_array) else {
+            continue;
+        };
+        for binding in bindings {
+            let species_name = binding
+                .get("speciesName")
+                .and_then(|v| v.get("value"))
+                .and_then(Value::as_str)
+                .ok_or("binding missing speciesName.value")?;
+            let symbol_name = binding
+                .get("symbolName")
+                .and_then(|v| v.get("value"))
+                .and_then(Value::as_str)
+                .ok_or("binding missing symbolName.value")?;
+            let comment = binding
+                .get("comment")
+                .and_then(|v| v.get("value"))
+                .and_then(Value::as_str)
+                .ok_or("binding missing comment.value")?;
+            let value = tokenize_words(comment).join(" ");
+            cache.insert(
+                (species_name.to_string(), symbol_name.to_string()),
+                value.clone(),
+            );
+            // Counter.update(value.lower().strip().split())
+            for word in value.to_lowercase().split_whitespace() {
+                match counts.get_mut(word) {
+                    Some(count) => *count += 1,
+                    None => {
+                        counts.insert(word.to_string(), 1);
+                        order.push(word.to_string());
+                    }
+                }
+            }
+        }
+    }
+
+    // remove_common_words: top-1000 by count (ties by first-seen order),
+    // then drop words with len<4 or cnt>3000.
+    let mut items: Vec<(String, u64)> = order
+        .iter()
+        .map(|word| (word.clone(), counts[word]))
+        .collect();
+    // Stable sort by count descending == heapq.nlargest(1000, ...) set
+    // (verified against CPython on the authoritative cache; see
+    // logs/cache-nlargest-check.log).
+    items.sort_by(|a, b| b.1.cmp(&a.1));
+    let mut dropped: std::collections::HashSet<&str> = std::collections::HashSet::new();
+    for (word, count) in items.iter().take(1000) {
+        if word.chars().count() < 4 || *count > 3000 {
+            dropped.insert(word);
+        }
+    }
+
+    let mut entries: HashMap<(String, String), String> = HashMap::new();
+    for (key, value) in cache {
+        let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new();
+        let mut words: Vec<&str> = Vec::new();
+        let lowered = value.to_lowercase();
+        for word in lowered.split_whitespace() {
+            if dropped.contains(word) || !seen.insert(word) {
+                continue;
+            }
+            words.push(word);
+        }
+        entries.insert(key, words.join(" "));
+    }
+
+    let hash = content_hash(&entries);
+    Ok(Dictionary { entries, hash })
+}
+
+/// Monolithic content hash: a canonical, order-independent
+/// serialization of the derived dictionary.
+fn content_hash(entries: &HashMap<(String, String), String>) -> String {
+    let mut keys: Vec<&(String, String)> = entries.keys().collect();
+    keys.sort();
+    let mut buffer: Vec<u8> = Vec::new();
+    for key in keys {
+        buffer.extend_from_slice(key.0.as_bytes());
+        buffer.push(0);
+        buffer.extend_from_slice(key.1.as_bytes());
+        buffer.push(0);
+        buffer.extend_from_slice(entries[key].as_bytes());
+        buffer.push(1);
+    }
+    hex_digest(&buffer)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn punctuation_strip_joins_word_runs() {
+        assert_eq!(tokenize_words("hello, world!").join(" "), "hello world");
+        assert_eq!(
+            tokenize_words("beta-1,4-galactosyl").join(" "),
+            "beta 1 4 galactosyl"
+        );
+        assert_eq!(tokenize_words("a_b c").join(" "), "a_b c");
+    }
+
+    #[test]
+    fn remove_common_words_matches_reference_slice() {
+        // Mirrors derive-dictionary.py for (mouse, Cd3g):
+        // rif='identifies cd3gamma infiltrating animals tcr graft cd3cd4'.
+        let json = std::fs::read_to_string("/home/kabui/spikes/index-self-build/rdf-cache.json");
+        let Ok(json) = json else {
+            eprintln!("skipping: authoritative RDF cache not present");
+            return;
+        };
+        let store: Value = serde_json::from_str(&json).unwrap();
+        let rif = derive(&store, "rif").unwrap();
+        // The dictionary text is fed to `index_text_without_positions`
+        // (positions off, wdf increment 0), so only its word SET
+        // reaches the index; Python's `" ".join(set(...))` order is
+        // irrelevant and deliberately not reproduced.
+        let mut got: Vec<&str> = rif
+            .get("mouse", "Cd3g")
+            .expect("mouse|Cd3g present")
+            .split_whitespace()
+            .collect();
+        got.sort_unstable();
+        let mut want: Vec<&str> = "identifies cd3gamma infiltrating animals tcr graft cd3cd4"
+            .split_whitespace()
+            .collect();
+        want.sort_unstable();
+        assert_eq!(got, want);
+    }
+}
diff --git a/src/indexer/mod.rs b/src/indexer/mod.rs
new file mode 100644
index 0000000..7486feb
--- /dev/null
+++ b/src/indexer/mod.rs
@@ -0,0 +1,32 @@
+//! The native indexer: crawl, filter, detect, update in place.
+//!
+//! This module tree is the Rust port of gn3's Python indexer
+//! (`~/project/genenetwork3/scripts/index-genenetwork`) described by
+//! `~/gn-gemtext-threads/topics/gn4/gn4-search-indexing.gmi`. It shares
+//! the crate's configuration (`SQL_URI`, `XAPIAN_DB_PATH`) with the
+//! native search engine and writes documents into the live index by
+//! their stable identity term, so a per-document update is visible with
+//! no service stop.
+//!
+//! Two modes, one mechanism:
+//!
+//! * FULL (bootstrap): crawl every public, non-confidential freeze and
+//!   build a fresh index. A missing or empty sidecar means FULL.
+//! * INCREMENTAL: crawl, hash each document's complete source, diff
+//!   against the sidecar, replace only changed documents, delete
+//!   vanished identity terms, and write nothing when nothing changed.
+//!
+//! The phenotype crawl carries `confidentiality < 1 AND public > 0`
+//! (the Python query has no WHERE clause, which is the metadata leak
+//! this module closes by construction).
+
+pub mod build;
+pub mod cache;
+pub mod pyjson;
+pub mod rows;
+pub mod run;
+pub mod schema;
+pub mod sha256;
+pub mod sidecar;
+
+pub use run::{RunConfig, Stats, run_full, run_incremental};
diff --git a/src/indexer/pyjson.rs b/src/indexer/pyjson.rs
new file mode 100644
index 0000000..01519d1
--- /dev/null
+++ b/src/indexer/pyjson.rs
@@ -0,0 +1,307 @@
+//! Byte-exact reproduction of Python's `json.dumps` for the indexer's
+//! document data.
+//!
+//! The reference index stores `json.dumps(trait.data)` (see
+//! `~/project/genenetwork3/scripts/index-genenetwork`). The twin gate
+//! compares document data byte-for-byte, so serde_json's defaults are
+//! not usable: Python emits `", "` / `": "` separators, keeps dict
+//! insertion order, escapes every non-ASCII character as `\uXXXX`
+//! (`ensure_ascii=True`), and formats floats with `repr` (shortest
+//! round-trip, fixed notation for decimal exponents in `[-4, 16)` and
+//! scientific `d.ddde±XX` outside it).
+//!
+//! [`JsonObject`] builds one document's data in insertion order from
+//! typed values; [`escape_string`] and [`float_repr`] are the two
+//! primitives that must match CPython exactly.
+
+use std::fmt::Write as _;
+
+/// One value in a document-data object.
+pub enum JsonValue {
+    Str(String),
+    Int(i64),
+    Float(f64),
+    StrList(Vec<String>),
+}
+
+/// Builds a JSON object in insertion order with Python's separators.
+#[derive(Default)]
+pub struct JsonObject {
+    entries: Vec<(String, JsonValue)>,
+}
+
+impl JsonObject {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Append `key: value`; insertion order is the output order.
+    pub fn push(&mut self, key: &str, value: JsonValue) {
+        self.entries.push((key.to_string(), value));
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.entries.is_empty()
+    }
+
+    /// Render the object exactly as `json.dumps` would.
+    pub fn to_string(&self) -> String {
+        let mut out = String::new();
+        out.push('{');
+        for (index, (key, value)) in self.entries.iter().enumerate() {
+            if index > 0 {
+                out.push_str(", ");
+            }
+            out.push('"');
+            escape_string(key, &mut out);
+            out.push_str("\": ");
+            write_value(value, &mut out);
+        }
+        out.push('}');
+        out
+    }
+}
+
+fn write_value(value: &JsonValue, out: &mut String) {
+    match value {
+        JsonValue::Str(text) => {
+            out.push('"');
+            escape_string(text, out);
+            out.push('"');
+        }
+        JsonValue::Int(number) => {
+            let _ = write!(out, "{number}");
+        }
+        JsonValue::Float(number) => out.push_str(&float_repr(*number)),
+        JsonValue::StrList(items) => {
+            out.push('[');
+            for (index, item) in items.iter().enumerate() {
+                if index > 0 {
+                    out.push_str(", ");
+                }
+                out.push('"');
+                escape_string(item, out);
+                out.push('"');
+            }
+            out.push(']');
+        }
+    }
+}
+
+/// Escape a string exactly as Python's `json.dumps` with
+/// `ensure_ascii=True`: the two-character escapes for the standard
+/// control characters, `\u00XX` for other control characters, and
+/// `\uXXXX` (a surrogate pair above the BMP) for every non-ASCII
+/// character.
+pub fn escape_string(text: &str, out: &mut String) {
+    for ch in text.chars() {
+        match ch {
+            '"' => out.push_str("\\\""),
+            '\\' => out.push_str("\\\\"),
+            '\n' => out.push_str("\\n"),
+            '\r' => out.push_str("\\r"),
+            '\t' => out.push_str("\\t"),
+            '\u{08}' => out.push_str("\\b"),
+            '\u{0c}' => out.push_str("\\f"),
+            c if (c as u32) < 0x20 => {
+                let _ = write!(out, "\\u{:04x}", c as u32);
+            }
+            c if (c as u32) < 0x7f => out.push(c),
+            c => {
+                // ensure_ascii: BMP code units as \uXXXX, astral as a
+                // UTF-16 surrogate pair.
+                let mut buf = [0u16; 2];
+                for unit in c.encode_utf16(&mut buf) {
+                    let _ = write!(out, "\\u{:04x}", *unit);
+                }
+            }
+        }
+    }
+}
+
+/// Format an `f64` exactly as CPython's `repr` (and therefore
+/// `json.dumps`).
+///
+/// CPython uses the shortest decimal string that round-trips, rounded
+/// to nearest with ties to even, then chooses fixed notation when the
+/// decimal exponent `e` (value = `d.ddd × 10^e`) satisfies
+/// `-4 <= e < 16`, and scientific `d.ddde±XX` otherwise.
+///
+/// Rust's `{:e}` is shortest but rounds ties away from zero (e.g.
+/// `859.44000244140625` becomes `859.4400024414063`, CPython gives
+/// `859.4400024414062`). Instead, search for the smallest significant
+/// digit count whose round-half-even rendering round-trips; that is
+/// exactly CPython's choice, and the reference docdata has such ties.
+pub fn float_repr(value: f64) -> String {
+    if value == 0.0 {
+        // Handles +0.0 and -0.0 (repr(-0.0) == "-0.0").
+        return if value.is_sign_negative() {
+            "-0.0".to_string()
+        } else {
+            "0.0".to_string()
+        };
+    }
+    let mut rendered = None;
+    for significant in 1..=17u32 {
+        let candidate = format!("{:.*e}", (significant - 1) as usize, value);
+        if candidate.parse::<f64>() == Ok(value) {
+            rendered = Some(candidate);
+            break;
+        }
+    }
+    let rendered = rendered.expect("17 significant digits always round-trip an f64");
+    let (mantissa, exponent) = rendered
+        .split_once('e')
+        .expect("LowerExp output always contains 'e'");
+    let exponent: i32 = exponent.parse().expect("LowerExp exponent is an integer");
+
+    let (sign, mantissa) = match mantissa.strip_prefix('-') {
+        Some(rest) => ("-", rest),
+        None => ("", mantissa),
+    };
+    let digits: String = mantissa.chars().filter(|c| *c != '.').collect();
+
+    let mut out = String::new();
+    out.push_str(sign);
+    if (-4..16).contains(&exponent) {
+        if exponent >= 0 {
+            let integer_digits = exponent as usize + 1;
+            if digits.len() > integer_digits {
+                out.push_str(&digits[..integer_digits]);
+                out.push('.');
+                out.push_str(&digits[integer_digits..]);
+            } else {
+                out.push_str(&digits);
+                for _ in digits.len()..integer_digits {
+                    out.push('0');
+                }
+                out.push_str(".0");
+            }
+        } else {
+            out.push_str("0.");
+            for _ in 0..(-exponent - 1) {
+                out.push('0');
+            }
+            out.push_str(&digits);
+        }
+    } else {
+        out.push_str(&digits[..1]);
+        if digits.len() > 1 {
+            out.push('.');
+            out.push_str(&digits[1..]);
+        }
+        out.push('e');
+        out.push(if exponent < 0 { '-' } else { '+' });
+        let magnitude = exponent.unsigned_abs();
+        if magnitude < 10 {
+            let _ = write!(out, "0{magnitude}");
+        } else {
+            let _ = write!(out, "{magnitude}");
+        }
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Values sampled verbatim from the reference index docdata
+    /// (`logs/probe-docdata.log`, `logs/scan-docdata.log`): the exact
+    /// byte forms Python produced, including the float edge shapes
+    /// (integral floats keep `.0`, tiny values go scientific).
+    #[test]
+    fn float_repr_matches_reference_bytes() {
+        let cases: &[(f64, &str)] = &[
+            (52.13529418496525, "52.13529418496525"),
+            (44.970689, "44.970689"),
+            (8.14033666666667, "8.14033666666667"),
+            (22.004269639323, "22.004269639323"),
+            (0.0, "0.0"),
+            (-0.0, "-0.0"),
+            (1.0, "1.0"),
+            (20.0, "20.0"),
+            (460.0, "460.0"),
+            (-17.0, "-17.0"),
+            (2.39444435069444, "2.39444435069444"),
+            (-3844.23775308921, "-3844.23775308921"),
+            (6350.554222735148, "6350.554222735148"),
+            // exponent shapes observed in reference docdata
+            (6.031019218040234e-08, "6.031019218040234e-08"),
+            (-2.977410049149484e-09, "-2.977410049149484e-09"),
+            (-1.4110948100234523e-11, "-1.4110948100234523e-11"),
+            (-2.8504644121442524e-05, "-2.8504644121442524e-05"),
+            (3.2393828682277513e-10, "3.2393828682277513e-10"),
+            // boundary shapes
+            (1e15, "1000000000000000.0"),
+            (1e16, "1e+16"),
+            (1e-4, "0.0001"),
+            (1e-5, "1e-05"),
+            (1.5e300, "1.5e+300"),
+            // observed tie cases where Rust's `{:e}` rounds the wrong
+            // way; CPython rounds half to even
+            (859.44000244140625, "859.4400024414062"),
+            (819.85333251953125, "819.8533325195312"),
+            (636.80230712890625, "636.8023071289062"),
+            (13.4032135009765625, "13.403213500976562"),
+            (3709.29327392578125, "3709.2932739257812"),
+            (8.1207427978515625, "8.120742797851562"),
+        ];
+        for (value, expected) in cases {
+            assert_eq!(float_repr(*value), *expected, "for {value:?}");
+        }
+    }
+
+    /// Every distinct float token shape from the reference docdata
+    /// (all exponent and `.0` forms, plus a deterministic plain
+    /// sample): parsing each token back to f64 and re-rendering must
+    /// reproduce the exact token bytes. Fixture generated read-only
+    /// from the reference index (`scripts/gen-float-fixture.py`,
+    /// `logs/gen-float-fixture.log`).
+    #[test]
+    fn float_repr_matches_reference_fixture() {
+        let path = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/pyfloat.txt");
+        let Ok(text) = std::fs::read_to_string(path) else {
+            eprintln!("skipping: float fixture not present at {path}");
+            return;
+        };
+        let mut checked = 0usize;
+        for line in text.lines() {
+            let token = line.trim();
+            if token.is_empty() {
+                continue;
+            }
+            let value: f64 = token.parse().expect("fixture token is a float");
+            assert_eq!(float_repr(value), token, "for {value:?}");
+            checked += 1;
+        }
+        assert!(checked > 1000, "fixture unexpectedly small: {checked}");
+    }
+
+    #[test]
+    fn escape_string_matches_python() {
+        let mut out = String::new();
+        escape_string("a\"b\\c\n\t\r\u{08}\u{0c}\u{01}", &mut out);
+        assert_eq!(out, "a\\\"b\\\\c\\n\\t\\r\\b\\f\\u0001");
+        out.clear();
+        escape_string("parré", &mut out);
+        assert_eq!(out, "parr\\u00e9");
+    }
+
+    #[test]
+    fn object_matches_python_layout() {
+        let mut object = JsonObject::new();
+        object.push("species", JsonValue::Str("mouse".into()));
+        object.push("year", JsonValue::Int(2001));
+        object.push("mean", JsonValue::Float(52.13529418496525));
+        object.push(
+            "authors",
+            JsonValue::StrList(vec!["Airey DC".into(), "Lu L".into()]),
+        );
+        assert_eq!(
+            object.to_string(),
+            "{\"species\": \"mouse\", \"year\": 2001, \"mean\": 52.13529418496525, \
+             \"authors\": [\"Airey DC\", \"Lu L\"]}"
+        );
+    }
+}
diff --git a/src/indexer/rows.rs b/src/indexer/rows.rs
new file mode 100644
index 0000000..b8669e1
--- /dev/null
+++ b/src/indexer/rows.rs
@@ -0,0 +1,229 @@
+//! The crawl: the SQL queries and typed row structs the document
+//! builder consumes.
+//!
+//! The queries and joins are the indexer's own (`genes_query` and
+//! `phenotypes_query` in index-genenetwork), rendered by its
+//! `serialize_sql`. The one deliberate difference is the phenotype
+//! filter: the Python query has NO `WHERE` clause, which is the leak
+//! this lane closes by construction — both queries now carry
+//! `confidentiality < 1 AND public > 0`.
+//!
+//! The phenotype join fans out on `InbredSetId` only, so a group's two
+//! PublishFreezes (e.g. BXDPublish and GITrMetPublish) each index every
+//! trait under their own dataset label. This cross-product is INTENTIONAL
+//! reference semantics and is reproduced, not "cleaned up".
+
+use mysql::prelude::Queryable;
+use mysql::{Conn, Opts, OptsBuilder, Row};
+
+/// The phenotype filter, fully qualified (the table's `public` column
+/// collides with InbredSet.public, so both predicates need the table
+/// name).
+const PHENOTYPE_PUBLIC_FILTER: &str =
+    "PublishFreeze.confidentiality < 1 AND PublishFreeze.public > 0";
+/// The gene filter, fully qualified against ProbeSetFreeze.
+const GENE_PUBLIC_FILTER: &str = "ProbeSetFreeze.confidentiality < 1 AND ProbeSetFreeze.public > 0";
+
+/// One expression (gene) crawl row; each field is absent when SQL
+/// returned NULL (the indexer's `MonadicDict` drops NULLs).
+#[derive(Default, Clone)]
+pub struct GeneRow {
+    pub name: Option<String>,
+    pub symbol: Option<String>,
+    pub description: Option<String>,
+    pub chr: Option<String>,
+    pub mb: Option<f64>,
+    pub alias: Option<String>,
+    pub genbankid: Option<String>,
+    pub unigeneid: Option<String>,
+    pub probe_target_description: Option<String>,
+    pub dataset: Option<String>,
+    pub dataset_fullname: Option<String>,
+    pub species: Option<String>,
+    pub group: Option<String>,
+    pub tissue: Option<String>,
+    pub mean: Option<f64>,
+    pub lrs: Option<f64>,
+    pub additive: Option<f64>,
+    pub geno_chr: Option<String>,
+    pub geno_mb: Option<f64>,
+}
+
+/// One published (phenotype) crawl row.
+#[derive(Default, Clone)]
+pub struct PhenotypeRow {
+    pub species: Option<String>,
+    pub group: Option<String>,
+    pub dataset: Option<String>,
+    pub dataset_fullname: Option<String>,
+    pub name: Option<i64>,
+    pub abbreviation: Option<String>,
+    pub description: Option<String>,
+    pub lab_code: Option<String>,
+    pub abstract_: Option<String>,
+    pub title: Option<String>,
+    pub authors: Option<String>,
+    pub year: Option<i64>,
+    pub pubmed_id: Option<i64>,
+    pub lrs: Option<f64>,
+    pub additive: Option<f64>,
+    pub inbredsetcode: Option<String>,
+    pub mean: Option<f64>,
+    pub nsamples: Option<i64>,
+    pub geno_chr: Option<String>,
+    pub geno_mb: Option<f64>,
+}
+
+fn connect(sql_uri: &str) -> Result<Conn, String> {
+    let opts = Opts::from_url(sql_uri).map_err(|e| format!("bad SQL_URI: {e}"))?;
+    Conn::new(OptsBuilder::from_opts(opts)).map_err(|e| format!("cannot connect: {e}"))
+}
+
+fn text(row: &Row, column: &str) -> Result<Option<String>, String> {
+    match row.get_opt::<Option<String>, _>(column) {
+        Some(Ok(value)) => Ok(value),
+        Some(Err(e)) => Err(format!("column {column}: {e}")),
+        None => Err(format!("column {column} not in result set")),
+    }
+}
+
+fn float(row: &Row, column: &str) -> Result<Option<f64>, String> {
+    match row.get_opt::<Option<f64>, _>(column) {
+        Some(Ok(value)) => Ok(value),
+        Some(Err(e)) => Err(format!("column {column}: {e}")),
+        None => Err(format!("column {column} not in result set")),
+    }
+}
+
+fn integer(row: &Row, column: &str) -> Result<Option<i64>, String> {
+    match row.get_opt::<Option<i64>, _>(column) {
+        Some(Ok(value)) => Ok(value),
+        Some(Err(e)) => Err(format!("column {column}: {e}")),
+        None => Err(format!("column {column} not in result set")),
+    }
+}
+
+fn genes_sql() -> String {
+    format!(
+        "SELECT ProbeSet.Name AS name, ProbeSet.Symbol AS symbol, \
+         ProbeSet.description AS description, ProbeSet.Chr AS chr, ProbeSet.Mb as mb, \
+         ProbeSet.alias AS alias, ProbeSet.GenbankId AS genbankid, \
+         ProbeSet.UniGeneId AS unigeneid, \
+         ProbeSet.Probe_Target_Description AS probe_target_description, \
+         ProbeSetFreeze.Name AS dataset, ProbeSetFreeze.FullName AS dataset_fullname, \
+         Species.Name AS species, InbredSet.Name AS `group`, Tissue.Name AS tissue, \
+         ProbeSetXRef.Mean AS mean, ProbeSetXRef.LRS AS lrs, \
+         ProbeSetXRef.additive AS additive, Geno.Chr AS geno_chr, Geno.Mb as geno_mb \
+         FROM Species \
+         INNER JOIN InbredSet ON InbredSet.SpeciesId = Species.Id \
+         INNER JOIN ProbeFreeze ON ProbeFreeze.InbredSetId = InbredSet.Id \
+         INNER JOIN Tissue ON ProbeFreeze.TissueId = Tissue.Id \
+         INNER JOIN ProbeSetFreeze ON ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id \
+         INNER JOIN ProbeSetXRef ON ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id \
+         INNER JOIN ProbeSet ON ProbeSet.Id = ProbeSetXRef.ProbeSetId \
+         LEFT JOIN Geno ON ProbeSetXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id \
+         WHERE {GENE_PUBLIC_FILTER}"
+    )
+}
+
+fn phenotypes_sql(filtered: bool) -> String {
+    let where_clause = if filtered {
+        format!(" WHERE {PHENOTYPE_PUBLIC_FILTER}")
+    } else {
+        String::new()
+    };
+    format!(
+        "SELECT Species.Name AS species, InbredSet.Name AS `group`, \
+         PublishFreeze.Name AS dataset, PublishFreeze.FullName AS dataset_fullname, \
+         PublishXRef.Id AS name, \
+         COALESCE(Phenotype.Post_publication_abbreviation, \
+                  Phenotype.Pre_publication_abbreviation) AS abbreviation, \
+         COALESCE(Phenotype.Post_publication_description, \
+                  Phenotype.Pre_publication_description) AS description, \
+         Phenotype.Lab_code, Publication.Abstract, Publication.Title, \
+         Publication.Authors AS authors, \
+         IF(CONVERT(Publication.Year, UNSIGNED)=0, NULL, \
+            CONVERT(Publication.Year, UNSIGNED)) AS year, \
+         Publication.PubMed_ID AS pubmed_id, PublishXRef.LRS as lrs, \
+         PublishXRef.additive, InbredSet.InbredSetCode AS inbredsetcode, \
+         PublishXRef.mean, PublishXRef.NSamples AS nsamples, \
+         Geno.Chr as geno_chr, Geno.Mb as geno_mb \
+         FROM Species \
+         INNER JOIN InbredSet ON InbredSet.SpeciesId = Species.Id \
+         INNER JOIN PublishFreeze ON PublishFreeze.InbredSetId = InbredSet.Id \
+         INNER JOIN PublishXRef ON PublishXRef.InbredSetId = InbredSet.Id \
+         INNER JOIN Phenotype ON PublishXRef.PhenotypeId = Phenotype.Id \
+         INNER JOIN Publication ON PublishXRef.PublicationId = Publication.Id \
+         LEFT JOIN Geno ON PublishXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id\
+         {where_clause}"
+    )
+}
+
+/// Crawl every public, non-confidential expression record.
+pub fn crawl_genes(sql_uri: &str) -> Result<Vec<GeneRow>, String> {
+    let mut conn = connect(sql_uri)?;
+    let rows: Vec<Row> = conn
+        .query(genes_sql())
+        .map_err(|e| format!("gene crawl failed: {e}"))?;
+    let mut out = Vec::with_capacity(rows.len());
+    for row in &rows {
+        out.push(GeneRow {
+            name: text(row, "name")?,
+            symbol: text(row, "symbol")?,
+            description: text(row, "description")?,
+            chr: text(row, "chr")?,
+            mb: float(row, "mb")?,
+            alias: text(row, "alias")?,
+            genbankid: text(row, "genbankid")?,
+            unigeneid: text(row, "unigeneid")?,
+            probe_target_description: text(row, "probe_target_description")?,
+            dataset: text(row, "dataset")?,
+            dataset_fullname: text(row, "dataset_fullname")?,
+            species: text(row, "species")?,
+            group: text(row, "group")?,
+            tissue: text(row, "tissue")?,
+            mean: float(row, "mean")?,
+            lrs: float(row, "lrs")?,
+            additive: float(row, "additive")?,
+            geno_chr: text(row, "geno_chr")?,
+            geno_mb: float(row, "geno_mb")?,
+        });
+    }
+    Ok(out)
+}
+
+/// Crawl published phenotype records. `filtered=false` drops the
+/// public/confidentiality predicate: that is the leak control, used
+/// only by the gate-4 RED run.
+pub fn crawl_phenotypes(sql_uri: &str, filtered: bool) -> Result<Vec<PhenotypeRow>, String> {
+    let mut conn = connect(sql_uri)?;
+    let rows: Vec<Row> = conn
+        .query(phenotypes_sql(filtered))
+        .map_err(|e| format!("phenotype crawl failed: {e}"))?;
+    let mut out = Vec::with_capacity(rows.len());
+    for row in &rows {
+        out.push(PhenotypeRow {
+            species: text(row, "species")?,
+            group: text(row, "group")?,
+            dataset: text(row, "dataset")?,
+            dataset_fullname: text(row, "dataset_fullname")?,
+            name: integer(row, "name")?,
+            abbreviation: text(row, "abbreviation")?,
+            description: text(row, "description")?,
+            lab_code: text(row, "Lab_code")?,
+            abstract_: text(row, "Abstract")?,
+            title: text(row, "Title")?,
+            authors: text(row, "authors")?,
+            year: integer(row, "year")?,
+            pubmed_id: integer(row, "pubmed_id")?,
+            lrs: float(row, "lrs")?,
+            additive: float(row, "additive")?,
+            inbredsetcode: text(row, "inbredsetcode")?,
+            mean: float(row, "mean")?,
+            nsamples: integer(row, "nsamples")?,
+            geno_chr: text(row, "geno_chr")?,
+            geno_mb: float(row, "geno_mb")?,
+        });
+    }
+    Ok(out)
+}
diff --git a/src/indexer/run.rs b/src/indexer/run.rs
new file mode 100644
index 0000000..5d26981
--- /dev/null
+++ b/src/indexer/run.rs
@@ -0,0 +1,319 @@
+//! Run orchestration: FULL (bootstrap) and INCREMENTAL (per-document
+//! in-place update) over one mechanism — build documents from a crawl,
+//! then write them into the index by identity term.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::{Path, PathBuf};
+
+use xapian_rs::{Database, DbAction, DbBackend, Enquire, Query, WritableDatabase};
+
+use super::build::{BuiltDocument, build_gene, build_phenotype};
+use super::cache::RdfCaches;
+use super::rows::{self, GeneRow, PhenotypeRow};
+use super::schema;
+use super::sidecar::Sidecar;
+
+/// What a run did.
+#[derive(Debug, Default, Clone)]
+pub struct Stats {
+    pub mode: String,
+    pub built: usize,
+    pub replaced: usize,
+    pub deleted: usize,
+    pub unchanged: usize,
+    pub index_doc_count: u32,
+    pub duplicate_identity_terms: Vec<String>,
+}
+
+/// Inputs for one run.
+pub struct RunConfig {
+    pub sql_uri: String,
+    pub index_path: PathBuf,
+    pub rdf_cache: PathBuf,
+    /// Apply the public/confidentiality filter to phenotypes. `false`
+    /// is the leak RED control only.
+    pub publish_filter: bool,
+}
+
+/// Test-only sabotage hooks for the RED controls (gate 5). Never set in
+/// a normal run; each value deliberately breaks one guarantee so the
+/// gate can be shown to fail.
+fn red_mode() -> Option<String> {
+    std::env::var("GN4_INDEXER_RED").ok()
+}
+
+/// Build every document for a crawl, asserting identity-term
+/// uniqueness (a duplicate `name:dataset` pair is a reported error, not
+/// last-writer-wins).
+fn build_all(
+    genes: &[GeneRow],
+    phenotypes: &[PhenotypeRow],
+    caches: &RdfCaches,
+    red_hash: bool,
+) -> Result<(Vec<BuiltDocument>, Vec<String>), String> {
+    let mut built = Vec::with_capacity(genes.len() + phenotypes.len());
+    let mut seen: BTreeMap<String, usize> = BTreeMap::new();
+    let mut duplicates = Vec::new();
+    for row in genes {
+        if let Some(mut document) = build_gene(row, caches) {
+            if let Some(nonce) = red_nonce(red_hash) {
+                document.source_hash.push_str(&nonce);
+            }
+            record(&mut seen, &mut duplicates, &mut built, document);
+        }
+    }
+    for row in phenotypes {
+        if let Some(mut document) = build_phenotype(row) {
+            if let Some(nonce) = red_nonce(red_hash) {
+                document.source_hash.push_str(&nonce);
+            }
+            record(&mut seen, &mut duplicates, &mut built, document);
+        }
+    }
+    Ok((built, duplicates))
+}
+
+/// Record one built document, flagging a duplicate identity term
+/// instead of silently overwriting.
+fn record(
+    seen: &mut BTreeMap<String, usize>,
+    duplicates: &mut Vec<String>,
+    built: &mut Vec<BuiltDocument>,
+    document: BuiltDocument,
+) {
+    if let Some(previous) = seen.get(&document.idterm) {
+        duplicates.push(format!(
+            "{} (first at position {previous}, duplicate at {})",
+            document.idterm,
+            built.len()
+        ));
+    } else {
+        seen.insert(document.idterm.clone(), built.len());
+    }
+    built.push(document);
+}
+
+fn red_nonce(red_hash: bool) -> Option<String> {
+    if !red_hash {
+        return None;
+    }
+    // Include the process id so every RUN produces different hashes:
+    // the point of the RED control is that a re-run over unchanged
+    // data still rewrites (i.e. the idempotence gate can fail).
+    static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+    let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+    Some(format!("-red{}-{n}", std::process::id()))
+}
+
+/// Enumerate the identity (`Q`-prefixed) terms the index actually holds.
+fn index_identity_terms(db: &WritableDatabase) -> BTreeSet<String> {
+    let read: Database = Database::from(db);
+    let mut enquire = Enquire::new(&read);
+    enquire.set_query(Query::match_all(), None);
+    let mset = enquire.mset(0, u32::MAX, None, None);
+    let mut terms = BTreeSet::new();
+    for item in mset.matches() {
+        let document = item.document();
+        for term in document.terms() {
+            let name: &str = term.as_ref();
+            if name.starts_with('Q') {
+                terms.insert(name.to_string());
+                break;
+            }
+        }
+    }
+    terms
+}
+
+/// Run a FULL (bootstrap) build: a fresh index written document by
+/// document through `replace_document_by_term`.
+pub fn run_full(config: &RunConfig) -> Result<Stats, String> {
+    let caches = RdfCaches::load(&config.rdf_cache)?;
+    let genes = rows::crawl_genes(&config.sql_uri)?;
+    let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?;
+    let red = red_mode();
+    let (built, duplicates) =
+        build_all(&genes, &phenotypes, &caches, red.as_deref() == Some("hash"))?;
+
+    let mut stats = Stats {
+        mode: "full".to_string(),
+        built: built.len(),
+        replaced: built.len(),
+        duplicate_identity_terms: duplicates,
+        ..Default::default()
+    };
+    if !stats.duplicate_identity_terms.is_empty() {
+        return Err(format!(
+            "duplicate identity terms in crawl: {}",
+            stats.duplicate_identity_terms.join(", ")
+        ));
+    }
+
+    let mut db = WritableDatabase::open(
+        &config.index_path,
+        Some(DbAction::CreateOrOverwrite),
+        Some(DbBackend::Glass),
+        None,
+        None,
+    );
+    db.begin_transaction(None);
+    for document in &built {
+        let _ = db.replace_document_by_term(&document.idterm, &document.document);
+    }
+    commit(&mut db, red.as_deref() == Some("commit"))?;
+    stats.index_doc_count = db.doc_count();
+
+    // Sidecar strictly after the index commit.
+    let sidecar = Sidecar {
+        version: 1,
+        mode: "full".to_string(),
+        doc_count: built.len(),
+        rif_hash: caches.rif.hash.clone(),
+        wiki_hash: caches.wiki.hash.clone(),
+        documents: built
+            .iter()
+            .map(|d| (d.idterm.clone(), d.source_hash.clone()))
+            .collect(),
+    };
+    sidecar.store(&config.index_path)?;
+    Ok(stats)
+}
+
+/// Run an INCREMENTAL pass: crawl, hash, diff against the sidecar,
+/// replace only changed documents, delete vanished identity terms, and
+/// write nothing when nothing changed.
+pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> {
+    let Some(previous) = Sidecar::load(&config.index_path) else {
+        // Missing or unreadable sidecar means FULL.
+        return run_full(config);
+    };
+    if previous.version != 1 {
+        return run_full(config);
+    }
+
+    let caches = RdfCaches::load(&config.rdf_cache)?;
+    let genes = rows::crawl_genes(&config.sql_uri)?;
+    let phenotypes = rows::crawl_phenotypes(&config.sql_uri, config.publish_filter)?;
+    let red = red_mode();
+    let (built, duplicates) =
+        build_all(&genes, &phenotypes, &caches, red.as_deref() == Some("hash"))?;
+    if !duplicates.is_empty() {
+        return Err(format!(
+            "duplicate identity terms in crawl: {}",
+            duplicates.join(", ")
+        ));
+    }
+
+    let mut db = WritableDatabase::open(
+        &config.index_path,
+        Some(DbAction::CreateOrOpen),
+        Some(DbBackend::Glass),
+        None,
+        None,
+    );
+
+    // Startup mismatch detection: the sidecar's key set must equal the
+    // index's actual identity terms, else the sidecar is behind the
+    // index (or the index was written elsewhere) and is not trusted; do
+    // a full compare against the index's own terms.
+    let index_terms = index_identity_terms(&db);
+    let sidecar_terms: BTreeSet<String> = previous.documents.keys().cloned().collect();
+    let mismatched = index_terms != sidecar_terms;
+
+    let built_by_term: BTreeMap<&str, &BuiltDocument> =
+        built.iter().map(|d| (d.idterm.as_str(), d)).collect();
+
+    let mut to_write: Vec<&BuiltDocument> = Vec::new();
+    let mut unchanged = 0usize;
+    for document in &built {
+        let known = if mismatched {
+            None
+        } else {
+            previous.documents.get(&document.idterm)
+        };
+        match known {
+            Some(hash) if *hash == document.source_hash => unchanged += 1,
+            _ => to_write.push(document),
+        }
+    }
+
+    // Existence truth: the sidecar's key set (or, on mismatch, the
+    // index's own terms). Anything there but not in the crawl is gone.
+    let mut to_delete: BTreeSet<String> = if mismatched {
+        index_terms.clone()
+    } else {
+        sidecar_terms.clone()
+    };
+    if red.as_deref() == Some("delete") {
+        to_delete.clear();
+    } else {
+        for term in built_by_term.keys() {
+            to_delete.remove(*term);
+        }
+    }
+
+    let mut stats = Stats {
+        mode: "incremental".to_string(),
+        built: built.len(),
+        unchanged,
+        ..Default::default()
+    };
+
+    let nothing_changed = to_write.is_empty() && to_delete.is_empty();
+    if !nothing_changed {
+        db.begin_transaction(None);
+        for document in &to_write {
+            let _ = db.replace_document_by_term(&document.idterm, &document.document);
+            stats.replaced += 1;
+        }
+        for term in &to_delete {
+            db.delete_document_by_term(term);
+            stats.deleted += 1;
+        }
+        commit(&mut db, red.as_deref() == Some("commit"))?;
+    }
+    stats.index_doc_count = db.doc_count();
+
+    // Sidecar after the commit; skipped when nothing changed so an
+    // idempotent run touches neither the index nor the sidecar.
+    if !nothing_changed {
+        let sidecar = Sidecar {
+            version: 1,
+            mode: "incremental".to_string(),
+            doc_count: built.len(),
+            rif_hash: caches.rif.hash.clone(),
+            wiki_hash: caches.wiki.hash.clone(),
+            documents: built
+                .iter()
+                .map(|d| (d.idterm.clone(), d.source_hash.clone()))
+                .collect(),
+        };
+        sidecar.store(&config.index_path)?;
+    }
+    Ok(stats)
+}
+
+fn commit(db: &mut WritableDatabase, skip: bool) -> Result<(), String> {
+    if skip {
+        // RED control: writes are made but never committed, so they are
+        // invisible to readers.
+        db.cancel_transaction();
+        return Ok(());
+    }
+    db.commit_transaction();
+    Ok(())
+}
+
+/// Where the sidecar lives for an index path.
+pub fn sidecar_location(index_path: &Path) -> PathBuf {
+    super::sidecar::sidecar_path(index_path)
+}
+
+/// The document type of an identity term (for reporting).
+pub fn idterm_doctype(idterm: &str) -> &str {
+    if idterm.starts_with("Qgene:") {
+        schema::DOCTYPE_GENE
+    } else {
+        schema::DOCTYPE_PHENOTYPE
+    }
+}
diff --git a/src/indexer/schema.rs b/src/indexer/schema.rs
new file mode 100644
index 0000000..5008864
--- /dev/null
+++ b/src/indexer/schema.rs
@@ -0,0 +1,96 @@
+//! The index schema shared by the writer (this module tree) and the
+//! native reader (`crate::search::native`).
+//!
+//! Writer and reader compile against the same constants, so
+//! writer/reader drift is a compile error rather than a search bug:
+//! the identity-term shape, the document-type terms, the prefixed
+//! field terms and the numeric value slots are defined once here.
+
+/// Document type for expression (ProbeSet) records; the type term is
+/// `XTgene` (`write_document`, index-genenetwork).
+pub const DOCTYPE_GENE: &str = "gene";
+/// Document type for published phenotype records; the type term is
+/// `XTphenotype`.
+pub const DOCTYPE_PHENOTYPE: &str = "phenotype";
+
+/// Prefixed term prefixes, exactly the set the indexer emits
+/// (`index_species`..`index_peakchr`, `index_from_dictionary`).
+pub mod prefix {
+    /// Authors (indexed WITH positions, wdf increment 0).
+    pub const AUTHOR: &str = "A";
+    /// Species (boolean, no positions).
+    pub const SPECIES: &str = "XS";
+    /// Group (boolean, no positions).
+    pub const GROUP: &str = "XG";
+    /// Tissue (WITH positions, wdf increment 0).
+    pub const TISSUE: &str = "XI";
+    /// Dataset (WITH positions, wdf increment 0).
+    pub const DATASET: &str = "XDS";
+    /// Symbol (boolean, no positions).
+    pub const SYMBOL: &str = "XY";
+    /// Chromosome (boolean, no positions).
+    pub const CHR: &str = "XC";
+    /// Peak chromosome (boolean, no positions).
+    pub const PEAK_CHR: &str = "XPC";
+    /// RIF dictionary text (no positions, wdf increment 0).
+    pub const RIF: &str = "XRF";
+    /// Wiki dictionary text (no positions, wdf increment 0).
+    pub const WIKI: &str = "XWK";
+}
+
+/// Numeric value slots (`add_mean`..`add_nsamples`).
+pub mod slot {
+    /// Trait mean (`add_mean`).
+    pub const MEAN: u32 = 0;
+    /// Peak / LRS (`add_peak`).
+    pub const PEAK: u32 = 1;
+    /// Position in megabases (`add_mb`).
+    pub const POSITION: u32 = 2;
+    /// Peak position in megabases (`add_peakmb`).
+    pub const PEAK_MB: u32 = 3;
+    /// Additive effect (`add_additive`).
+    pub const ADDITIVE: u32 = 4;
+    /// Publication year (`add_year`).
+    pub const YEAR: u32 = 5;
+    /// Number of samples (`add_nsamples`).
+    pub const NSAMPLES: u32 = 6;
+}
+
+/// The document-type term `XT<doctype>` written by `write_document`.
+pub fn type_term(doctype: &str) -> String {
+    format!("XT{doctype}")
+}
+
+/// The stable identity term `Q<doctype>:<name>:<dataset>` (the whole
+/// `name:dataset` identifier lowercased), matching `write_document` in
+/// the Python indexer.
+pub fn identity_term(doctype: &str, name: &str, dataset: &str) -> String {
+    format!(
+        "Q{}:{}",
+        doctype,
+        format!("{name}:{dataset}").to_lowercase()
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn identity_term_is_lowercased_with_dataset() {
+        assert_eq!(
+            identity_term(DOCTYPE_PHENOTYPE, "10583", "BXDPublish"),
+            "Qphenotype:10583:bxdpublish"
+        );
+        assert_eq!(
+            identity_term(DOCTYPE_GENE, "100001_at", "HC_U_0304_R"),
+            "Qgene:100001_at:hc_u_0304_r"
+        );
+    }
+
+    #[test]
+    fn type_term_matches_reference() {
+        assert_eq!(type_term(DOCTYPE_GENE), "XTgene");
+        assert_eq!(type_term(DOCTYPE_PHENOTYPE), "XTphenotype");
+    }
+}
diff --git a/src/indexer/sha256.rs b/src/indexer/sha256.rs
new file mode 100644
index 0000000..0556491
--- /dev/null
+++ b/src/indexer/sha256.rs
@@ -0,0 +1,196 @@
+//! A small, dependency-free SHA-256.
+//!
+//! The sidecar and cache hashes need a stable cryptographic digest and
+//! the crate must not gain a new dependency (the pinned lockfile stays
+//! untouched apart from the documented scratch `[patch]`). Standard
+//! FIPS 180-4; unit-tested against the published vectors.
+
+const K: [u32; 64] = [
+    0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+    0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+    0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+    0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+    0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+    0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+    0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+    0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
+];
+
+const H0: [u32; 8] = [
+    0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19,
+];
+
+/// Streaming SHA-256.
+pub struct Sha256 {
+    state: [u32; 8],
+    buffer: [u8; 64],
+    buffer_len: usize,
+    total_len: u64,
+}
+
+impl Default for Sha256 {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Sha256 {
+    pub fn new() -> Self {
+        Sha256 {
+            state: H0,
+            buffer: [0u8; 64],
+            buffer_len: 0,
+            total_len: 0,
+        }
+    }
+
+    pub fn update(&mut self, mut data: &[u8]) {
+        self.total_len = self.total_len.wrapping_add(data.len() as u64);
+        if self.buffer_len > 0 {
+            let take = (64 - self.buffer_len).min(data.len());
+            self.buffer[self.buffer_len..self.buffer_len + take].copy_from_slice(&data[..take]);
+            self.buffer_len += take;
+            data = &data[take..];
+            if self.buffer_len == 64 {
+                let block = self.buffer;
+                self.compress(&block);
+                self.buffer_len = 0;
+            }
+        }
+        while data.len() >= 64 {
+            let mut block = [0u8; 64];
+            block.copy_from_slice(&data[..64]);
+            self.compress(&block);
+            data = &data[64..];
+        }
+        if !data.is_empty() {
+            self.buffer[..data.len()].copy_from_slice(data);
+            self.buffer_len = data.len();
+        }
+    }
+
+    pub fn finalize(mut self) -> [u8; 32] {
+        let bit_len = self.total_len.wrapping_mul(8);
+        self.update(&[0x80]);
+        while self.buffer_len != 56 {
+            self.update(&[0x00]);
+        }
+        // update() advances total_len, but the length field uses the
+        // pre-padding bit length captured above.
+        self.total_len = self.total_len; // no-op, kept for clarity
+        self.update(&bit_len.to_be_bytes());
+        let mut out = [0u8; 32];
+        for (index, word) in self.state.iter().enumerate() {
+            out[index * 4..index * 4 + 4].copy_from_slice(&word.to_be_bytes());
+        }
+        out
+    }
+
+    fn compress(&mut self, block: &[u8; 64]) {
+        let mut w = [0u32; 64];
+        for i in 0..16 {
+            w[i] = u32::from_be_bytes([
+                block[i * 4],
+                block[i * 4 + 1],
+                block[i * 4 + 2],
+                block[i * 4 + 3],
+            ]);
+        }
+        for i in 16..64 {
+            let s0 = w[i - 15].rotate_right(7) ^ w[i - 15].rotate_right(18) ^ (w[i - 15] >> 3);
+            let s1 = w[i - 2].rotate_right(17) ^ w[i - 2].rotate_right(19) ^ (w[i - 2] >> 10);
+            w[i] = w[i - 16]
+                .wrapping_add(s0)
+                .wrapping_add(w[i - 7])
+                .wrapping_add(s1);
+        }
+        let mut a = self.state[0];
+        let mut b = self.state[1];
+        let mut c = self.state[2];
+        let mut d = self.state[3];
+        let mut e = self.state[4];
+        let mut f = self.state[5];
+        let mut g = self.state[6];
+        let mut h = self.state[7];
+        for i in 0..64 {
+            let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25);
+            let ch = (e & f) ^ (!e & g);
+            let temp1 = h
+                .wrapping_add(s1)
+                .wrapping_add(ch)
+                .wrapping_add(K[i])
+                .wrapping_add(w[i]);
+            let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22);
+            let maj = (a & b) ^ (a & c) ^ (b & c);
+            let temp2 = s0.wrapping_add(maj);
+            h = g;
+            g = f;
+            f = e;
+            e = d.wrapping_add(temp1);
+            d = c;
+            c = b;
+            b = a;
+            a = temp1.wrapping_add(temp2);
+        }
+        self.state[0] = self.state[0].wrapping_add(a);
+        self.state[1] = self.state[1].wrapping_add(b);
+        self.state[2] = self.state[2].wrapping_add(c);
+        self.state[3] = self.state[3].wrapping_add(d);
+        self.state[4] = self.state[4].wrapping_add(e);
+        self.state[5] = self.state[5].wrapping_add(f);
+        self.state[6] = self.state[6].wrapping_add(g);
+        self.state[7] = self.state[7].wrapping_add(h);
+    }
+}
+
+/// Hex digest of `data`.
+pub fn hex_digest(data: &[u8]) -> String {
+    let digest = {
+        let mut hasher = Sha256::new();
+        hasher.update(data);
+        hasher.finalize()
+    };
+    let mut out = String::with_capacity(64);
+    for byte in digest {
+        use std::fmt::Write as _;
+        let _ = write!(out, "{byte:02x}");
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn matches_published_vectors() {
+        assert_eq!(
+            hex_digest(b""),
+            "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+        );
+        assert_eq!(
+            hex_digest(b"abc"),
+            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
+        );
+        assert_eq!(
+            hex_digest(b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"),
+            "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"
+        );
+    }
+
+    #[test]
+    fn streaming_matches_one_shot() {
+        let data: Vec<u8> = (0..1000u32).map(|n| (n % 251) as u8).collect();
+        let one = hex_digest(&data);
+        let mut hasher = Sha256::new();
+        for chunk in data.chunks(7) {
+            hasher.update(chunk);
+        }
+        let mut out = String::new();
+        for byte in hasher.finalize() {
+            use std::fmt::Write as _;
+            let _ = write!(out, "{byte:02x}");
+        }
+        assert_eq!(one, out);
+    }
+}
diff --git a/src/indexer/sidecar.rs b/src/indexer/sidecar.rs
new file mode 100644
index 0000000..8dcc5a3
--- /dev/null
+++ b/src/indexer/sidecar.rs
@@ -0,0 +1,58 @@
+//! The incremental sidecar: per-document source hashes stored next to
+//! the index (not inside it), written atomically and strictly after the
+//! index commit.
+//!
+//! The sidecar's key set is the existence truth for the incremental
+//! path: a document absent from a crawl is deleted by its identity term
+//! even if the index still holds it. Write order is index commit first,
+//! sidecar rename after, so a crash between the two leaves a sidecar
+//! that is behind the index — the startup key-set check detects exactly
+//! that and falls back to a full compare.
+
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+
+/// The on-disk sidecar.
+#[derive(Serialize, Deserialize, Default, PartialEq)]
+pub struct Sidecar {
+    /// Format version; a mismatch triggers a full rebuild.
+    pub version: u32,
+    /// How the sidecar was last written (`full` or `incremental`).
+    pub mode: String,
+    /// Document count of the index the sidecar describes.
+    pub doc_count: usize,
+    /// Monolithic RIF cache hash at the time of writing.
+    pub rif_hash: String,
+    /// Monolithic wiki cache hash at the time of writing.
+    pub wiki_hash: String,
+    /// Identity term -> source hash.
+    pub documents: BTreeMap<String, String>,
+}
+
+/// The sidecar path for an index directory (`<index>.sidecar.json`).
+pub fn sidecar_path(index_path: &Path) -> PathBuf {
+    PathBuf::from(format!(
+        "{}.sidecar.json",
+        index_path.to_string_lossy().trim_end_matches('/')
+    ))
+}
+
+impl Sidecar {
+    pub fn load(index_path: &Path) -> Option<Sidecar> {
+        let text = std::fs::read_to_string(sidecar_path(index_path)).ok()?;
+        serde_json::from_str(&text).ok()
+    }
+
+    /// Atomically replace the sidecar: write a sibling temp file, sync,
+    /// then rename over the target.
+    pub fn store(&self, index_path: &Path) -> Result<(), String> {
+        let path = sidecar_path(index_path);
+        let temp = PathBuf::from(format!("{}.tmp.{}", path.display(), std::process::id()));
+        let json = serde_json::to_string(self).map_err(|e| format!("sidecar encode: {e}"))?;
+        std::fs::write(&temp, json).map_err(|e| format!("sidecar write: {e}"))?;
+        std::fs::rename(&temp, &path).map_err(|e| format!("sidecar rename: {e}"))?;
+        Ok(())
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 4d90785..50db2b5 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -10,6 +10,7 @@ pub mod config;
 mod db;
 pub mod embedded;
 mod error;
+pub mod indexer;
 mod gemma;
 pub mod response;
 mod routes;