diff options
| author | Alexander_Kabui <alexanderkabui@proton.me> | 2026-10-14 07:28:47 +0300 |
|---|---|---|
| committer | Alexander_Kabui <alexanderkabui@proton.me> | 2026-10-14 07:28:47 +0300 |
| commit | d607e8faa7692f28f7e6cf3d0517a69a44557a27 (patch) | |
| tree | 5643ec45ed1f1d1296bf0732f775e9fd93afd882 /src | |
| parent | b0c027a40c1e64d5c3a6b7e5dd79d1f3c4cb35eb (diff) | |
| download | genenetwork4-d607e8faa7692f28f7e6cf3d0517a69a44557a27.tar.gz | |
indexer: add versioning-based DELTA mode
* src/indexer/delta.rs: change detection over MariaDB system versioning (ROW_START/ROW_END): coverage marker, versioning probe, purge check, per-table candidate queries and identity-term fan-out. * src/indexer/run.rs (run_delta): reuse the streaming crawl and shard diff, scoped to the affected datasets. * src/indexer/sidecar.rs: manifest v3 with versioning_marker/versioning_flavour. * src/bin/gn-indexer.rs: add the delta mode. * scripts/gn-indexer-run.sh, scripts/gn-indexer-lag.sh: flock wrapper and lag SLA.
Diffstat (limited to 'src')
| -rw-r--r-- | src/bin/gn-indexer.rs | 19 | ||||
| -rw-r--r-- | src/indexer/delta.rs | 1087 | ||||
| -rw-r--r-- | src/indexer/mod.rs | 10 | ||||
| -rw-r--r-- | src/indexer/rows.rs | 246 | ||||
| -rw-r--r-- | src/indexer/run.rs | 418 | ||||
| -rw-r--r-- | src/indexer/sidecar.rs | 89 |
6 files changed, 1791 insertions, 78 deletions
diff --git a/src/bin/gn-indexer.rs b/src/bin/gn-indexer.rs index 224edb8..57a3f57 100644 --- a/src/bin/gn-indexer.rs +++ b/src/bin/gn-indexer.rs @@ -3,6 +3,7 @@ //! Usage: //! gn-indexer full [--index PATH] [--rdf-cache PATH] [--no-publish-filter] //! gn-indexer incremental [--index PATH] [--rdf-cache PATH] [--no-publish-filter] +//! gn-indexer delta [--index PATH] [--rdf-cache PATH] [--no-publish-filter] //! //! The target index comes from `--index`, else `XAPIAN_DB_PATH` (the //! same key the search engine reads), and is written IN PLACE. The RDF @@ -10,6 +11,10 @@ //! `/tmp/gn4-rdf-cache.json`. `SQL_URI` comes from the crate's normal //! layered configuration. //! +//! `delta` names changed rows via system versioning and re-diffs only the +//! affected datasets; it falls back to a FULL rebuild on any doubt. The +//! periodic `incremental` sweep remains the correctness bound. +//! //! Test-only RED controls for the incremental gates are read from //! `GN4_INDEXER_RED` (values `hash`, `delete`, `commit`); they sabotage //! one guarantee each and are never set in a normal run. @@ -18,11 +23,11 @@ use std::path::PathBuf; use std::process::ExitCode; use genenetwork4::config::Config; -use genenetwork4::indexer::{RunConfig, run_full, run_incremental}; +use genenetwork4::indexer::{RunConfig, run_delta, run_full, run_incremental}; fn usage() -> ExitCode { eprintln!( - "usage: gn-indexer <full|incremental> [--index PATH] [--rdf-cache PATH] \ + "usage: gn-indexer <full|incremental|delta> [--index PATH] [--rdf-cache PATH] \ [--no-publish-filter]" ); ExitCode::from(2) @@ -70,6 +75,7 @@ fn main() -> ExitCode { let result = match mode.as_str() { "full" => run_full(&run_config), "incremental" => run_incremental(&run_config), + "delta" => run_delta(&run_config), _ => return usage(), }; @@ -97,6 +103,15 @@ fn main() -> ExitCode { println!("commit_chunks={}", stats.commit_chunks); println!("chunk_rss_peak_kib={}", stats.chunk_rss_peak_kib); println!("startup={}", stats.startup); + for (table, count) in &stats.candidates { + println!("candidates_{table}={count}"); + } + println!("affected_datasets={}", stats.affected_datasets); + println!( + "marker_from={}", + stats.marker_from.as_deref().unwrap_or("-") + ); + println!("marker_to={}", stats.marker_to.as_deref().unwrap_or("-")); ExitCode::SUCCESS } Err(error) => { diff --git a/src/indexer/delta.rs b/src/indexer/delta.rs new file mode 100644 index 0000000..e0843f8 --- /dev/null +++ b/src/indexer/delta.rs @@ -0,0 +1,1087 @@ +//! The versioning-based change-detection (DELTA) path: a DB-clock +//! coverage marker, the seven system-versioned candidate queries, and the +//! fan-out from changed rows to affected dataset labels. +//! +//! This module is deliberately split: the marker/window/decision logic is +//! pure (no DB) and unit-tested directly; the DB helpers (`probe`, +//! `NOW(6)`, `oldest_history`, candidate + fan-out queries) are exercised +//! by the lane integration gate against `db_webqtl_empty`. +//! +//! The delta only ever *narrows* the work: it names changed rows and the +//! datasets they touch, then hands those datasets to the existing +//! build+hash+compare path. It is structurally blind to cache text and to +//! the unversioned contributing tables (`Geno`, `Phenotype`, `Publication`, +//! `ProbeFreeze`, `Tissue`); the periodic incremental sweep remains the +//! correctness bound. + +use std::collections::BTreeSet; + +use mysql::prelude::Queryable; +use mysql::{Conn, Row}; + +use super::schema; + +/// Timestamp flavour the owner's `ADD SYSTEM VERSIONING` produces: +/// `ROW_START`/`ROW_END` are `TIMESTAMP(6)` (microsecond precision). +pub const FLAVOUR: &str = "timestamp6"; + +/// The test-only table-prefix seam (see [`table_name`]): the env var that +/// supplies the prefix and the second explicit opt-in required to honour +/// it (a debug build also enables it). +pub const PREFIX_ENV: &str = "GN4_INDEXER_DELTA_TABLE_PREFIX"; +/// Second explicit opt-in for the table-prefix seam (A6): the prefix is +/// ignored unless this is `1` or the binary is a debug build. +pub const TEST_ENV: &str = "GN4_INDEXER_DELTA_TEST"; + +/// The seven tables the owner versions, each mapped to its candidate key +/// columns (`ROW_START`/`ROW_END` are added by the query template). +pub const VERSIONED_TABLES: [&str; 7] = [ + "PublishXRef", + "PublishFreeze", + "ProbeSetXRef", + "ProbeSetFreeze", + "ProbeSet", + "InbredSet", + "Species", +]; + +/// How many labels/keys go into one `IN (…)` list before it is split into +/// OR-groups. `Species`/`InbredSet` fan-out can name every dataset. +const IN_CHUNK: usize = 1000; + +// --------------------------------------------------------------------------- +// Timestamp + window math (pure, no DB, no new dependency) +// --------------------------------------------------------------------------- + +/// A DB timestamp in the fixed `YYYY-MM-DD HH:MM:SS.ffffff` format. The +/// fixed width and zero padding make lexicographic order equal to +/// chronological order, and the hand-rolled civil-date arithmetic avoids +/// adding a time dependency. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DbTimestamp(String); + +impl DbTimestamp { + /// Strict parse: exact length, separators, and digit positions, with + /// calendar/clock range checks. `None` for anything else (so a + /// NULL/unparseable marker cannot silently compare to 0 rows). + pub fn parse(text: &str) -> Option<DbTimestamp> { + let bytes = text.as_bytes(); + if bytes.len() != 26 { + return None; + } + let sep = |i: usize, c: u8| bytes[i] == c; + if !(sep(4, b'-') + && sep(7, b'-') + && sep(10, b' ') + && sep(13, b':') + && sep(16, b':') + && sep(19, b'.')) + { + return None; + } + let digits = |a: usize, b: usize| -> Option<u32> { + let mut value = 0u32; + for &byte in &bytes[a..b] { + if !byte.is_ascii_digit() { + return None; + } + value = value * 10 + (byte - b'0') as u32; + } + Some(value) + }; + digits(0, 4)?; + let month = digits(5, 7)?; + let day = digits(8, 10)?; + let hour = digits(11, 13)?; + let minute = digits(14, 16)?; + let second = digits(17, 19)?; + let micros = digits(20, 26)?; + if !(1..=12).contains(&month) + || !(1..=31).contains(&day) + || hour > 23 + || minute > 59 + || second > 59 + || micros > 999_999 + { + return None; + } + Some(DbTimestamp(text.to_string())) + } + + /// The timestamp as the canonical string. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Microseconds since the civil epoch (1970-01-01 00:00:00), used only + /// for ordering and the overlap subtraction. + fn to_micros(&self) -> i64 { + let (year, month, day) = self.civil(); + let (hour, minute, second, micros) = self.clock(); + let days = days_from_civil(year, month, day); + ((days * 86_400 + hour as i64 * 3600 + minute as i64 * 60 + second as i64) * 1_000_000) + + micros as i64 + } + + fn civil(&self) -> (i64, u32, u32) { + let b = self.0.as_bytes(); + let num = |a: usize, n: usize| -> i64 { + b[a..a + n] + .iter() + .fold(0i64, |acc, &c| acc * 10 + (c - b'0') as i64) + }; + (num(0, 4), num(5, 2) as u32, num(8, 2) as u32) + } + + fn clock(&self) -> (u32, u32, u32, u32) { + let b = self.0.as_bytes(); + let num = |a: usize, n: usize| -> u32 { + b[a..a + n] + .iter() + .fold(0u32, |acc, &c| acc * 10 + (c - b'0') as u32) + }; + (num(11, 2), num(14, 2), num(17, 2), num(20, 6)) + } + + fn from_micros(total: i64) -> DbTimestamp { + let secs = total.div_euclid(1_000_000); + let micros = total.rem_euclid(1_000_000) as u32; + let days = secs.div_euclid(86_400); + let secs_of_day = secs.rem_euclid(86_400); + let (year, month, day) = civil_from_days(days); + let hour = (secs_of_day / 3600) as u32; + let minute = ((secs_of_day % 3600) / 60) as u32; + let second = (secs_of_day % 60) as u32; + DbTimestamp(format!( + "{year:04}-{month:02}-{day:02} {hour:02}:{minute:02}:{second:02}.{micros:06}" + )) + } +} + +impl std::fmt::Display for DbTimestamp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl PartialOrd for DbTimestamp { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for DbTimestamp { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.to_micros().cmp(&other.to_micros()) + } +} + +/// Days from 1970-01-01 for a proleptic-Gregorian civil date (Howard +/// Hinnant's `days_from_civil`). +fn days_from_civil(year: i64, month: u32, day: u32) -> i64 { + let y = if month <= 2 { year - 1 } else { year }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = y - era * 400; + let mp = if month > 2 { month - 3 } else { month + 9 } as i64; + let doy = (153 * mp + 2) / 5 + day as i64 - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146_097 + doe - 719_468 +} + +/// Inverse of [`days_from_civil`]. +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let z = days + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = (doy - (153 * mp + 2) / 5 + 1) as u32; + let month = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + (if month <= 2 { y + 1 } else { y }, month, day) +} + +/// The overlap floor: `prev - overlap_secs`, saturating at the civil +/// epoch. Re-examines the tail of the previous window so a late-committed +/// transaction (started before `prev`, committed after the scan) is caught +/// on the next run. +pub fn window_floor(prev: &DbTimestamp, overlap_secs: u64) -> DbTimestamp { + // MySQL TIMESTAMP cannot precede the civil epoch, so clamp there. + let micros = prev + .to_micros() + .saturating_sub(overlap_secs as i64 * 1_000_000) + .max(0); + DbTimestamp::from_micros(micros) +} + +/// Overlap seconds, `GN4_INDEXER_DELTA_OVERLAP_SECONDS` (default 30). +pub fn overlap_seconds() -> u64 { + std::env::var("GN4_INDEXER_DELTA_OVERLAP_SECONDS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(30) +} + +/// The startup decision: run the delta at `floor`, or fall back to FULL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Startup { + /// Run the delta. `prev` is the previous marker, `new` the captured + /// one, `floor` the overlap-adjusted lower bound. + Delta { + prev: DbTimestamp, + new: DbTimestamp, + floor: DbTimestamp, + }, + /// Fall back to FULL, with the reason to log. + Full(String), +} + +/// Decide whether the delta can run, encoding the fallback rules exactly: +/// versioning absent, marker NULL/unparseable, marker in the future, +/// marker not strictly monotonic (log "retry next tick" on the rare +/// equal-microsecond case), and history purged (oldest resolvable version +/// newer than the marker). +/// +/// The caller must probe versioning presence **before** computing +/// `oldest_history` (a `FOR SYSTEM_TIME ALL` query errors on an +/// unversioned table), so `probe_ok` is expected to be `true` on the +/// DB path and is kept in the signature so the rule is directly testable. +pub fn decide( + prev_marker: Option<&str>, + new_marker: &DbTimestamp, + probe_ok: bool, + oldest_history: Option<&DbTimestamp>, +) -> Startup { + if !probe_ok { + return Startup::Full("versioning absent".to_string()); + } + let Some(prev) = prev_marker.and_then(DbTimestamp::parse) else { + return Startup::Full("no marker".to_string()); + }; + if prev > *new_marker { + return Startup::Full("marker in future".to_string()); + } + if prev == *new_marker { + return Startup::Full( + "marker not monotonic (equal timestamp; retry next tick)".to_string(), + ); + } + if let Some(oldest) = oldest_history { + if *oldest > prev { + return Startup::Full("history purged".to_string()); + } + } + let floor = window_floor(&prev, overlap_seconds()); + Startup::Delta { + prev, + new: new_marker.clone(), + floor, + } +} + +// --------------------------------------------------------------------------- +// Table-name resolver (the test-only prefix seam, A6) +// --------------------------------------------------------------------------- + +/// Resolve the effective table prefix from raw inputs. Kept pure so the +/// guard is directly testable without touching process env. +pub fn resolve_prefix( + raw_prefix: Option<&str>, + test_env: Option<&str>, + debug_build: bool, +) -> Option<String> { + let prefix = raw_prefix.unwrap_or(""); + if prefix.is_empty() { + return None; + } + if debug_build || test_env == Some("1") { + Some(prefix.to_string()) + } else { + None + } +} + +#[cfg(test)] +thread_local! { + static TABLE_PREFIX_OVERRIDE: std::cell::RefCell<Option<String>> = + const { std::cell::RefCell::new(None) }; +} + +/// The effective table prefix. Honoured only when the explicit test env +/// (or a debug build) is present (A6); otherwise the prefix env is ignored +/// and a warning is expected from the run entry point. +pub fn table_prefix() -> String { + #[cfg(test)] + { + if let Some(prefix) = TABLE_PREFIX_OVERRIDE.with(|cell| cell.borrow().clone()) { + return prefix; + } + } + resolve_prefix( + std::env::var(PREFIX_ENV).ok().as_deref(), + std::env::var(TEST_ENV).ok().as_deref(), + cfg!(debug_assertions), + ) + .unwrap_or_default() +} + +/// Test-only override for [`table_prefix`] (thread-local, so parallel +/// tests cannot race on process env). +#[cfg(test)] +pub(crate) fn set_table_prefix_override(prefix: Option<String>) { + TABLE_PREFIX_OVERRIDE.with(|cell| *cell.borrow_mut() = prefix); +} + +/// The single resolver for the seven versioned table names (used by both +/// the delta queries and the crawl SQL, so the integration gate can point +/// both at the `vdelta_*` copies). Non-versioned contributing tables +/// (`Geno`, `Phenotype`, `Publication`, `ProbeFreeze`, `Tissue`) are never +/// prefixed. +pub fn table_name(base: &str) -> String { + format!("{}{}", table_prefix(), base) +} + +/// Emit the seam warning once at a run's entry point (A6/R6). +pub fn log_prefix_warning() { + let raw = std::env::var(PREFIX_ENV).unwrap_or_default(); + if raw.is_empty() { + return; + } + if table_prefix().is_empty() { + eprintln!( + "gn-indexer: WARNING: {PREFIX_ENV}={raw:?} ignored: {TEST_ENV}=1 (or a debug build) is required to enable the test seam" + ); + } else { + eprintln!( + "gn-indexer: WARNING: {PREFIX_ENV}={raw:?} active: the indexer is reading PREFIXED tables (test seam)" + ); + } +} + +// --------------------------------------------------------------------------- +// SQL literal helpers (shared with the crawl's scoped filter, A5) +// --------------------------------------------------------------------------- + +/// Quote a raw DB label as a MySQL string literal, escaping backslashes +/// and single quotes. The label is compared with `COLLATE utf8mb4_bin`, +/// so escaping must be exact. +pub fn quote_sql_string(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('\''); + for ch in value.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '\'' => out.push_str("''"), + _ => out.push(ch), + } + } + out.push('\''); + out +} + +/// A chunked `column IN (…) [OR column IN (…)]` clause over labels, or +/// `0=1` for an empty set (so the caller can always append it). +pub fn label_in_clause(column: &str, labels: &BTreeSet<String>) -> String { + if labels.is_empty() { + return "0=1".to_string(); + } + let mut groups: Vec<String> = Vec::new(); + let mut current: Vec<String> = Vec::new(); + for label in labels { + current.push(quote_sql_string(label)); + if current.len() == IN_CHUNK { + groups.push(current.join(", ")); + current.clear(); + } + } + if !current.is_empty() { + groups.push(current.join(", ")); + } + if groups.len() == 1 { + format!("{column} IN ({})", groups[0]) + } else { + let joined = groups + .iter() + .map(|group| format!("{column} IN ({group})")) + .collect::<Vec<_>>() + .join(" OR "); + format!("({joined})") + } +} + +/// A chunked `column IN (…)` clause over integer keys, or `0=1` when the +/// set is empty. +pub fn int_in_clause(column: &str, values: &BTreeSet<i64>) -> String { + if values.is_empty() { + return "0=1".to_string(); + } + let mut groups: Vec<String> = Vec::new(); + let mut current: Vec<String> = Vec::new(); + for value in values { + current.push(value.to_string()); + if current.len() == IN_CHUNK { + groups.push(current.join(", ")); + current.clear(); + } + } + if !current.is_empty() { + groups.push(current.join(", ")); + } + if groups.len() == 1 { + format!("{column} IN ({})", groups[0]) + } else { + let joined = groups + .iter() + .map(|group| format!("{column} IN ({group})")) + .collect::<Vec<_>>() + .join(" OR "); + format!("({joined})") + } +} + +// --------------------------------------------------------------------------- +// DB helpers +// --------------------------------------------------------------------------- + +/// Connect for the delta scan: reuse the crawl's connection, then pin +/// READ COMMITTED (design §1.6 — a long REPEATABLE READ snapshot would +/// hide versions committed during the scan). +pub fn connect(sql_uri: &str) -> Result<Conn, String> { + let mut conn = super::rows::connect(sql_uri)?; + conn.query_drop("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED") + .map_err(|e| format!("set read committed: {e}"))?; + Ok(conn) +} + +/// `SELECT NOW(6)` on the DB clock. +pub fn capture_now6(conn: &mut Conn) -> Result<DbTimestamp, String> { + let value: Option<String> = conn + .query_first("SELECT NOW(6)") + .map_err(|e| format!("SELECT NOW(6): {e}"))?; + value + .as_deref() + .and_then(DbTimestamp::parse) + .ok_or_else(|| "NOW(6) returned an unparseable value".to_string()) +} + +/// True when all seven versioned tables are system-versioned. Probing +/// `information_schema.TABLES.TABLE_TYPE` is safe on unversioned tables +/// (unlike a `FOR SYSTEM_TIME` query, which errors `Unknown column +/// 'ROW_START'`), and — on MariaDB 10.6 — it is also the *only* reliable +/// tell: the implicit `ROW_START`/`ROW_END` columns are system columns and +/// do NOT appear in `information_schema.COLUMNS`, and the table reports +/// `TABLE_TYPE = 'SYSTEM VERSIONED'` instead of `BASE TABLE`. +pub fn versioning_probe(conn: &mut Conn) -> Result<bool, String> { + for base in VERSIONED_TABLES { + let table = table_name(base); + let sql = format!( + "SELECT COUNT(*) FROM information_schema.TABLES \ + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = {} \ + AND TABLE_TYPE = 'SYSTEM VERSIONED'", + quote_sql_string(&table) + ); + let count: Option<i64> = conn + .query_first(&sql) + .map_err(|e| format!("versioning probe {table}: {e}"))?; + if count.unwrap_or(0) == 0 { + return Ok(false); + } + } + Ok(true) +} + +/// The oldest resolvable history across the seven tables: `MIN(ROW_START)` +/// of the seven-way `FOR SYSTEM_TIME ALL` UNION (design §4). +pub fn oldest_history(conn: &mut Conn) -> Result<Option<DbTimestamp>, String> { + let mut parts: Vec<String> = Vec::new(); + for base in VERSIONED_TABLES { + let table = table_name(base); + parts.push(format!( + "SELECT MIN(ROW_START) AS oldest FROM {table} FOR SYSTEM_TIME ALL" + )); + } + let sql = format!("SELECT MIN(oldest) FROM ({}) u", parts.join(" UNION ALL ")); + let value: Option<String> = conn + .query_first(&sql) + .map_err(|e| format!("oldest_history: {e}"))?; + Ok(value.as_deref().and_then(DbTimestamp::parse)) +} + +/// Capture a fresh coverage marker when versioning is present, for FULL +/// and INCREMENTAL (both re-hash every source, so both are full coverage +/// points). Returns `(marker, flavour)`, or `None` when versioning is +/// absent or the DB cannot be reached — a marker is optional and must +/// never fail an otherwise-valid FULL. +pub fn capture_coverage_marker(sql_uri: &str) -> Option<(String, String)> { + let mut conn = connect(sql_uri).ok()?; + if !versioning_probe(&mut conn).unwrap_or(false) { + return None; + } + let now = capture_now6(&mut conn).ok()?; + Some((now.as_str().to_string(), FLAVOUR.to_string())) +} + +// --------------------------------------------------------------------------- +// Candidate scan + fan-out +// --------------------------------------------------------------------------- + +/// The change set of one delta window: per-table candidate counts (for +/// stats), the identity terms the candidates name (reporting/twin-gate +/// reasoning), and the affected dataset labels (the write path's unit). +#[derive(Debug, Default, Clone)] +pub struct Candidates { + pub per_table: std::collections::BTreeMap<&'static str, usize>, + pub idterms: BTreeSet<String>, + pub datasets: BTreeSet<String>, +} + +/// The `[floor, new)` window predicate: `ROW_START`-in-window (upserts) +/// OR `ROW_END`-in-window (deletes — a `DELETE` leaves `ROW_START` alone +/// and only moves `ROW_END`). +fn window_predicate(floor: &DbTimestamp, new_marker: &DbTimestamp) -> String { + let floor = quote_sql_string(floor.as_str()); + let new_marker = quote_sql_string(new_marker.as_str()); + format!( + "(ROW_START >= {floor} AND ROW_START < {new_marker}) \ + OR (ROW_END >= {floor} AND ROW_END < {new_marker})" + ) +} + +/// Scan the window and fan the candidates out to affected dataset labels. +pub fn scan( + conn: &mut Conn, + floor: &DbTimestamp, + new_marker: &DbTimestamp, +) -> Result<Candidates, String> { + let mut out = Candidates::default(); + let window = window_predicate(floor, new_marker); + + // --- PublishXRef (phenotype rows; key (InbredSetId, Id)) --- + let pubxref = table_name("PublishXRef"); + let pubfreeze = table_name("PublishFreeze"); + let rows = query_i64_rows( + conn, + &format!( + "SELECT DISTINCT InbredSetId, Id FROM {pubxref} FOR SYSTEM_TIME ALL WHERE {window}" + ), + 2, + )?; + out.per_table.insert("PublishXRef", rows.len()); + let mut pub_ids: BTreeSet<i64> = BTreeSet::new(); + for row in &rows { + if let (Some(_isid), Some(id)) = (row[0], row[1]) { + pub_ids.insert(id); + } + } + // The InbredSetId(s) of those rows across ALL versions, so a row moved + // between groups fans out to both old and new labels. + let mut inbred_sets: BTreeSet<i64> = BTreeSet::new(); + for row in &rows { + if let Some(isid) = row[0] { + inbred_sets.insert(isid); + } + } + for value in query_ints( + conn, + &format!( + "SELECT DISTINCT InbredSetId FROM {pubxref} FOR SYSTEM_TIME ALL WHERE {}", + int_in_clause("Id", &pub_ids) + ), + )? { + inbred_sets.insert(value); + } + let freeze_names = query_strings( + conn, + &format!( + "SELECT DISTINCT Name FROM {pubfreeze} WHERE {}", + int_in_clause("InbredSetId", &inbred_sets) + ), + )?; + for name in &freeze_names { + out.datasets.insert(name.clone()); + } + for id in &pub_ids { + for name in &freeze_names { + out.idterms.insert(schema::identity_term( + schema::DOCTYPE_PHENOTYPE, + &id.to_string(), + name, + )); + } + } + + // --- PublishFreeze (dataset-label owner) --- + let rows = query_i64_rows( + conn, + &format!("SELECT DISTINCT Id FROM {pubfreeze} FOR SYSTEM_TIME ALL WHERE {window}"), + 1, + )?; + out.per_table.insert("PublishFreeze", rows.len()); + let freeze_ids: BTreeSet<i64> = rows.iter().filter_map(|r| r[0]).collect(); + for name in query_strings( + conn, + &format!( + "SELECT DISTINCT Name FROM {pubfreeze} FOR SYSTEM_TIME ALL WHERE {}", + int_in_clause("Id", &freeze_ids) + ), + )? { + out.datasets.insert(name); + } + + // --- ProbeSetXRef (gene rows; key (ProbeSetFreezeId, ProbeSetId)) --- + let probexref = table_name("ProbeSetXRef"); + let probeset = table_name("ProbeSet"); + let probefreeze = table_name("ProbeSetFreeze"); + let rows = query_i64_rows( + conn, + &format!( + "SELECT DISTINCT ProbeSetFreezeId, ProbeSetId FROM {probexref} FOR SYSTEM_TIME ALL WHERE {window}" + ), + 2, + )?; + out.per_table.insert("ProbeSetXRef", rows.len()); + let mut probe_ids: BTreeSet<i64> = BTreeSet::new(); + let mut probe_freeze_ids: BTreeSet<i64> = BTreeSet::new(); + for row in &rows { + if let Some(psf) = row[0] { + probe_freeze_ids.insert(psf); + } + if let Some(psid) = row[1] { + probe_ids.insert(psid); + } + } + for value in query_ints( + conn, + &format!( + "SELECT DISTINCT ProbeSetFreezeId FROM {probexref} FOR SYSTEM_TIME ALL WHERE {}", + int_in_clause("ProbeSetId", &probe_ids) + ), + )? { + probe_freeze_ids.insert(value); + } + let gene_freeze_names = query_strings( + conn, + &format!( + "SELECT DISTINCT Name FROM {probefreeze} WHERE {}", + int_in_clause("Id", &probe_freeze_ids) + ), + )?; + for name in &gene_freeze_names { + out.datasets.insert(name.clone()); + } + let probe_names = query_strings( + conn, + &format!( + "SELECT DISTINCT Name FROM {probeset} WHERE {}", + int_in_clause("Id", &probe_ids) + ), + )?; + for probe in &probe_names { + for freeze in &gene_freeze_names { + out.idterms + .insert(schema::identity_term(schema::DOCTYPE_GENE, probe, freeze)); + } + } + + // --- ProbeSetFreeze (dataset-label owner) --- + let rows = query_i64_rows( + conn, + &format!("SELECT DISTINCT Id FROM {probefreeze} FOR SYSTEM_TIME ALL WHERE {window}"), + 1, + )?; + out.per_table.insert("ProbeSetFreeze", rows.len()); + let rows_ids: BTreeSet<i64> = rows.iter().filter_map(|r| r[0]).collect(); + for name in query_strings( + conn, + &format!( + "SELECT DISTINCT Name FROM {probefreeze} FOR SYSTEM_TIME ALL WHERE {}", + int_in_clause("Id", &rows_ids) + ), + )? { + out.datasets.insert(name); + } + + // --- ProbeSet (gene name owner) --- + let rows = query_i64_rows( + conn, + &format!("SELECT DISTINCT Id FROM {probeset} FOR SYSTEM_TIME ALL WHERE {window}"), + 1, + )?; + out.per_table.insert("ProbeSet", rows.len()); + let probeset_ids: BTreeSet<i64> = rows.iter().filter_map(|r| r[0]).collect(); + for name in query_strings( + conn, + &format!( + "SELECT DISTINCT f.Name FROM {probefreeze} f \ + JOIN {probexref} x ON x.ProbeSetFreezeId = f.Id \ + WHERE {}", + int_in_clause("x.ProbeSetId", &probeset_ids) + ), + )? { + out.datasets.insert(name); + } + + // --- InbredSet (group term owner) --- + let inbredset = table_name("InbredSet"); + let rows = query_i64_rows( + conn, + &format!("SELECT DISTINCT Id FROM {inbredset} FOR SYSTEM_TIME ALL WHERE {window}"), + 1, + )?; + out.per_table.insert("InbredSet", rows.len()); + let group_ids: BTreeSet<i64> = rows.iter().filter_map(|r| r[0]).collect(); + for name in query_strings( + conn, + &format!( + "SELECT DISTINCT Name FROM {pubfreeze} WHERE {}", + int_in_clause("InbredSetId", &group_ids) + ), + )? { + out.datasets.insert(name); + } + for name in query_strings( + conn, + &format!( + "SELECT DISTINCT f.Name FROM {probefreeze} f \ + JOIN ProbeFreeze pr ON pr.Id = f.ProbeFreezeId \ + WHERE {}", + int_in_clause("pr.InbredSetId", &group_ids) + ), + )? { + out.datasets.insert(name); + } + + // --- Species (species term owner) --- + let species = table_name("Species"); + let rows = query_i64_rows( + conn, + &format!("SELECT DISTINCT Id FROM {species} FOR SYSTEM_TIME ALL WHERE {window}"), + 1, + )?; + out.per_table.insert("Species", rows.len()); + let species_ids: BTreeSet<i64> = rows.iter().filter_map(|r| r[0]).collect(); + for name in query_strings( + conn, + &format!( + "SELECT DISTINCT pf.Name FROM {pubfreeze} pf \ + JOIN {inbredset} i ON i.Id = pf.InbredSetId \ + WHERE {}", + int_in_clause("i.SpeciesId", &species_ids) + ), + )? { + out.datasets.insert(name); + } + for name in query_strings( + conn, + &format!( + "SELECT DISTINCT f.Name FROM {probefreeze} f \ + JOIN ProbeFreeze pr ON pr.Id = f.ProbeFreezeId \ + JOIN {inbredset} i2 ON i2.Id = pr.InbredSetId \ + WHERE {}", + int_in_clause("i2.SpeciesId", &species_ids) + ), + )? { + out.datasets.insert(name); + } + + Ok(out) +} + +/// Run a query and return each row as a vector of `Option<i64>` (one per +/// key column), skipping rows whose key columns are all NULL. +fn query_i64_rows( + conn: &mut Conn, + sql: &str, + columns: usize, +) -> Result<Vec<Vec<Option<i64>>>, String> { + let mut out = Vec::new(); + let result = conn + .query_iter(sql) + .map_err(|e| format!("delta query failed: {e}"))?; + for row in result { + let row = row.map_err(|e| format!("delta query failed: {e}"))?; + let mut values = Vec::with_capacity(columns); + for index in 0..columns { + let value: Option<i64> = int_at(&row, index)?; + values.push(value); + } + if values.iter().any(|v| v.is_some()) { + out.push(values); + } + } + Ok(out) +} + +/// Run a query returning a single integer column, dropping NULLs. +fn query_ints(conn: &mut Conn, sql: &str) -> Result<Vec<i64>, String> { + let rows = query_i64_rows(conn, sql, 1)?; + Ok(rows + .into_iter() + .filter_map(|mut row| row.pop().flatten()) + .collect()) +} + +/// Run a query returning a single string column, dropping NULLs. +fn query_strings(conn: &mut Conn, sql: &str) -> Result<Vec<String>, String> { + let mut out = Vec::new(); + let result = conn + .query_iter(sql) + .map_err(|e| format!("delta query failed: {e}"))?; + for row in result { + let row = row.map_err(|e| format!("delta query failed: {e}"))?; + if let Some(value) = text_at(&row, 0)? { + out.push(value); + } + } + Ok(out) +} + +fn int_at(row: &Row, index: usize) -> Result<Option<i64>, String> { + match row.get_opt::<Option<i64>, _>(index) { + Some(Ok(value)) => Ok(value), + Some(Err(e)) => Err(format!("column {index}: {e}")), + None => Err(format!("column {index} not in result set")), + } +} + +fn text_at(row: &Row, index: usize) -> Result<Option<String>, String> { + match row.get_opt::<Option<String>, _>(index) { + Some(Ok(value)) => Ok(value), + Some(Err(e)) => Err(format!("column {index}: {e}")), + None => Err(format!("column {index} not in result set")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ts(text: &str) -> DbTimestamp { + DbTimestamp::parse(text).expect("valid timestamp") + } + + #[test] + fn marker_parse_round_trips_and_rejects_garbage() { + for text in [ + "2026-10-13 19:37:47.469634", + "2038-01-19 06:14:07.999999", + "1970-01-01 00:00:00.000000", + "2026-12-31 23:59:59.999999", + ] { + let parsed = ts(text); + assert_eq!(parsed.as_str(), text); + assert_eq!(parsed.to_string(), text); + } + for bad in [ + "", + "2026-10-13", + "2026-10-13 19:37:47", + "2026-10-13 19:37:47.46963", // 5 micro digits + "2026-10-13 19:37:47.4696340", // 7 micro digits + "2026-13-13 19:37:47.000000", // month 13 + "2026-10-13 24:37:47.000000", // hour 24 + "2026-10-1 19:37:47.000000", // misaligned + "not a timestamp", + ] { + assert!(DbTimestamp::parse(bad).is_none(), "must reject {bad:?}"); + } + } + + #[test] + fn ordering_is_chronological() { + assert!(ts("2026-10-13 19:37:47.469634") < ts("2026-10-13 19:37:47.469635")); + assert!(ts("2026-10-13 19:37:47.469634") < ts("2026-10-13 19:37:48.000000")); + assert!(ts("2026-10-13 23:59:59.999999") < ts("2026-10-14 00:00:00.000000")); + } + + #[test] + fn floor_subtraction_crosses_second_and_minute_boundaries() { + assert_eq!( + window_floor(&ts("2026-10-13 19:37:47.469634"), 30).as_str(), + "2026-10-13 19:37:17.469634" + ); + assert_eq!( + window_floor(&ts("2026-10-13 19:37:10.000000"), 30).as_str(), + "2026-10-13 19:36:40.000000" + ); + // Across a minute/hour/day boundary. + assert_eq!( + window_floor(&ts("2026-10-14 00:00:05.000000"), 30).as_str(), + "2026-10-13 23:59:35.000000" + ); + // Saturates at the epoch instead of underflowing. + assert_eq!( + window_floor(&ts("1970-01-01 00:00:00.000000"), 30).as_str(), + "1970-01-01 00:00:00.000000" + ); + } + + #[test] + fn default_overlap_is_thirty_seconds() { + // No env var in the unit-test process; the default is 30. + assert_eq!(overlap_seconds(), 30); + } + + #[test] + fn decide_fallback_table() { + let new = ts("2026-10-13 19:40:00.000000"); + + // Rule 2: versioning absent wins even with an otherwise-good marker. + assert_eq!( + decide(Some("2026-10-13 19:39:00.000000"), &new, false, None), + Startup::Full("versioning absent".to_string()) + ); + // Rule 3: NULL / unparseable marker. + assert_eq!( + decide(None, &new, true, None), + Startup::Full("no marker".to_string()) + ); + assert_eq!( + decide(Some("garbage"), &new, true, None), + Startup::Full("no marker".to_string()) + ); + // Rule 4: marker in the future. + assert_eq!( + decide(Some("2026-10-13 19:41:00.000000"), &new, true, None), + Startup::Full("marker in future".to_string()) + ); + // Rule 5: non-monotonic (equal) -> retry next tick. + let equal = decide(Some(new.as_str()), &new, true, None); + match equal { + Startup::Full(reason) => { + assert!(reason.contains("not monotonic"), "{reason}"); + assert!(reason.contains("retry next tick"), "{reason}"); + } + other => panic!("equal marker must fall back, got {other:?}"), + } + // Rule 6: history purged (oldest resolvable newer than the marker). + let prev = ts("2026-10-13 19:39:00.000000"); + assert_eq!( + decide( + Some(prev.as_str()), + &new, + true, + Some(&ts("2026-10-13 19:39:30.000000")) + ), + Startup::Full("history purged".to_string()) + ); + // Healthy: prev < new and oldest <= prev -> delta at the overlap floor. + match decide( + Some(prev.as_str()), + &new, + true, + Some(&ts("2026-10-13 19:00:00.000000")), + ) { + Startup::Delta { floor, .. } => { + assert_eq!(floor.as_str(), "2026-10-13 19:38:30.000000") + } + other => panic!("healthy marker must run delta, got {other:?}"), + } + } + + #[test] + fn prefix_resolver_requires_the_second_opt_in() { + // Prefix set, but no opt-in and a release build: ignored. + assert_eq!(resolve_prefix(Some("vdelta_"), None, false), None); + assert_eq!(resolve_prefix(Some("vdelta_"), Some("0"), false), None); + // Explicit test env, or a debug build: honoured. + assert_eq!( + resolve_prefix(Some("vdelta_"), Some("1"), false), + Some("vdelta_".to_string()) + ); + assert_eq!( + resolve_prefix(Some("vdelta_"), None, true), + Some("vdelta_".to_string()) + ); + // Empty prefix: nothing. + assert_eq!(resolve_prefix(Some(""), Some("1"), true), None); + assert_eq!(resolve_prefix(None, Some("1"), true), None); + } + + #[test] + fn table_name_honours_the_prefix_override() { + set_table_prefix_override(None); + assert_eq!(table_name("PublishXRef"), "PublishXRef"); + set_table_prefix_override(Some("vdelta_".to_string())); + assert_eq!(table_name("PublishXRef"), "vdelta_PublishXRef"); + set_table_prefix_override(None); + } + + #[test] + fn quote_sql_string_escapes_quotes_and_backslashes() { + assert_eq!(quote_sql_string("BXDPublish"), "'BXDPublish'"); + assert_eq!(quote_sql_string("O'Brien"), "'O''Brien'"); + assert_eq!(quote_sql_string("a\\b"), "'a\\\\b'"); + assert_eq!(quote_sql_string("a\\'b"), "'a\\\\''b'"); + } + + #[test] + fn label_in_clause_chunks_and_escapes() { + let labels: BTreeSet<String> = ["BXDPublish".to_string(), "O'Brien".to_string()] + .into_iter() + .collect(); + let clause = label_in_clause("PublishFreeze.Name COLLATE utf8mb4_bin", &labels); + assert!( + clause.contains("PublishFreeze.Name COLLATE utf8mb4_bin IN ("), + "{clause}" + ); + assert!(clause.contains("'O''Brien'"), "{clause}"); + assert_eq!(label_in_clause("x", &BTreeSet::new()), "0=1"); + + // More than IN_CHUNK labels split into OR-groups. + let many: BTreeSet<String> = (0..(IN_CHUNK + 5)).map(|i| format!("D{i}")).collect(); + let clause = label_in_clause("col", &many); + assert!(clause.starts_with('('), "{clause}"); + assert_eq!(clause.matches("col IN (").count(), 2, "{clause}"); + } + + #[test] + fn window_predicate_covers_start_and_end() { + let predicate = window_predicate( + &ts("2026-10-13 19:37:00.000000"), + &ts("2026-10-13 19:40:00.000000"), + ); + assert!(predicate.contains("ROW_START >=") && predicate.contains("ROW_START <")); + assert!(predicate.contains("ROW_END >=") && predicate.contains("ROW_END <")); + } + + #[test] + fn versioning_probe_sql_shape() { + set_table_prefix_override(None); + // The probe must key on TABLE_TYPE (MariaDB hides the implicit + // ROW_START column from information_schema.COLUMNS) and cover all + // seven tables. + for base in VERSIONED_TABLES { + let table = table_name(base); + let sql = format!( + "SELECT COUNT(*) FROM information_schema.TABLES \ + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = {} \ + AND TABLE_TYPE = 'SYSTEM VERSIONED'", + quote_sql_string(&table) + ); + assert!(sql.contains("information_schema.TABLES"), "{sql}"); + assert!(sql.contains("TABLE_TYPE = 'SYSTEM VERSIONED'"), "{sql}"); + assert!(sql.contains(&format!("'{}'", table)), "{sql}"); + } + // With the prefix active the probe follows it. + set_table_prefix_override(Some("vdelta_".to_string())); + assert_eq!(table_name("PublishXRef"), "vdelta_PublishXRef"); + set_table_prefix_override(None); + } + + #[test] + fn oldest_history_sql_is_the_seven_way_union() { + set_table_prefix_override(None); + let mut parts: Vec<String> = Vec::new(); + for base in VERSIONED_TABLES { + let table = table_name(base); + parts.push(format!( + "SELECT MIN(ROW_START) AS oldest FROM {table} FOR SYSTEM_TIME ALL" + )); + } + let sql = format!("SELECT MIN(oldest) FROM ({}) u", parts.join(" UNION ALL ")); + assert_eq!(sql.matches("FOR SYSTEM_TIME ALL").count(), 7); + assert_eq!(sql.matches("UNION ALL").count(), 6); + } +} diff --git a/src/indexer/mod.rs b/src/indexer/mod.rs index 7972b54..336a620 100644 --- a/src/indexer/mod.rs +++ b/src/indexer/mod.rs @@ -8,13 +8,18 @@ //! their stable identity term, so a per-document update is visible with //! no service stop. //! -//! Two modes, one mechanism: +//! Three modes, one mechanism: //! //! * FULL (bootstrap): crawl every public, non-confidential freeze and //! build a fresh index. A missing or empty sidecar means FULL. //! * INCREMENTAL: crawl, hash each document's complete source, diff //! against the sidecar, replace only changed documents, delete //! vanished identity terms, and write nothing when nothing changed. +//! * DELTA: name changed rows via system versioning (`ROW_START`/ +//! `ROW_END`), re-diff only the affected datasets, and fall back to +//! FULL on any doubt. The periodic INCREMENTAL sweep remains the +//! correctness bound (cache text and unversioned tables are invisible +//! to the delta). //! //! The phenotype crawl carries `confidentiality < 1 AND public > 0` //! (the Python query has no WHERE clause, which is the metadata leak @@ -22,6 +27,7 @@ pub mod build; pub mod cache; +pub mod delta; pub mod livemem; pub mod pyjson; pub mod rows; @@ -30,4 +36,4 @@ pub mod schema; pub mod sha256; pub mod sidecar; -pub use run::{RunConfig, Stats, run_full, run_incremental}; +pub use run::{RunConfig, Stats, run_delta, run_full, run_incremental}; diff --git a/src/indexer/rows.rs b/src/indexer/rows.rs index 7c9d5a4..996cef6 100644 --- a/src/indexer/rows.rs +++ b/src/indexer/rows.rs @@ -29,18 +29,26 @@ //! assumption the byte-order merge below relies on. Pinning //! `COLLATE utf8mb4_bin` makes the SQL order match the merge's byte order. +use std::collections::BTreeSet; use std::sync::mpsc::{Receiver, Sender}; use mysql::prelude::Queryable; use mysql::{Conn, Opts, OptsBuilder, Row}; -/// The phenotype filter, fully qualified (the table's `public` column -/// collides with InbredSet.public, so both predicates need the table -/// name). -const PHENOTYPE_PUBLIC_FILTER: &str = - "PublishFreeze.confidentiality < 1 AND PublishFreeze.public > 0"; -/// The gene filter, fully qualified against ProbeSetFreeze. -const GENE_PUBLIC_FILTER: &str = "ProbeSetFreeze.confidentiality < 1 AND ProbeSetFreeze.public > 0"; +use super::delta; + +/// The gene filter, fully qualified against the (resolved) ProbeSetFreeze. +fn gene_public_filter() -> String { + let psf = delta::table_name("ProbeSetFreeze"); + format!("{psf}.confidentiality < 1 AND {psf}.public > 0") +} + +/// The phenotype filter, fully qualified against the (resolved) +/// PublishFreeze. +fn phenotype_public_filter() -> String { + let pf = delta::table_name("PublishFreeze"); + format!("{pf}.confidentiality < 1 AND {pf}.public > 0") +} /// One expression (gene) crawl row; each field is absent when SQL /// returned NULL (the indexer's `MonadicDict` drops NULLs). @@ -120,7 +128,7 @@ pub trait DatasetSink { fn end_dataset(&mut self, label: &str) -> Result<(), String>; } -fn connect(sql_uri: &str) -> Result<Conn, String> { +pub(crate) fn connect(sql_uri: &str) -> Result<Conn, String> { let opts = Opts::from_url(sql_uri).map_err(|e| format!("bad SQL_URI: {e}"))?; Conn::new(OptsBuilder::from_opts(opts)).map_err(|e| format!("cannot connect: {e}")) } @@ -149,40 +157,71 @@ fn integer(row: &Row, column: &str) -> Result<Option<i64>, String> { } } -fn genes_sql() -> String { +fn genes_sql(datasets: Option<&BTreeSet<String>>) -> String { + let species = delta::table_name("Species"); + let inbredset = delta::table_name("InbredSet"); + // ProbeFreeze and Tissue are contributing but NOT versioned, so they + // are never prefixed (the joins by `Id` resolve against the real tables). + let probefreeze = "ProbeFreeze"; + let tissue = "Tissue"; + let psf = delta::table_name("ProbeSetFreeze"); + let psx = delta::table_name("ProbeSetXRef"); + let probeset = delta::table_name("ProbeSet"); + // The scoped filter (delta path): restrict the crawl to the affected + // dataset labels, compared with the same binary collation the stream + // groups by. `None` is the unscoped FULL/INCREMENTAL crawl. + let scope = match datasets { + Some(labels) => format!( + " AND {}", + delta::label_in_clause(&format!("{psf}.Name COLLATE utf8mb4_bin"), labels) + ), + None => String::new(), + }; + let public_filter = gene_public_filter(); format!( - "SELECT ProbeSet.Name AS name, ProbeSet.Symbol AS symbol, \ - ProbeSet.description AS description, ProbeSet.Chr AS chr, ProbeSet.Mb as mb, \ - ProbeSet.alias AS alias, ProbeSet.GenbankId AS genbankid, \ - ProbeSet.UniGeneId AS unigeneid, \ - ProbeSet.Probe_Target_Description AS probe_target_description, \ - ProbeSetFreeze.Name AS dataset, ProbeSetFreeze.FullName AS dataset_fullname, \ - Species.Name AS species, InbredSet.Name AS `group`, Tissue.Name AS tissue, \ - ProbeSetXRef.Mean AS mean, ProbeSetXRef.LRS AS lrs, \ - ProbeSetXRef.additive AS additive, Geno.Chr AS geno_chr, Geno.Mb as geno_mb \ - FROM Species \ - INNER JOIN InbredSet ON InbredSet.SpeciesId = Species.Id \ - INNER JOIN ProbeFreeze ON ProbeFreeze.InbredSetId = InbredSet.Id \ - INNER JOIN Tissue ON ProbeFreeze.TissueId = Tissue.Id \ - INNER JOIN ProbeSetFreeze ON ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id \ - INNER JOIN ProbeSetXRef ON ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id \ - INNER JOIN ProbeSet ON ProbeSet.Id = ProbeSetXRef.ProbeSetId \ - LEFT JOIN Geno ON ProbeSetXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id \ - WHERE {GENE_PUBLIC_FILTER} \ + "SELECT {probeset}.Name AS name, {probeset}.Symbol AS symbol, \ + {probeset}.description AS description, {probeset}.Chr AS chr, {probeset}.Mb as mb, \ + {probeset}.alias AS alias, {probeset}.GenbankId AS genbankid, \ + {probeset}.UniGeneId AS unigeneid, \ + {probeset}.Probe_Target_Description AS probe_target_description, \ + {psf}.Name AS dataset, {psf}.FullName AS dataset_fullname, \ + {species}.Name AS species, {inbredset}.Name AS `group`, {tissue}.Name AS tissue, \ + {psx}.Mean AS mean, {psx}.LRS AS lrs, \ + {psx}.additive AS additive, Geno.Chr AS geno_chr, Geno.Mb as geno_mb \ + FROM {species} \ + INNER JOIN {inbredset} ON {inbredset}.SpeciesId = {species}.Id \ + INNER JOIN {probefreeze} ON {probefreeze}.InbredSetId = {inbredset}.Id \ + INNER JOIN {tissue} ON {probefreeze}.TissueId = {tissue}.Id \ + INNER JOIN {psf} ON {psf}.ProbeFreezeId = {probefreeze}.Id \ + INNER JOIN {psx} ON {psx}.ProbeSetFreezeId = {psf}.Id \ + INNER JOIN {probeset} ON {probeset}.Id = {psx}.ProbeSetId \ + LEFT JOIN Geno ON {psx}.Locus = Geno.Name AND Geno.SpeciesId = {species}.Id \ + WHERE {public_filter}{scope} \ ORDER BY dataset COLLATE utf8mb4_bin" ) } -fn phenotypes_sql(filtered: bool) -> String { +fn phenotypes_sql(filtered: bool, datasets: Option<&BTreeSet<String>>) -> String { + let species = delta::table_name("Species"); + let inbredset = delta::table_name("InbredSet"); + let pf = delta::table_name("PublishFreeze"); + let pubxref = delta::table_name("PublishXRef"); + let scope = match datasets { + Some(labels) => format!( + " AND {}", + delta::label_in_clause(&format!("{pf}.Name COLLATE utf8mb4_bin"), labels) + ), + None => String::new(), + }; let where_clause = if filtered { - format!(" WHERE {PHENOTYPE_PUBLIC_FILTER}") + format!(" WHERE {}", phenotype_public_filter()) } else { String::new() }; format!( - "SELECT Species.Name AS species, InbredSet.Name AS `group`, \ - PublishFreeze.Name AS dataset, PublishFreeze.FullName AS dataset_fullname, \ - PublishXRef.Id AS name, \ + "SELECT {species}.Name AS species, {inbredset}.Name AS `group`, \ + {pf}.Name AS dataset, {pf}.FullName AS dataset_fullname, \ + {pubxref}.Id AS name, \ COALESCE(Phenotype.Post_publication_abbreviation, \ Phenotype.Pre_publication_abbreviation) AS abbreviation, \ COALESCE(Phenotype.Post_publication_description, \ @@ -191,18 +230,18 @@ fn phenotypes_sql(filtered: bool) -> String { Publication.Authors AS authors, \ IF(CONVERT(Publication.Year, UNSIGNED)=0, NULL, \ CONVERT(Publication.Year, UNSIGNED)) AS year, \ - Publication.PubMed_ID AS pubmed_id, PublishXRef.LRS as lrs, \ - PublishXRef.additive, InbredSet.InbredSetCode AS inbredsetcode, \ - PublishXRef.mean, PublishXRef.NSamples AS nsamples, \ + Publication.PubMed_ID AS pubmed_id, {pubxref}.LRS as lrs, \ + {pubxref}.additive, {inbredset}.InbredSetCode AS inbredsetcode, \ + {pubxref}.mean, {pubxref}.NSamples AS nsamples, \ Geno.Chr as geno_chr, Geno.Mb as geno_mb \ - FROM Species \ - INNER JOIN InbredSet ON InbredSet.SpeciesId = Species.Id \ - INNER JOIN PublishFreeze ON PublishFreeze.InbredSetId = InbredSet.Id \ - INNER JOIN PublishXRef ON PublishXRef.InbredSetId = InbredSet.Id \ - INNER JOIN Phenotype ON PublishXRef.PhenotypeId = Phenotype.Id \ - INNER JOIN Publication ON PublishXRef.PublicationId = Publication.Id \ - LEFT JOIN Geno ON PublishXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id\ - {where_clause} \ + FROM {species} \ + INNER JOIN {inbredset} ON {inbredset}.SpeciesId = {species}.Id \ + INNER JOIN {pf} ON {pf}.InbredSetId = {inbredset}.Id \ + INNER JOIN {pubxref} ON {pubxref}.InbredSetId = {inbredset}.Id \ + INNER JOIN Phenotype ON {pubxref}.PhenotypeId = Phenotype.Id \ + INNER JOIN Publication ON {pubxref}.PublicationId = Publication.Id \ + LEFT JOIN Geno ON {pubxref}.Locus = Geno.Name AND Geno.SpeciesId = {species}.Id\ + {where_clause}{scope} \ ORDER BY dataset COLLATE utf8mb4_bin" ) } @@ -212,10 +251,14 @@ type Item = Result<(String, CrawlRow), String>; /// Producer thread for the gene crawl: stream rows ordered by dataset and /// send each as `(dataset, CrawlRow)`. Rows with no dataset produce no /// document and are skipped. -fn produce_genes(sql_uri: &str, tx: &Sender<Item>) -> Result<(), String> { +fn produce_genes( + sql_uri: &str, + datasets: Option<BTreeSet<String>>, + tx: &Sender<Item>, +) -> Result<(), String> { let mut conn = connect(sql_uri)?; let result = conn - .query_iter(genes_sql()) + .query_iter(genes_sql(datasets.as_ref())) .map_err(|e| format!("gene crawl failed: {e}"))?; for row in result { let row = row.map_err(|e| format!("gene crawl failed: {e}"))?; @@ -251,10 +294,15 @@ fn produce_genes(sql_uri: &str, tx: &Sender<Item>) -> Result<(), String> { } /// Producer thread for the phenotype crawl. -fn produce_phenotypes(sql_uri: &str, filtered: bool, tx: &Sender<Item>) -> Result<(), String> { +fn produce_phenotypes( + sql_uri: &str, + filtered: bool, + datasets: Option<BTreeSet<String>>, + tx: &Sender<Item>, +) -> Result<(), String> { let mut conn = connect(sql_uri)?; let result = conn - .query_iter(phenotypes_sql(filtered)) + .query_iter(phenotypes_sql(filtered, datasets.as_ref())) .map_err(|e| format!("phenotype crawl failed: {e}"))?; for row in result { let row = row.map_err(|e| format!("phenotype crawl failed: {e}"))?; @@ -320,6 +368,20 @@ pub fn stream_dataset_groups( filtered: bool, sink: &mut dyn DatasetSink, ) -> Result<(), String> { + stream_dataset_groups_scoped(sql_uri, filtered, None, sink) +} + +/// The dataset-scoped crawl: like [`stream_dataset_groups`], but the SQL +/// restricts both producers to the affected `datasets` (compared with +/// `COLLATE utf8mb4_bin`), so labels outside the set cannot appear. This +/// is the DELTA write path's crawl; `None` reproduces the unscoped FULL/ +/// INCREMENTAL crawl exactly. +pub fn stream_dataset_groups_scoped( + sql_uri: &str, + filtered: bool, + datasets: Option<&BTreeSet<String>>, + sink: &mut dyn DatasetSink, +) -> Result<(), String> { // Unbounded channels: the producers stream the two SQL result sets in // their own threads and the merge consumes them; this is the fast // path (a bounded channel was tried and made the scaled run several @@ -328,9 +390,13 @@ pub fn stream_dataset_groups( let (phen_tx, phen_rx) = std::sync::mpsc::channel::<Item>(); let gene_uri = sql_uri.to_string(); let phen_uri = sql_uri.to_string(); - let genes = spawn_producer(gene_tx, move |tx| produce_genes(&gene_uri, tx)); + let gene_datasets = datasets.cloned(); + let phen_datasets = datasets.cloned(); + let genes = spawn_producer(gene_tx, move |tx| { + produce_genes(&gene_uri, gene_datasets, tx) + }); let phenotypes = spawn_producer(phen_tx, move |tx| { - produce_phenotypes(&phen_uri, filtered, tx) + produce_phenotypes(&phen_uri, filtered, phen_datasets, tx) }); let outcome = merge_streams(&gene_rx, &phen_rx, sink); @@ -397,6 +463,7 @@ fn drain_label( #[cfg(test)] mod tests { use super::*; + use crate::indexer::delta; /// The stream order must match the byte order `merge_streams` compares /// with. The `Name` columns are `utf8mb4_general_ci`; without the @@ -404,11 +471,90 @@ mod tests { /// dataset group across two `begin_dataset` calls. #[test] fn crawl_orders_datasets_bin_collated() { - for sql in [genes_sql(), phenotypes_sql(true), phenotypes_sql(false)] { + for sql in [ + genes_sql(None), + phenotypes_sql(true, None), + phenotypes_sql(false, None), + ] { assert!( sql.contains("ORDER BY dataset COLLATE utf8mb4_bin"), "crawl must order by the binary collation: {sql}" ); } } + + /// Unscoped SQL (the production FULL/INCREMENTAL shape) names the base + /// tables and carries no prefix or dataset filter. + #[test] + fn unscoped_crawl_has_no_prefix_and_no_scope_filter() { + delta::set_table_prefix_override(None); + for sql in [ + genes_sql(None), + phenotypes_sql(true, None), + phenotypes_sql(false, None), + ] { + assert!( + !sql.contains("vdelta_"), + "unscoped SQL must not be prefixed: {sql}" + ); + assert!( + sql.contains("FROM Species"), + "unscoped SQL uses base names: {sql}" + ); + assert!( + !sql.contains("COLLATE utf8mb4_bin IN ("), + "unscoped SQL has no dataset IN filter: {sql}" + ); + } + } + + /// The scoped crawl appends the affected-label filter with the binary + /// collation, on the *dataset-owner* table, and honours the prefix seam + /// for the versioned tables (A2/A4/A5). + #[test] + fn scoped_crawl_filters_by_dataset_with_binary_collation() { + delta::set_table_prefix_override(None); + let labels: BTreeSet<String> = ["BXDPublish".to_string(), "O'Brien".to_string()] + .into_iter() + .collect(); + + let genes = genes_sql(Some(&labels)); + assert!( + genes.contains("ProbeSetFreeze.Name COLLATE utf8mb4_bin IN ("), + "gene scope filter: {genes}" + ); + assert!(genes.contains("'O''Brien'"), "label escaping (A5): {genes}"); + assert!( + genes.contains("WHERE ProbeSetFreeze.confidentiality"), + "{genes}" + ); + + let phen = phenotypes_sql(true, Some(&labels)); + assert!( + phen.contains("PublishFreeze.Name COLLATE utf8mb4_bin IN ("), + "phenotype scope filter: {phen}" + ); + assert!(phen.contains(" AND PublishFreeze.Name COLLATE"), "{phen}"); + + // Prefix seam: both the FROM tables and the dataset filter move to + // the vdelta_ namespace, so delta and crawl agree (A4). + delta::set_table_prefix_override(Some("vdelta_".to_string())); + let genes = genes_sql(Some(&labels)); + assert!(genes.contains("FROM vdelta_Species"), "{genes}"); + assert!( + genes.contains("INNER JOIN vdelta_ProbeSetFreeze"), + "{genes}" + ); + assert!( + genes.contains("vdelta_ProbeSetFreeze.Name COLLATE utf8mb4_bin IN ("), + "{genes}" + ); + assert!( + genes.contains("WHERE vdelta_ProbeSetFreeze.confidentiality"), + "{genes}" + ); + // Non-versioned contributing tables stay unprefixed. + assert!(genes.contains("LEFT JOIN Geno ON"), "{genes}"); + delta::set_table_prefix_override(None); + } } diff --git a/src/indexer/run.rs b/src/indexer/run.rs index 1578590..30335f2 100644 --- a/src/indexer/run.rs +++ b/src/indexer/run.rs @@ -37,6 +37,7 @@ use xapian_rs::{Database, DbAction, DbBackend, Enquire, Query, WritableDatabase} use super::build::{BuiltDocument, build_gene, build_phenotype, gene_idterm, phenotype_idterm}; use super::cache::RdfCaches; +use super::delta; use super::livemem::{self, CountedDocument}; use super::rows::{self, CrawlRow, DatasetSink}; use super::schema; @@ -78,6 +79,15 @@ pub struct Stats { pub chunk_rss_peak_kib: u64, /// How the run classified startup (`incremental` or a FULL reason). pub startup: String, + /// DELTA: candidates per versioned table (this run's window). + pub candidates: BTreeMap<String, usize>, + /// DELTA: dataset labels the window affects. + pub affected_datasets: usize, + /// DELTA: the previous coverage marker (`None` when absent). + pub marker_from: Option<String>, + /// DELTA: the marker this run advanced to (`None` for FULL/INCREMENTAL + /// when versioning is absent). + pub marker_to: Option<String>, } /// Inputs for one run. @@ -242,6 +252,15 @@ impl DuplicateSink { labels: HashMap::new(), } } + + /// Pre-seed the cross-dataset label map from an existing label set + /// (the manifest's shard keys). "First raw label wins", exactly as if + /// the crawl itself had yielded them. + fn seed_labels<I: IntoIterator<Item = String>>(&mut self, labels: I) { + for label in labels { + self.labels.entry(label.to_lowercase()).or_insert(label); + } + } } impl DatasetSink for DuplicateSink { @@ -281,8 +300,25 @@ impl DatasetSink for DuplicateSink { /// (empty when the crawl is unique). The per-dataset set is bounded by /// the largest dataset, and the pass finishes before any index write. fn duplicate_prepass(sql_uri: &str, filtered: bool) -> Result<(Vec<String>, usize), String> { + duplicate_prepass_scoped(sql_uri, filtered, None, std::iter::empty()) +} + +/// The scoped duplicate pre-pass (B1): stream only `datasets` through the +/// [`DuplicateSink`], with its cross-dataset label map PRE-SEEDED from +/// every existing label (`seed_labels`, typically all manifest shard +/// keys). This catches a case-variant collision between an affected label +/// and an out-of-scope shard, and (via the per-dataset set) a +/// within-dataset identity-term collision. `datasets = None` is the +/// unscoped FULL/INCREMENTAL pass. +fn duplicate_prepass_scoped<I: IntoIterator<Item = String>>( + sql_uri: &str, + filtered: bool, + datasets: Option<&BTreeSet<String>>, + seed_labels: I, +) -> Result<(Vec<String>, usize), String> { let mut sink = DuplicateSink::new(); - rows::stream_dataset_groups(sql_uri, filtered, &mut sink)?; + sink.seed_labels(seed_labels); + rows::stream_dataset_groups_scoped(sql_uri, filtered, datasets, &mut sink)?; Ok((sink.duplicates, sink.max_terms)) } @@ -430,7 +466,14 @@ struct SidecarBuild { } impl SidecarBuild { - fn new(index_path: &Path, mode: &str, rif_hash: String, wiki_hash: String) -> Self { + fn new( + index_path: &Path, + mode: &str, + rif_hash: String, + wiki_hash: String, + versioning_marker: Option<String>, + versioning_flavour: Option<String>, + ) -> Self { let stage_dir = PathBuf::from(format!( "{}.tmp.{}", sidecar::shard_dir(index_path).display(), @@ -446,6 +489,8 @@ impl SidecarBuild { doc_count: 0, rif_hash, wiki_hash, + versioning_marker, + versioning_flavour, shards: BTreeMap::new(), }, staged: Vec::new(), @@ -532,6 +577,9 @@ pub fn run_full(config: &RunConfig) -> Result<Stats, String> { fn run_full_with_reason(config: &RunConfig, reason: Option<String>) -> Result<Stats, String> { let caches = RdfCaches::load(&config.rdf_cache)?; + // A fresh coverage point, captured BEFORE the crawl; `None` when + // versioning is absent. Rides the manifest, written last. + let coverage = delta::capture_coverage_marker(&config.sql_uri); // Duplicate validation happens BEFORE any index is opened or written. let (duplicates, max_dataset_terms) = duplicate_prepass(&config.sql_uri, config.publish_filter)?; @@ -561,6 +609,8 @@ fn run_full_with_reason(config: &RunConfig, reason: Option<String>) -> Result<St "full", caches.rif.hash.clone(), caches.wiki.hash.clone(), + coverage.as_ref().map(|c| c.0.clone()), + coverage.as_ref().map(|c| c.1.clone()), ); let outcome = stream_full(&temp, config, &caches, &mut build, retain); let (index_doc_count, built, pacer_chunks, pacer_rss_peak) = match outcome { @@ -606,6 +656,10 @@ fn run_full_with_reason(config: &RunConfig, reason: Option<String>) -> Result<St Some(reason) => format!("full ({reason})"), None => "full".to_string(), }, + candidates: BTreeMap::new(), + affected_datasets: 0, + marker_from: None, + marker_to: coverage.as_ref().map(|c| c.0.clone()), }; stats.mode = "full".to_string(); Ok(stats) @@ -704,22 +758,31 @@ fn stream_full( // INCREMENTAL // --------------------------------------------------------------------------- -/// Run an INCREMENTAL pass: cheap-first startup check, then a dataset- -/// grouped streaming diff that replaces changed documents, deletes -/// vanished ones, and writes nothing when nothing changed. -pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { +/// The outcome of the shared startup pre-flight (obj-3 cheap-first +/// checks). Both INCREMENTAL and DELTA use it; any doubt returns `Full`. +enum Pre { + /// The manifest is trustworthy; proceed with the given manifest. + Manifest(Manifest), + /// Fall back to FULL for the given reason. + Full(String), +} + +/// Cheap-first startup shared by INCREMENTAL and DELTA: load the manifest +/// (missing/old-version => FULL), check the index exists, compare the +/// manifest `doc_count` to the index's own (O(1)), and verify every shard +/// (bit-rot). Any mismatch routes to FULL with a reason; the expensive +/// identity-term enumeration runs only on the doc-count mismatch path. +fn preflight(config: &RunConfig) -> Result<Pre, String> { let Some(manifest) = Manifest::load(&config.index_path) else { // Missing, unreadable, or old-version sidecar means FULL. - return run_full_with_reason(config, Some("no/old sidecar".to_string())); + return Ok(Pre::Full("no/old sidecar".to_string())); }; if !index_exists(&config.index_path) { - return run_full_with_reason(config, Some("index missing".to_string())); + return Ok(Pre::Full("index missing".to_string())); } sidecar::cleanup_temp(&config.index_path)?; - // Cheap-first startup (O(1)): the manifest doc count against the - // index's own doc count. let probe = WritableDatabase::open( &config.index_path, Some(DbAction::CreateOrOpen), @@ -736,14 +799,14 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { index_doc_count, manifest.doc_count ); // Expensive path only here: enumerate index identity terms to - // characterise the key-set mismatch, then rebuild. + // characterise the key-set mismatch. let index_terms = index_identity_terms(&probe); eprintln!( "gn-indexer: startup mismatch ({reason}); index identity terms = {}; falling back to FULL", index_terms.len() ); drop(probe); - return run_full_with_reason(config, Some(reason)); + return Ok(Pre::Full(reason)); } drop(probe); @@ -756,11 +819,32 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { if let Err(error) = ResidentShard::load(&dir, label, entry) { let reason = format!("shard {label}: {error}"); eprintln!("gn-indexer: startup mismatch ({reason}); falling back to FULL"); - return run_full_with_reason(config, Some(reason)); + return Ok(Pre::Full(reason)); } } } + Ok(Pre::Manifest(manifest)) +} + +/// Run an INCREMENTAL pass: cheap-first startup check, then a dataset- +/// grouped streaming diff that replaces changed documents, deletes +/// vanished ones, and writes nothing when nothing changed. +pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { + let manifest = match preflight(config)? { + Pre::Manifest(manifest) => manifest, + Pre::Full(reason) => return run_full_with_reason(config, Some(reason)), + }; + + // An incremental re-hashes every source, so it is a full coverage + // point: capture a fresh marker before its crawl (versioning absent => + // None). On the `nothing_changed` skip below the manifest is not + // written, so the previous marker stays (safe: the next delta window + // is simply larger). + let coverage = delta::capture_coverage_marker(&config.sql_uri); + let red = red_mode(); + let no_recovery = red.as_deref() == Some("no-recovery"); + let caches = RdfCaches::load(&config.rdf_cache)?; // Duplicate validation before any write. let (duplicates, max_dataset_terms) = @@ -796,6 +880,8 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { "incremental", caches.rif.hash.clone(), caches.wiki.hash.clone(), + coverage.as_ref().map(|c| c.0.clone()), + coverage.as_ref().map(|c| c.1.clone()), ); // Carry forward every existing shard entry; changed datasets replace // their entry, removed datasets are dropped below. @@ -815,6 +901,7 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { build: &mut build, shard_dir, by_label, + affected_scope: None, seen: BTreeSet::new(), previous: None, previous_label: String::new(), @@ -839,12 +926,7 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { // Datasets in the manifest that the crawl never produced: every one // of their documents is gone. - let leftovers: Vec<String> = sink - .by_label - .keys() - .filter(|label| !sink.seen.contains(*label)) - .cloned() - .collect(); + let leftovers = sink.leftover_labels(); for label in leftovers { sink.delete_dataset(&label)?; sink.removed_shards += 1; @@ -897,9 +979,31 @@ pub fn run_incremental(config: &RunConfig) -> Result<Stats, String> { commit_chunks, chunk_rss_peak_kib, startup: "incremental".to_string(), + candidates: BTreeMap::new(), + affected_datasets: 0, + marker_from: None, + marker_to: coverage.as_ref().map(|c| c.0.clone()), }) } +/// The leftover-delete set: manifest labels the crawl never produced, +/// restricted to `scope` when set. Pure so the DELTA scoping is directly +/// testable (a scoped crawl must never delete an out-of-scope shard). +fn leftover_labels_in_scope<'a, I>( + labels: I, + seen: &BTreeSet<String>, + scope: Option<&BTreeSet<String>>, +) -> Vec<String> +where + I: Iterator<Item = &'a String>, +{ + labels + .filter(|label| !seen.contains(*label)) + .filter(|label| scope.map_or(true, |scope| scope.contains(*label))) + .cloned() + .collect() +} + struct IncrementalSink<'a> { db: WritableDatabase, caches: &'a RdfCaches, @@ -912,6 +1016,9 @@ struct IncrementalSink<'a> { build: &'a mut SidecarBuild, shard_dir: PathBuf, by_label: HashMap<String, ShardEntry>, + /// DELTA only: when set, the leftover-delete path is restricted to + /// these datasets (a scoped crawl cannot delete out-of-scope shards). + affected_scope: Option<BTreeSet<String>>, seen: BTreeSet<String>, previous: Option<ResidentShard>, previous_label: String, @@ -927,6 +1034,18 @@ struct IncrementalSink<'a> { } impl<'a> IncrementalSink<'a> { + /// Manifest labels the crawl never produced (all their documents are + /// gone), optionally restricted to `affected_scope`. Without a scope + /// this is the INCREMENTAL semantics (every leftover); with one it is + /// the DELTA semantics (only in-window datasets may be deleted). + fn leftover_labels(&self) -> Vec<String> { + leftover_labels_in_scope( + self.by_label.keys(), + &self.seen, + self.affected_scope.as_ref(), + ) + } + /// Delete every document of a dataset (all its rows vanished) and /// drop its shard. fn delete_dataset(&mut self, label: &str) -> Result<(), String> { @@ -1067,6 +1186,231 @@ impl<'a> DatasetSink for IncrementalSink<'a> { } } +/// Run a DELTA pass: name changed rows via system versioning, re-diff only +/// the affected datasets through the existing build+hash+compare path, and +/// advance the coverage marker. Every doubt (missing/old sidecar, index or +/// shard mismatch, versioning absent, NULL/future/non-monotonic marker, +/// purged history) falls back to FULL with a logged reason. +pub fn run_delta(config: &RunConfig) -> Result<Stats, String> { + delta::log_prefix_warning(); + + let manifest = match preflight(config)? { + Pre::Manifest(manifest) => manifest, + Pre::Full(reason) => return run_full_with_reason(config, Some(reason)), + }; + + // Probe versioning presence BEFORE any `FOR SYSTEM_TIME` query (A1): + // an unversioned table would error `Unknown column 'ROW_START'`. + let mut conn = delta::connect(&config.sql_uri)?; + if !delta::versioning_probe(&mut conn)? { + return run_full_with_reason(config, Some("versioning absent".to_string())); + } + // Capture the new marker BEFORE any candidate query (design §1.3). + let new_marker = delta::capture_now6(&mut conn)?; + let oldest = delta::oldest_history(&mut conn)?; + let floor = match delta::decide( + manifest.versioning_marker.as_deref(), + &new_marker, + true, + oldest.as_ref(), + ) { + delta::Startup::Full(reason) => return run_full_with_reason(config, Some(reason)), + delta::Startup::Delta { floor, .. } => floor, + }; + let candidates = delta::scan(&mut conn, &floor, &new_marker)?; + drop(conn); + + let marker_from = manifest.versioning_marker.clone(); + let marker_to = Some(new_marker.as_str().to_string()); + let candidate_counts: BTreeMap<String, usize> = candidates + .per_table + .iter() + .map(|(table, count)| (table.to_string(), *count)) + .collect(); + + // No candidates: the window was scanned and nothing needs rebuilding, + // so advance the marker only (an atomic manifest rewrite; the window + // must not grow unboundedly). + if candidates.datasets.is_empty() { + let mut updated = manifest.clone(); + updated.version = sidecar::SIDECAR_VERSION; + updated.mode = "delta".to_string(); + updated.versioning_marker = marker_to.clone(); + updated.versioning_flavour = Some(delta::FLAVOUR.to_string()); + updated.store(&config.index_path)?; + return Ok(Stats { + mode: "delta".to_string(), + built: updated.doc_count, + replaced: 0, + deleted: 0, + unchanged: updated.doc_count, + index_doc_count: updated.doc_count as u32, + duplicate_identity_terms: Vec::new(), + max_live_documents: 0, + shards_written: 0, + shards_deleted: 0, + max_resident_sidecar_entries: 0, + max_resident_dataset_terms: 0, + peak_open_shards: 0, + shard_bytes_read: 0, + commit_chunks: 0, + chunk_rss_peak_kib: 0, + startup: "delta".to_string(), + candidates: candidate_counts, + affected_datasets: 0, + marker_from, + marker_to, + }); + } + + let caches = RdfCaches::load(&config.rdf_cache)?; + + // B1: run the duplicate check, SCOPED, with the cross-dataset label map + // PRE-SEEDED from ALL manifest labels (so a case-variant collision + // against an out-of-scope shard is still caught). This is the only + // guard against within-dataset identity-term collisions and + // cross-dataset case-variant labels; skipping it would let the delta + // diverge from a fresh FULL. + let (duplicates, max_dataset_terms) = duplicate_prepass_scoped( + &config.sql_uri, + config.publish_filter, + Some(&candidates.datasets), + manifest.shards.keys().cloned(), + )?; + if !duplicates.is_empty() { + return Err(format!( + "duplicate identity terms in affected datasets: {}", + duplicates.join(", ") + )); + } + + livemem::reset(); + sidecar::mem::reset(); + let retain = livemem::red_retain(); + + let db = WritableDatabase::open( + &config.index_path, + Some(DbAction::CreateOrOpen), + Some(DbBackend::Glass), + None, + None, + ); + let shard_dir = sidecar::shard_dir(&config.index_path); + let by_label: HashMap<String, ShardEntry> = manifest + .shards + .iter() + .map(|(label, entry)| (label.clone(), entry.clone())) + .collect(); + + let mut build = SidecarBuild::new( + &config.index_path, + "delta", + caches.rif.hash.clone(), + caches.wiki.hash.clone(), + marker_to.clone(), + Some(delta::FLAVOUR.to_string()), + ); + // Carry forward every existing shard entry; affected datasets replace + // or remove theirs below. + for (label, entry) in &manifest.shards { + build.keep(label, entry.clone()); + } + + let mut sink = IncrementalSink { + db, + caches: &caches, + red_hash: false, + red_delete: false, + no_recovery: false, + retain, + kept: Vec::new(), + pacer: CommitPacer::new(false), + build: &mut build, + shard_dir, + by_label, + affected_scope: Some(candidates.datasets.clone()), + seen: BTreeSet::new(), + previous: None, + previous_label: String::new(), + dataset_terms: BTreeSet::new(), + dataset_map: BTreeMap::new(), + retainer: sidecar::ResidentRetainer::new(), + replaced: 0, + deleted: 0, + unchanged: 0, + max_dataset_terms: 0, + new_shards: 0, + removed_shards: 0, + }; + + let result = rows::stream_dataset_groups_scoped( + &config.sql_uri, + config.publish_filter, + Some(&candidates.datasets), + &mut sink, + ); + if let Err(error) = result { + if sink.pacer.open { + sink.db.cancel_transaction(); + } + return Err(error); + } + + // Affected datasets the scoped crawl never produced: every one of + // their documents is gone (a withdrawn dataset). Out-of-scope shards + // are never touched (`affected_scope`). + let leftovers = sink.leftover_labels(); + for label in leftovers { + sink.delete_dataset(&label)?; + sink.removed_shards += 1; + } + + sink.pacer.finish(&mut sink.db)?; + let doc_count = sink.db.doc_count(); + let replaced = sink.replaced; + let deleted = sink.deleted; + let unchanged = sink.unchanged; + let max_resident = sidecar::mem::peak_resident_entries(); + let peak_open = sidecar::mem::peak_open_shards(); + let bytes_read = sidecar::mem::bytes_read(); + let shards_written = sink.new_shards; + let shards_deleted = sink.removed_shards; + let sink_dataset_terms = sink.max_dataset_terms; + let commit_chunks = sink.pacer.chunks; + let chunk_rss_peak_kib = sink.pacer.rss_peak_kib; + let max_live = livemem::max_live(); + drop(sink); + + let built = build.total_count(); + // Always finalize: even an idempotent re-examination must advance the + // marker (`marker_to`), which is written in the manifest LAST. + build.finalize()?; + + Ok(Stats { + mode: "delta".to_string(), + built, + replaced, + deleted, + unchanged, + index_doc_count: doc_count, + duplicate_identity_terms: Vec::new(), + max_live_documents: max_live, + shards_written, + shards_deleted, + max_resident_sidecar_entries: max_resident, + max_resident_dataset_terms: max_dataset_terms.max(sink_dataset_terms), + peak_open_shards: peak_open, + shard_bytes_read: bytes_read, + commit_chunks, + chunk_rss_peak_kib, + startup: "delta".to_string(), + candidates: candidate_counts, + affected_datasets: candidates.datasets.len(), + marker_from, + marker_to, + }) +} + /// Where the sidecar lives for an index path. pub fn sidecar_location(index_path: &Path) -> PathBuf { sidecar::sidecar_path(index_path) @@ -1121,4 +1465,40 @@ mod tests { sink.duplicates ); } + + /// A label pre-seeded from the manifest still collides with a + /// case-variant seen in the scoped crawl (B1: the guard must reach + /// out-of-scope shards). + #[test] + fn duplicate_sink_seeded_labels_catch_case_variants() { + let mut sink = DuplicateSink::new(); + sink.seed_labels(["FooPublish".to_string()]); + sink.begin_dataset("foopublish").unwrap(); + assert_eq!(sink.duplicates.len(), 1, "{:?}", sink.duplicates); + // The same label verbatim is not a collision. + let mut sink = DuplicateSink::new(); + sink.seed_labels(["BXDPublish".to_string()]); + sink.begin_dataset("BXDPublish").unwrap(); + assert!(sink.duplicates.is_empty(), "{:?}", sink.duplicates); + } + + /// The DELTA leftover path may only name in-scope datasets; without a + /// scope it is the full INCREMENTAL leftover set. + #[test] + fn leftover_labels_respect_the_affected_scope() { + let labels: BTreeSet<String> = ["A", "B", "C", "D"].into_iter().map(String::from).collect(); + let seen: BTreeSet<String> = ["B"].into_iter().map(String::from).collect(); + + // INCREMENTAL (no scope): every label the crawl did not see. + let mut leftovers = leftover_labels_in_scope(labels.iter(), &seen, None); + leftovers.sort(); + assert_eq!(leftovers, vec!["A", "C", "D"]); + + // DELTA (scope = {A, C}): only in-scope leftovers; D, though absent + // from the crawl, is untouched because it is out of scope. + let scope: BTreeSet<String> = ["A", "C"].into_iter().map(String::from).collect(); + let mut leftovers = leftover_labels_in_scope(labels.iter(), &seen, Some(&scope)); + leftovers.sort(); + assert_eq!(leftovers, vec!["A", "C"]); + } } diff --git a/src/indexer/sidecar.rs b/src/indexer/sidecar.rs index e3f91b7..a47deda 100644 --- a/src/indexer/sidecar.rs +++ b/src/indexer/sidecar.rs @@ -1,13 +1,20 @@ -//! The incremental sidecar, sharded by dataset (format version 2). +//! The incremental sidecar, sharded by dataset (format version 3). //! -//! A small **manifest** at `<index>.sidecar.json` (version 2) holds the +//! A small **manifest** at `<index>.sidecar.json` (version 3) holds the //! run metadata and one entry per dataset shard: //! //! ```text //! { version, mode, doc_count, rif_hash, wiki_hash, +//! versioning_marker, versioning_flavour, //! shards: { "<dataset>": { file, count, content_sha256 } } } //! ``` //! +//! `versioning_marker` is the DB `NOW(6)` coverage point for the delta +//! path (`src/indexer/delta.rs`), and `versioning_flavour` guards a +//! future change of timestamp flavour. Both are `#[serde(default)]` so a +//! marker-less v3 manifest parses as the "adoption" state (versioning +//! not yet observed); the strict version gate still rejects v1/v2. +//! //! Each shard is a JSON map `identity-term -> source-hash` at //! `<index>.sidecar.d/<file>`, where the filename is the dataset label //! sanitized for the filesystem plus a 64-bit digest of the raw label (so @@ -47,13 +54,13 @@ use super::sha256::{Sha256, hex_digest}; /// Sidecar format version. A manifest with any other version is ignored /// and the run falls back to FULL (so a version-1 monolithic file, which /// additionally fails to deserialize, is never mis-parsed). -pub const SIDECAR_VERSION: u32 = 2; +pub const SIDECAR_VERSION: u32 = 3; /// The shard directory suffix: `<index>.sidecar.d`. const SHARD_DIR_SUFFIX: &str = ".sidecar.d"; /// One shard's manifest entry. -#[derive(Serialize, Deserialize, Clone, PartialEq, Eq)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct ShardEntry { /// Filename inside the shard directory. pub file: String, @@ -64,7 +71,7 @@ pub struct ShardEntry { } /// The on-disk manifest. -#[derive(Serialize, Deserialize, Clone, PartialEq, Eq)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct Manifest { pub version: u32, /// How the sidecar was last written (`full` or `incremental`). @@ -75,6 +82,14 @@ pub struct Manifest { pub rif_hash: String, /// Monolithic wiki cache hash at the time of writing. pub wiki_hash: String, + /// DB `NOW(6)` of the run whose results were committed (the delta + /// coverage point, format `YYYY-MM-DD HH:MM:SS.ffffff`). `None` when + /// system versioning was absent at the time of the run. + #[serde(default)] + pub versioning_marker: Option<String>, + /// Timestamp flavour guard (`"timestamp6"`); `None` when no marker. + #[serde(default)] + pub versioning_flavour: Option<String>, /// Dataset label -> shard entry. pub shards: BTreeMap<String, ShardEntry>, } @@ -472,4 +487,68 @@ mod tests { assert!(digest_a.chars().all(|c| c.is_ascii_hexdigit())); assert_ne!(a, b, "distinct labels must not share a shard file"); } + + /// A pre-bump v2 manifest (no `versioning_marker`/`versioning_flavour`) + /// deserializes but is rejected by the strict version gate, so it can + /// never steer a delta. + #[test] + fn load_rejects_v2_manifest() { + let index = temp_index_path("v2-reject"); + let v2 = r#"{ + "version": 2, + "mode": "incremental", + "doc_count": 3, + "rif_hash": "aa", + "wiki_hash": "bb", + "shards": {} + }"#; + std::fs::write(sidecar_path(&index), v2).unwrap(); + assert!(Manifest::load(&index).is_none(), "v2 must route to FULL"); + cleanup(&index); + } + + /// A v3 manifest round-trips with its marker/flavour, and a v3 + /// manifest written before versioning was observed (fields absent) + /// parses as the marker-less adoption state. + #[test] + fn v3_manifest_round_trips_and_adopts_missing_marker() { + let index = temp_index_path("v3-roundtrip"); + let manifest = Manifest { + version: SIDECAR_VERSION, + mode: "delta".to_string(), + doc_count: 1, + rif_hash: "aa".to_string(), + wiki_hash: "bb".to_string(), + versioning_marker: Some("2026-10-13 19:37:47.469634".to_string()), + versioning_flavour: Some("timestamp6".to_string()), + shards: BTreeMap::new(), + }; + manifest.store(&index).unwrap(); + let loaded = Manifest::load(&index).expect("v3 must load"); + assert_eq!(loaded, manifest); + + let no_marker = r#"{ + "version": 3, + "mode": "full", + "doc_count": 1, + "rif_hash": "aa", + "wiki_hash": "bb", + "shards": {} + }"#; + std::fs::write(sidecar_path(&index), no_marker).unwrap(); + let loaded = Manifest::load(&index).expect("marker-less v3 must load"); + assert_eq!(loaded.versioning_marker, None); + assert_eq!(loaded.versioning_flavour, None); + cleanup(&index); + } + + /// A unique scratch index path for a self-contained manifest test. + fn temp_index_path(tag: &str) -> PathBuf { + let pid = std::process::id(); + std::env::temp_dir().join(format!("gn4-sidecar-test-{tag}-{pid}")) + } + + fn cleanup(index: &Path) { + let _ = std::fs::remove_file(sidecar_path(index)); + } } |
