From 53c3f26ec2fdf209876a8a8eab561acc6fd755d4 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Tue, 13 Oct 2026 15:11:33 +0300 Subject: * src/indexer/sidecar.rs (sharding): one sidecar shard per dataset. A monolithic sidecar rewrote and re-read the whole identity map each run; sharding it by dataset under .sidecar.json plus .sidecar.d/ lets a run load and rewrite only the shards it touches and keeps resident memory at O(largest dataset). * src/indexer/run.rs: crawl the SQL sources one dataset at a time through a sink and commit every 5000 documents instead of buffering the whole crawl; a bounded-channel variant was tried and reverted (several times slower). * src/indexer/rows.rs: order and group the crawl streams by dataset. * src/bin/gn-indexer.rs: follow the sharded sink wiring. --- src/bin/gn-indexer.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'src/bin') diff --git a/src/bin/gn-indexer.rs b/src/bin/gn-indexer.rs index d81990e..224edb8 100644 --- a/src/bin/gn-indexer.rs +++ b/src/bin/gn-indexer.rs @@ -82,6 +82,21 @@ fn main() -> ExitCode { println!("unchanged={}", stats.unchanged); println!("index_doc_count={}", stats.index_doc_count); println!("max_live_documents={}", stats.max_live_documents); + println!("shards_written={}", stats.shards_written); + println!("shards_deleted={}", stats.shards_deleted); + println!( + "max_resident_sidecar_entries={}", + stats.max_resident_sidecar_entries + ); + println!( + "max_resident_dataset_terms={}", + stats.max_resident_dataset_terms + ); + println!("peak_open_shards={}", stats.peak_open_shards); + println!("shard_bytes_read={}", stats.shard_bytes_read); + println!("commit_chunks={}", stats.commit_chunks); + println!("chunk_rss_peak_kib={}", stats.chunk_rss_peak_kib); + println!("startup={}", stats.startup); ExitCode::SUCCESS } Err(error) => { -- cgit 1.4.1