about summary refs log tree commit diff
path: root/src/bin
diff options
context:
space:
mode:
authorAlexander_Kabui <alexanderkabui@proton.me>2026-10-13 15:11:33 +0300
committerAlexander_Kabui <alexanderkabui@proton.me>2026-10-14 07:27:03 +0300
commit53c3f26ec2fdf209876a8a8eab561acc6fd755d4 (patch)
tree25fc2be0d09e37da759be90fc3590f1496afaa89 /src/bin
parenta6003ffbfe40aadaa1d9548bdc3294987b97d304 (diff)
downloadgenenetwork4-53c3f26ec2fdf209876a8a8eab561acc6fd755d4.tar.gz
* src/indexer/sidecar.rs (sharding): one sidecar shard per dataset.
A monolithic sidecar rewrote and re-read the whole identity map each run;
sharding it by dataset under <index>.sidecar.json plus <index>.sidecar.d/
lets a run load and rewrite only the shards it touches and keeps resident
memory at O(largest dataset).
* src/indexer/run.rs: crawl the SQL sources one dataset at a time through a
sink and commit every 5000 documents instead of buffering the whole crawl;
a bounded-channel variant was tried and reverted (several times slower).
* src/indexer/rows.rs: order and group the crawl streams by dataset.
* src/bin/gn-indexer.rs: follow the sharded sink wiring.
Diffstat (limited to 'src/bin')
-rw-r--r--src/bin/gn-indexer.rs15
1 files changed, 15 insertions, 0 deletions
diff --git a/src/bin/gn-indexer.rs b/src/bin/gn-indexer.rs
index d81990e..224edb8 100644
--- a/src/bin/gn-indexer.rs
+++ b/src/bin/gn-indexer.rs
@@ -82,6 +82,21 @@ fn main() -> ExitCode {
             println!("unchanged={}", stats.unchanged);
             println!("index_doc_count={}", stats.index_doc_count);
             println!("max_live_documents={}", stats.max_live_documents);
+            println!("shards_written={}", stats.shards_written);
+            println!("shards_deleted={}", stats.shards_deleted);
+            println!(
+                "max_resident_sidecar_entries={}",
+                stats.max_resident_sidecar_entries
+            );
+            println!(
+                "max_resident_dataset_terms={}",
+                stats.max_resident_dataset_terms
+            );
+            println!("peak_open_shards={}", stats.peak_open_shards);
+            println!("shard_bytes_read={}", stats.shard_bytes_read);
+            println!("commit_chunks={}", stats.commit_chunks);
+            println!("chunk_rss_peak_kib={}", stats.chunk_rss_peak_kib);
+            println!("startup={}", stats.startup);
             ExitCode::SUCCESS
         }
         Err(error) => {