about summary refs log tree commit diff
path: root/scripts
diff options
context:
space:
mode:
authorpi coding agent <pi@localhost>2026-10-15 20:44:47 +0000
committerFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-10-15 15:58:42 -0500
commite67ce3f2a7e707e38806475b5886d84b77391080 (patch)
tree181d312dd9e4ba11602cfeaad7f4ecbe7fc6a6ef /scripts
parent52d72868086c2f0860eaf1417599e3d740f2407b (diff)
downloadgn-uploader-e67ce3f2a7e707e38806475b5886d84b77391080.tar.gz
Add chunk statistics merge helpers
Add `_merge_chunk_stats` to combine the statistics from two consecutive
chunks of a file while preserving file order, and `_merge_all_chunk_stats`
to fold an iterable of chunk statistics into a single value.

For transposed files, the markers of the last markers line win, falling back
to the earlier chunk when a later chunk saw no markers line. For
untransposed files, the markers are concatenated in order. Counters are
summed, genotype symbols are unioned, dot-geno fields are combined (later
values override earlier ones) and errors are concatenated in order.
Diffstat (limited to 'scripts')
-rw-r--r--scripts/genotypes/preprocess_csv_collect_info.py43
1 files changed, 42 insertions, 1 deletions
diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py
index a690367..79fc1fa 100644
--- a/scripts/genotypes/preprocess_csv_collect_info.py
+++ b/scripts/genotypes/preprocess_csv_collect_info.py
@@ -5,7 +5,7 @@ import json
 import logging
 from pathlib import Path
 from dataclasses import asdict, dataclass, field, fields
-from typing import Union, Iterator, Optional, Callable
+from typing import Union, Iterable, Iterator, Optional, Callable
 from argparse import Namespace, ArgumentParser
 
 from gn_libs.cli import SysExit
@@ -248,6 +248,47 @@ def _accumulate(
     return stats
 
 
+def _merge_chunk_stats(
+        left: _ChunkStats,
+        right: _ChunkStats,
+        transposed: bool
+) -> _ChunkStats:
+    """Merge two sets of chunk statistics, preserving file order.
+
+    The two sets of statistics are expected to have been accumulated from
+    consecutive chunks of the same file, with ``left`` covering the earlier
+    chunk.
+    """
+    if transposed:
+        # The markers come from the last markers line encountered. Keep the
+        # earlier markers when the later chunk did not see a markers line.
+        markers = right.markers if right.markers_seen else left.markers
+    else:
+        markers = left.markers + right.markers
+    return _ChunkStats(
+        total_lines=left.total_lines + right.total_lines,
+        data_lines=left.data_lines + right.data_lines,
+        comment_lines=left.comment_lines + right.comment_lines,
+        blank_lines=left.blank_lines + right.blank_lines,
+        dot_geno_lines=left.dot_geno_lines + right.dot_geno_lines,
+        dot_geno_fields={**left.dot_geno_fields, **right.dot_geno_fields},
+        geno_symbols=left.geno_symbols | right.geno_symbols,
+        markers=markers,
+        markers_seen=left.markers_seen or right.markers_seen,
+        errors=left.errors + right.errors)
+
+
+def _merge_all_chunk_stats(
+        chunks: Iterable[_ChunkStats],
+        transposed: bool
+) -> _ChunkStats:
+    """Merge a sequence of chunk statistics into a single set of statistics."""
+    merged = _ChunkStats()
+    for chunk_stats in chunks:
+        merged = _merge_chunk_stats(merged, chunk_stats, transposed)
+    return merged
+
+
 def _as_file_stats(
         stats: _ChunkStats,
         samples: tuple[str, ...],