diff options
| author | pi coding agent <pi@localhost> | 2026-10-15 20:44:47 +0000 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-15 15:58:42 -0500 |
| commit | e67ce3f2a7e707e38806475b5886d84b77391080 (patch) | |
| tree | 181d312dd9e4ba11602cfeaad7f4ecbe7fc6a6ef /scripts | |
| parent | 52d72868086c2f0860eaf1417599e3d740f2407b (diff) | |
| download | gn-uploader-e67ce3f2a7e707e38806475b5886d84b77391080.tar.gz | |
Add chunk statistics merge helpers
Add `_merge_chunk_stats` to combine the statistics from two consecutive chunks of a file while preserving file order, and `_merge_all_chunk_stats` to fold an iterable of chunk statistics into a single value. For transposed files, the markers of the last markers line win, falling back to the earlier chunk when a later chunk saw no markers line. For untransposed files, the markers are concatenated in order. Counters are summed, genotype symbols are unioned, dot-geno fields are combined (later values override earlier ones) and errors are concatenated in order.
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/genotypes/preprocess_csv_collect_info.py | 43 |
1 files changed, 42 insertions, 1 deletions
diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py index a690367..79fc1fa 100644 --- a/scripts/genotypes/preprocess_csv_collect_info.py +++ b/scripts/genotypes/preprocess_csv_collect_info.py @@ -5,7 +5,7 @@ import json import logging from pathlib import Path from dataclasses import asdict, dataclass, field, fields -from typing import Union, Iterator, Optional, Callable +from typing import Union, Iterable, Iterator, Optional, Callable from argparse import Namespace, ArgumentParser from gn_libs.cli import SysExit @@ -248,6 +248,47 @@ def _accumulate( return stats +def _merge_chunk_stats( + left: _ChunkStats, + right: _ChunkStats, + transposed: bool +) -> _ChunkStats: + """Merge two sets of chunk statistics, preserving file order. + + The two sets of statistics are expected to have been accumulated from + consecutive chunks of the same file, with ``left`` covering the earlier + chunk. + """ + if transposed: + # The markers come from the last markers line encountered. Keep the + # earlier markers when the later chunk did not see a markers line. + markers = right.markers if right.markers_seen else left.markers + else: + markers = left.markers + right.markers + return _ChunkStats( + total_lines=left.total_lines + right.total_lines, + data_lines=left.data_lines + right.data_lines, + comment_lines=left.comment_lines + right.comment_lines, + blank_lines=left.blank_lines + right.blank_lines, + dot_geno_lines=left.dot_geno_lines + right.dot_geno_lines, + dot_geno_fields={**left.dot_geno_fields, **right.dot_geno_fields}, + geno_symbols=left.geno_symbols | right.geno_symbols, + markers=markers, + markers_seen=left.markers_seen or right.markers_seen, + errors=left.errors + right.errors) + + +def _merge_all_chunk_stats( + chunks: Iterable[_ChunkStats], + transposed: bool +) -> _ChunkStats: + """Merge a sequence of chunk statistics into a single set of statistics.""" + merged = _ChunkStats() + for chunk_stats in chunks: + merged = _merge_chunk_stats(merged, chunk_stats, transposed) + return merged + + def _as_file_stats( stats: _ChunkStats, samples: tuple[str, ...], |
