diff options
| author | pi coding agent <pi@localhost> | 2026-10-15 20:44:47 +0000 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-15 15:53:35 -0500 |
| commit | cdafc493256ad56fe2e50ba80d93baffa7c2d149 (patch) | |
| tree | e0c91a732483dbee1cbf59457c458a859075abae /scripts | |
| parent | 82b9270d344f75302bc06aaaf328b4f432d997c0 (diff) | |
| download | gn-uploader-cdafc493256ad56fe2e50ba80d93baffa7c2d149.tar.gz | |
Refactor: extract per-line accumulation helper
Move the per-line processing loop out of `file_stats_singlethread` into a reusable `_accumulate` helper that returns a `_ChunkStats` value. This makes the accumulation logic available to both the single-threaded and the multi-processed implementations and allows a single source of truth for the processing logic. The new module-level `_make_line_splitter` and `_make_is_skippable_line` helpers build the closures from plain configuration values, so callers no longer need to pass unpicklable lambdas around. `file_stats_singlethread` is re-expressed in terms of the extracted helpers and its signature simplified to `(fileconfigs, headers, lines)`.
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/genotypes/preprocess_csv_collect_info.py | 179 |
1 files changed, 110 insertions, 69 deletions
diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py index 334f47a..a690367 100644 --- a/scripts/genotypes/preprocess_csv_collect_info.py +++ b/scripts/genotypes/preprocess_csv_collect_info.py @@ -4,7 +4,7 @@ import sys import json import logging from pathlib import Path -from dataclasses import asdict, fields +from dataclasses import asdict, dataclass, field, fields from typing import Union, Iterator, Optional, Callable from argparse import Namespace, ArgumentParser @@ -165,6 +165,109 @@ def is_dot_geno_line(line: str) -> bool: """ return line.startswith("@") +def _make_line_splitter(separator: str) -> Callable[[str], tuple[str, ...]]: + """Create a function to split a line into its fields.""" + def _line_splitter(line: str) -> tuple[str, ...]: + return tuple(field.strip() for field in line.split(separator)) + return _line_splitter + + +def _make_is_skippable_line( + fileconfigs: dict, + headers: tuple[str, ...] +) -> Callable[[str], bool]: + """Create a function to check whether a line is skippable.""" + is_comment_line = make_comment_line_checker(fileconfigs["comment.char"]) + line_splitter = _make_line_splitter(fileconfigs["separator"]) + + def _is_skippable_line(line: str) -> bool: + return (is_blank_line(line) + or is_comment_line(line) + or is_dot_geno_line(line) + or line_splitter(line) == headers) + + return _is_skippable_line + + +@dataclass +class _ChunkStats:# pylint: disable=[too-many-instance-attributes] + """Statistics accumulated from a chunk of lines.""" + total_lines: int = 0 + data_lines: int = 0 + comment_lines: int = 0 + blank_lines: int = 0 + dot_geno_lines: int = 0 + dot_geno_fields: dict = field(default_factory=dict) + geno_symbols: set = field(default_factory=set) + markers: tuple = field(default_factory=tuple) + markers_seen: bool = False + errors: tuple = field(default_factory=tuple) + + +def _accumulate( + headers: tuple[str, ...], + fileconfigs: dict, + lines: Iterator[str] +) -> _ChunkStats: + """Accumulate statistics from the provided lines.""" + is_skippable_line = _make_is_skippable_line(fileconfigs, headers) + line_splitter = _make_line_splitter(fileconfigs["separator"]) + stats = _ChunkStats() + for line in (ln.strip() for ln in lines): + stats.total_lines += 1 + if not is_skippable_line(line): + stats.data_lines += 1 + fields_values = line_splitter(line) + if fileconfigs["transposed"]: + t_line_dets = process_transposed(fields_values, fileconfigs) + stats.geno_symbols.update(t_line_dets.geno_symbols) + stats.errors = stats.errors + tuple(t_line_dets.errors) + if t_line_dets.is_markers_line: + stats.markers = tuple(t_line_dets.line_content[1:]) + stats.markers_seen = True + else: + line_dets = process_untransposed( + fields_values, + fileconfigs, + headers=headers) + stats.geno_symbols.update(line_dets.geno_symbols) + stats.errors = stats.errors + tuple(line_dets.errors) + if bool(line_dets.marker): + stats.markers = stats.markers + (str(line_dets.marker),) + # TOD0: do more processing + continue # end of line processing + if line.startswith("@"): #handle special geno formats + stats.dot_geno_lines += 1 + stats.dot_geno_fields = { + **stats.dot_geno_fields, + **parse_dot_geno_line(line)} + if line.startswith("#"): #handle special geno formats + stats.comment_lines += 1 + if line == "": # Blank line + stats.blank_lines += 1 + return stats + + +def _as_file_stats( + stats: _ChunkStats, + samples: tuple[str, ...], + header_errors: tuple[LineError, ...] +) -> FileStats: + """Convert accumulated statistics into file statistics.""" + return FileStats( + total_lines=stats.total_lines, + data_lines=stats.data_lines, + comment_lines=stats.comment_lines, + blank_lines=stats.blank_lines, + dot_geno_lines=stats.dot_geno_lines, + dot_geno_fields=(DotGenoFields(**stats.dot_geno_fields) + if bool(stats.dot_geno_fields) else None), + geno_symbols=tuple(stats.geno_symbols), + samples_list=tuple(samples), + markers_list=stats.markers, + errors=tuple(header_errors) + tuple(stats.errors)) + + def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats: """Compute file statistics from the lines in the file.""" is_comment_line = make_comment_line_checker(fileconfigs["comment.char"]) @@ -175,19 +278,10 @@ def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats: or is_comment_line(line) or is_dot_geno_line(line))) basic_stats = os.stat(filepath) - line_splitter = lambda line: tuple( - field.strip() for field in line.split(fileconfigs["separator"])) - - is_skippable_line = lambda line: (is_blank_line(line) - or is_comment_line(line) - or is_dot_geno_line(line) - or line_splitter(line) == headers) - lines = read_text_file(filepath) - return (file_stats_multiprocessing(headers, is_skippable_line, line_splitter, lines) + return (file_stats_multiprocessing(fileconfigs, headers, lines) if (basic_stats.st_size/(1024*1024)) >= FILE_SIZE_THRESHOLD - else file_stats_singlethread( - fileconfigs, headers, is_skippable_line, line_splitter, lines)) + else file_stats_singlethread(fileconfigs, headers, lines)) def parse_dot_geno_line(line) -> dict[str, str]: @@ -203,67 +297,14 @@ def parse_dot_geno_line(line) -> dict[str, str]: def file_stats_singlethread( fileconfigs: dict, headers: tuple[str, ...], - is_skippable_line: Callable[[str], bool], - line_splitter: Callable[[str], tuple[str, ...]], lines: Iterator[str] ) -> FileStats: """File stats from the lines in the file using a single-threaded process.""" logger.debug("THE FILE HEADERS: %s", headers) - col_ids = identify_columns(headers, fileconfigs) - _errors = validate_headers(headers, fileconfigs) - _markers: tuple[str, ...] = tuple() - _total_lines: int = 0 - _data_lines: int = 0 - _comment_lines: int = 0 - _blank_lines: int = 0 - _dot_geno_lines: int = 0 - _dot_geno_fields: dict = {} - genosymbols: set[str] = set() - # NOTES: Maybe do the .geno file custom fields check in a separate script? - - for line in (ln.strip() for ln in lines): - _total_lines += 1 - if not is_skippable_line(line): - _data_lines += 1 - _fields_values = line_splitter(line) - if fileconfigs["transposed"]: - t_line_dets = process_transposed(_fields_values, fileconfigs) - headers = headers + (t_line_dets.line_header,) - genosymbols.update(t_line_dets.geno_symbols) - _errors = _errors + tuple(t_line_dets.errors) - if t_line_dets.is_markers_line: - _markers = tuple(t_line_dets.line_content[1:]) - else: - line_dets = process_untransposed( - _fields_values, - fileconfigs, - headers=headers) - genosymbols.update(line_dets.geno_symbols) - _errors = _errors + tuple(line_dets.errors) - if bool(line_dets.marker): - _markers = _markers + (str(line_dets.marker),) - # TOD0: do more processing - continue # end of line processing - if line.startswith("@"): #handle special geno formats - _dot_geno_lines += 1 - _dot_geno_fields = {**_dot_geno_fields, **parse_dot_geno_line(line)} - if line.startswith("#"): #handle special geno formats - _comment_lines += 1 - if line == "": # Blank line - _blank_lines += 1 - - return FileStats( - total_lines=_total_lines, - data_lines=_data_lines, - comment_lines=_comment_lines, - blank_lines=_blank_lines, - dot_geno_lines=_dot_geno_lines, - dot_geno_fields=(DotGenoFields(**_dot_geno_fields) - if bool(_dot_geno_fields) else None), - geno_symbols=tuple(genosymbols), - samples_list=col_ids.samples, - markers_list=_markers, - errors=_errors) + return _as_file_stats( + _accumulate(headers, fileconfigs, lines), + identify_columns(headers, fileconfigs).samples, + validate_headers(headers, fileconfigs)) if __name__ == "__main__": |
