about summary refs log tree commit diff
path: root/scripts
diff options
context:
space:
mode:
authorpi coding agent <pi@localhost>2026-10-15 20:44:47 +0000
committerFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-10-15 15:53:35 -0500
commitcdafc493256ad56fe2e50ba80d93baffa7c2d149 (patch)
treee0c91a732483dbee1cbf59457c458a859075abae /scripts
parent82b9270d344f75302bc06aaaf328b4f432d997c0 (diff)
downloadgn-uploader-cdafc493256ad56fe2e50ba80d93baffa7c2d149.tar.gz
Refactor: extract per-line accumulation helper
Move the per-line processing loop out of `file_stats_singlethread` into a
reusable `_accumulate` helper that returns a `_ChunkStats` value. This
makes the accumulation logic available to both the single-threaded and the
multi-processed implementations and allows a single source of truth for the
processing logic.

The new module-level `_make_line_splitter` and `_make_is_skippable_line`
helpers build the closures from plain configuration values, so callers no
longer need to pass unpicklable lambdas around.

`file_stats_singlethread` is re-expressed in terms of the extracted helpers
and its signature simplified to `(fileconfigs, headers, lines)`.
Diffstat (limited to 'scripts')
-rw-r--r--scripts/genotypes/preprocess_csv_collect_info.py179
1 files changed, 110 insertions, 69 deletions
diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py
index 334f47a..a690367 100644
--- a/scripts/genotypes/preprocess_csv_collect_info.py
+++ b/scripts/genotypes/preprocess_csv_collect_info.py
@@ -4,7 +4,7 @@ import sys
 import json
 import logging
 from pathlib import Path
-from dataclasses import asdict, fields
+from dataclasses import asdict, dataclass, field, fields
 from typing import Union, Iterator, Optional, Callable
 from argparse import Namespace, ArgumentParser
 
@@ -165,6 +165,109 @@ def is_dot_geno_line(line: str) -> bool:
     """
     return line.startswith("@")
 
+def _make_line_splitter(separator: str) -> Callable[[str], tuple[str, ...]]:
+    """Create a function to split a line into its fields."""
+    def _line_splitter(line: str) -> tuple[str, ...]:
+        return tuple(field.strip() for field in line.split(separator))
+    return _line_splitter
+
+
+def _make_is_skippable_line(
+        fileconfigs: dict,
+        headers: tuple[str, ...]
+) -> Callable[[str], bool]:
+    """Create a function to check whether a line is skippable."""
+    is_comment_line = make_comment_line_checker(fileconfigs["comment.char"])
+    line_splitter = _make_line_splitter(fileconfigs["separator"])
+
+    def _is_skippable_line(line: str) -> bool:
+        return (is_blank_line(line)
+                or is_comment_line(line)
+                or is_dot_geno_line(line)
+                or line_splitter(line) == headers)
+
+    return _is_skippable_line
+
+
+@dataclass
+class _ChunkStats:# pylint: disable=[too-many-instance-attributes]
+    """Statistics accumulated from a chunk of lines."""
+    total_lines: int = 0
+    data_lines: int = 0
+    comment_lines: int = 0
+    blank_lines: int = 0
+    dot_geno_lines: int = 0
+    dot_geno_fields: dict = field(default_factory=dict)
+    geno_symbols: set = field(default_factory=set)
+    markers: tuple = field(default_factory=tuple)
+    markers_seen: bool = False
+    errors: tuple = field(default_factory=tuple)
+
+
+def _accumulate(
+        headers: tuple[str, ...],
+        fileconfigs: dict,
+        lines: Iterator[str]
+) -> _ChunkStats:
+    """Accumulate statistics from the provided lines."""
+    is_skippable_line = _make_is_skippable_line(fileconfigs, headers)
+    line_splitter = _make_line_splitter(fileconfigs["separator"])
+    stats = _ChunkStats()
+    for line in (ln.strip() for ln in lines):
+        stats.total_lines += 1
+        if not is_skippable_line(line):
+            stats.data_lines += 1
+            fields_values = line_splitter(line)
+            if fileconfigs["transposed"]:
+                t_line_dets = process_transposed(fields_values, fileconfigs)
+                stats.geno_symbols.update(t_line_dets.geno_symbols)
+                stats.errors = stats.errors + tuple(t_line_dets.errors)
+                if t_line_dets.is_markers_line:
+                    stats.markers = tuple(t_line_dets.line_content[1:])
+                    stats.markers_seen = True
+            else:
+                line_dets = process_untransposed(
+                    fields_values,
+                    fileconfigs,
+                    headers=headers)
+                stats.geno_symbols.update(line_dets.geno_symbols)
+                stats.errors = stats.errors + tuple(line_dets.errors)
+                if bool(line_dets.marker):
+                    stats.markers = stats.markers + (str(line_dets.marker),)
+            # TOD0: do more processing
+            continue # end of line processing
+        if line.startswith("@"): #handle special geno formats
+            stats.dot_geno_lines += 1
+            stats.dot_geno_fields = {
+                **stats.dot_geno_fields,
+                **parse_dot_geno_line(line)}
+        if line.startswith("#"): #handle special geno formats
+            stats.comment_lines += 1
+        if line == "": # Blank line
+            stats.blank_lines += 1
+    return stats
+
+
+def _as_file_stats(
+        stats: _ChunkStats,
+        samples: tuple[str, ...],
+        header_errors: tuple[LineError, ...]
+) -> FileStats:
+    """Convert accumulated statistics into file statistics."""
+    return FileStats(
+        total_lines=stats.total_lines,
+        data_lines=stats.data_lines,
+        comment_lines=stats.comment_lines,
+        blank_lines=stats.blank_lines,
+        dot_geno_lines=stats.dot_geno_lines,
+        dot_geno_fields=(DotGenoFields(**stats.dot_geno_fields)
+                         if bool(stats.dot_geno_fields) else None),
+        geno_symbols=tuple(stats.geno_symbols),
+        samples_list=tuple(samples),
+        markers_list=stats.markers,
+        errors=tuple(header_errors) + tuple(stats.errors))
+
+
 def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats:
     """Compute file statistics from the lines in the file."""
     is_comment_line = make_comment_line_checker(fileconfigs["comment.char"])
@@ -175,19 +278,10 @@ def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats:
                                                   or is_comment_line(line)
                                                   or is_dot_geno_line(line)))
     basic_stats = os.stat(filepath)
-    line_splitter = lambda line: tuple(
-        field.strip() for field in line.split(fileconfigs["separator"]))
-
-    is_skippable_line = lambda line: (is_blank_line(line)
-                                      or is_comment_line(line)
-                                      or is_dot_geno_line(line)
-                                      or line_splitter(line) == headers)
-
     lines = read_text_file(filepath)
-    return (file_stats_multiprocessing(headers, is_skippable_line, line_splitter, lines)
+    return (file_stats_multiprocessing(fileconfigs, headers, lines)
             if (basic_stats.st_size/(1024*1024)) >= FILE_SIZE_THRESHOLD
-            else file_stats_singlethread(
-                    fileconfigs, headers, is_skippable_line, line_splitter, lines))
+            else file_stats_singlethread(fileconfigs, headers, lines))
 
 
 def parse_dot_geno_line(line) -> dict[str, str]:
@@ -203,67 +297,14 @@ def parse_dot_geno_line(line) -> dict[str, str]:
 def file_stats_singlethread(
         fileconfigs: dict,
         headers: tuple[str, ...],
-        is_skippable_line: Callable[[str], bool],
-        line_splitter: Callable[[str], tuple[str, ...]],
         lines: Iterator[str]
 ) -> FileStats:
     """File stats from the lines in the file using a single-threaded process."""
     logger.debug("THE FILE HEADERS: %s", headers)
-    col_ids = identify_columns(headers, fileconfigs)
-    _errors = validate_headers(headers, fileconfigs)
-    _markers: tuple[str, ...]  = tuple()
-    _total_lines: int = 0
-    _data_lines: int  = 0
-    _comment_lines: int = 0
-    _blank_lines: int = 0
-    _dot_geno_lines: int = 0
-    _dot_geno_fields: dict = {}
-    genosymbols: set[str] = set()
-    # NOTES: Maybe do the .geno file custom fields check in a separate script?
-
-    for line in (ln.strip() for ln in lines):
-        _total_lines += 1
-        if not is_skippable_line(line):
-            _data_lines += 1
-            _fields_values = line_splitter(line)
-            if fileconfigs["transposed"]:
-                t_line_dets = process_transposed(_fields_values, fileconfigs)
-                headers = headers + (t_line_dets.line_header,)
-                genosymbols.update(t_line_dets.geno_symbols)
-                _errors = _errors + tuple(t_line_dets.errors)
-                if t_line_dets.is_markers_line:
-                    _markers = tuple(t_line_dets.line_content[1:])
-            else:
-                line_dets = process_untransposed(
-                    _fields_values,
-                    fileconfigs,
-                    headers=headers)
-                genosymbols.update(line_dets.geno_symbols)
-                _errors = _errors + tuple(line_dets.errors)
-                if bool(line_dets.marker):
-                    _markers = _markers + (str(line_dets.marker),)
-            # TOD0: do more processing
-            continue # end of line processing
-        if line.startswith("@"): #handle special geno formats
-            _dot_geno_lines += 1
-            _dot_geno_fields = {**_dot_geno_fields, **parse_dot_geno_line(line)}
-        if line.startswith("#"): #handle special geno formats
-            _comment_lines += 1
-        if line == "": # Blank line
-            _blank_lines += 1
-
-    return FileStats(
-        total_lines=_total_lines,
-        data_lines=_data_lines,
-        comment_lines=_comment_lines,
-        blank_lines=_blank_lines,
-        dot_geno_lines=_dot_geno_lines,
-        dot_geno_fields=(DotGenoFields(**_dot_geno_fields)
-                         if bool(_dot_geno_fields) else None),
-        geno_symbols=tuple(genosymbols),
-        samples_list=col_ids.samples,
-        markers_list=_markers,
-        errors=_errors)
+    return _as_file_stats(
+        _accumulate(headers, fileconfigs, lines),
+        identify_columns(headers, fileconfigs).samples,
+        validate_headers(headers, fileconfigs))
 
 
 if __name__ == "__main__":