about summary refs log tree commit diff
path: root/scripts/genotypes
diff options
context:
space:
mode:
authorFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-10-14 16:23:41 -0500
committerFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-10-14 16:23:41 -0500
commitef45d22628bd06c0282bd586206bc3494773f132 (patch)
tree13c191b9662d67dc52d77275762f930921d864da /scripts/genotypes
parentb6d95c2ff301e572f024f525f6b5b81e73dafe5b (diff)
downloadgn-uploader-ef45d22628bd06c0282bd586206bc3494773f132.tar.gz
Refactor design to allow processing file with multiple processes.
If the file is larger (in bytes) than the value set in the
`FILE_SIZE_THRESHOLD` constant, then the file will be chunked and
processed with multiple processes, using the multiprocessing library,
otherwise, it will be processed in a single-thread process.
Diffstat (limited to 'scripts/genotypes')
-rw-r--r--scripts/genotypes/preprocess_csv_collect_info.py118
-rw-r--r--scripts/genotypes/types.py5
2 files changed, 87 insertions, 36 deletions
diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py
index 06e7abd..2f687c4 100644
--- a/scripts/genotypes/preprocess_csv_collect_info.py
+++ b/scripts/genotypes/preprocess_csv_collect_info.py
@@ -1,16 +1,18 @@
 """Preprocess the genotypes CSV files: Collect basic info."""
+import os
 import sys
 import json
 import logging
 from pathlib import Path
-from dataclasses import asdict
-from typing import Union, Iterator, Optional
+from dataclasses import asdict, fields
+from typing import Union, Iterator, Optional, Callable
 from argparse import Namespace, ArgumentParser
 
 from gn_libs.cli import SysExit
 
 from uploader.csvprocessing import (is_blank_line,
                                     read_csv_file,
+                                    read_text_file,
                                     read_csv_file_headers,
                                     make_comment_line_checker)
 
@@ -20,6 +22,7 @@ from scripts.cli.validators import file_exists
 
 from .types import (FileStats,
                     LineError,
+                    DotGenoFields,
                     FieldsIdentity,
                     TransposedLineDetails,
                     UntransposedLineDetails)
@@ -27,6 +30,8 @@ from .types import (FileStats,
 
 logger = logging.getLogger(__name__)
 
+FILE_SIZE_THRESHOLD = 100 * 1024 * 1024 # 100MB
+
 
 def filecontents(path: Path) -> Iterator[str]:
     """Return an iterator to the file contents"""
@@ -160,54 +165,95 @@ def is_dot_geno_line(line: str) -> bool:
     """
     return line.startswith("@")
 
-
-def file_statistics(filepath: Path, fileconfigs: dict) -> FileStats:
+def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats:
     """Compute file statistics from the lines in the file."""
-
     is_comment_line = make_comment_line_checker(fileconfigs["comment.char"])
-    def _is_skippable_line_(line: str) -> bool:
-        return (is_blank_line(line)
-                or is_comment_line(line)
-                or is_dot_geno_line(line))
     headers = read_csv_file_headers(filepath,
                                     fileconfigs["transposed"],
                                     fileconfigs["separator"],
-                                    _is_skippable_line_)
+                                    lambda line: (is_blank_line(line)
+                                                  or is_comment_line(line)
+                                                  or is_dot_geno_line(line)))
+    basic_stats = os.stat(filepath)
+    is_skippable_line = lambda line: (is_blank_line(line)
+                                      or is_comment_line(line)
+                                      or is_dot_geno_line(line)
+                                      or line == headers)
+    line_splitter = lambda line: tuple(
+        field.strip() for field in line.split(fileconfigs["separator"]))
+    lines = read_text_file(filepath)
+    return (file_stats_multiprocessing(headers, is_skippable_line, line_splitter, lines)
+            if (basic_stats.st_size/(1024*1024)) >= FILE_SIZE_THRESHOLD
+            else file_stats_singlethread(
+                    fileconfigs, headers, is_skippable_line, line_splitter, lines))
+
+
+def parse_dot_geno_line(line) -> dict[str, str]:
+    """Parse the dot_geno format line."""
+    assert line.startswith("@")
+    acceptable_fields = tuple(field.name for field in fields(DotGenoFields))
+    parts = tuple(item.strip() for item in line[1:].split(":"))
+    if len(parts) == 2 and parts[0] in acceptable_fields:
+        return {parts[0]: parts[1]}
+    return {}
+
+
+def file_stats_singlethread(
+        fileconfigs: dict,
+        headers: tuple[str, ...],
+        is_skippable_line: Callable[[str], bool],
+        line_splitter: Callable[[str], tuple[str, ...]],
+        lines: Iterator[str]
+) -> FileStats:
+    """File stats from the lines in the file using a single-threaded process."""
     logger.debug("THE FILE HEADERS: %s", headers)
     col_ids = identify_columns(headers, fileconfigs)
     _errors = validate_headers(headers, fileconfigs)
     _markers: tuple[str, ...]  = tuple()
-    _data_lines = 0
+    _total_lines: int = 0
+    _data_lines: int  = 0
+    _comment_lines: int = 0
+    _dot_geno_lines: int = 0
+    _dot_geno_fields: dict = {}
     genosymbols: set[str] = set()
     # NOTES: Maybe do the .geno file custom fields check in a separate script?
 
-    lines = read_csv_file(
-        filepath, fileconfigs["separator"], _is_skippable_line_)
     for line in lines:
-        _data_lines += 1
-        if line == headers:
-            continue# skip headers line
-
-        if fileconfigs["transposed"]:
-            t_line_dets = process_transposed(line, fileconfigs)
-            headers = headers + (t_line_dets.line_header,)
-            genosymbols.update(t_line_dets.geno_symbols)
-            _errors = _errors + tuple(t_line_dets.errors)
-            if t_line_dets.is_markers_line:
-                _markers = tuple(t_line_dets.line_content[1:])
-        else:
-            line_dets = process_untransposed(
-                line,
-                fileconfigs,
-                headers=headers)
-            genosymbols.update(line_dets.geno_symbols)
-            _errors = _errors + tuple(line_dets.errors)
-            if bool(line_dets.marker):
-                _markers = _markers + (str(line_dets.marker),)
-        continue # end of line processing
+        _total_lines += 1
+        if not is_skippable_line(line):
+            _data_lines += 1
+            _fields_values = line_splitter(line)
+            if fileconfigs["transposed"]:
+                t_line_dets = process_transposed(_fields_values, fileconfigs)
+                headers = headers + (t_line_dets.line_header,)
+                genosymbols.update(t_line_dets.geno_symbols)
+                _errors = _errors + tuple(t_line_dets.errors)
+                if t_line_dets.is_markers_line:
+                    _markers = tuple(t_line_dets.line_content[1:])
+            else:
+                line_dets = process_untransposed(
+                    _fields_values,
+                    fileconfigs,
+                    headers=headers)
+                genosymbols.update(line_dets.geno_symbols)
+                _errors = _errors + tuple(line_dets.errors)
+                if bool(line_dets.marker):
+                    _markers = _markers + (str(line_dets.marker),)
+            # TOD0: do more processing
+            continue # end of line processing
+        if line.startswith("@"): #handle special geno formats
+            _dot_geno_lines += 1
+            _dot_geno_fields = {**_dot_geno_fields, **parse_dot_geno_line(line)}
+        if line.startswith("#"): #handle special geno formats
+            _comment_lines += 1
 
     return FileStats(
+        total_lines=_total_lines,
         data_lines=_data_lines,
+        comment_lines=_comment_lines,
+        dot_geno_lines=_dot_geno_lines,
+        dot_geno_fields=(DotGenoFields(**_dot_geno_fields)
+                         if bool(_dot_geno_fields) else None),
         geno_symbols=tuple(genosymbols),
         samples_list=col_ids.samples,
         markers_list=_markers,
@@ -308,7 +354,7 @@ if __name__ == "__main__":
             setup_logging(logger, args.log_level.upper(), tuple())
             logger.debug("CLI Arguments: %s", args)
 
-            stats = file_statistics(
+            stats = compute_statistics(
                 args.csv_file,
                 {
                     "markers": args.markers_field,
@@ -323,7 +369,7 @@ if __name__ == "__main__":
             print(json.dumps(asdict(stats)))
             return SysExit.OK
         except FileNotFoundError as _fnf:
-            logger.error(", ".join(_fnf.args))
+            logger.error(", ".join(str(arg) for arg in _fnf.args), exc_info=True)
             return SysExit.FILENOTFOUND
 
     sys.exit(main().value)
diff --git a/scripts/genotypes/types.py b/scripts/genotypes/types.py
index 7f0e6cb..d765fe5 100644
--- a/scripts/genotypes/types.py
+++ b/scripts/genotypes/types.py
@@ -56,9 +56,14 @@ class UntransposedLineDetails:
 @dataclass(frozen=True)
 class FileStats:# pylint: disable=[too-many-instance-attributes]
     """Custom file statistics."""
+    total_lines: int
     data_lines: int
     geno_symbols: Sequence[str]
 
+    comment_lines: int = 0
+    blank_lines: int = 0
+    dot_geno_lines: int = 0
+    dot_geno_fields: Optional[DotGenoFields] = None
     samples_list: Sequence[str] = field(default_factory=tuple)
     markers_list: Sequence[str] = field(default_factory=tuple)
     errors: Sequence[LineError] = field(default_factory=tuple)