about summary refs log tree commit diff
path: root/scripts
diff options
context:
space:
mode:
authorFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-10-02 11:58:40 -0500
committerFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-10-02 11:58:40 -0500
commit31437e9b7bed0fe2c2c499d91bc6308014b8421d (patch)
tree3a2918f86ea6da71892e6132e6112e40b147cc6a /scripts
parent9652c118028ea30c423c3c08deb031c5fe2df76e (diff)
downloadgn-uploader-31437e9b7bed0fe2c2c499d91bc6308014b8421d.tar.gz
Refactor: Simplify code by reusing common functionality.
Diffstat (limited to 'scripts')
-rw-r--r--scripts/genotypes/preprocess_csv_collect_info.py87
-rw-r--r--scripts/genotypes/types.py3
2 files changed, 40 insertions, 50 deletions
diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py
index 026936d..32ef1b2 100644
--- a/scripts/genotypes/preprocess_csv_collect_info.py
+++ b/scripts/genotypes/preprocess_csv_collect_info.py
@@ -8,6 +8,11 @@ from argparse import Namespace, ArgumentParser
 
 from gn_libs.cli import SysExit
 
+from uploader.csvprocessing import (is_blank_line,
+                                    read_csv_file,
+                                    read_csv_file_headers,
+                                    make_comment_line_checker)
+
 from scripts.cli.options import add_logging
 from scripts.cli.logging import setup_logging
 from scripts.cli.validators import file_exists
@@ -15,7 +20,6 @@ from scripts.cli.validators import file_exists
 from .types import (FileStats,
                     LineError,
                     LineDetails,
-                    DotGenoFields,
                     FieldsIdentity)
 
 
@@ -128,8 +132,31 @@ def validate_headers(
     return _errors
 
 
-def file_statistics(lines: Iterator[str], fileconfigs: dict) -> FileStats:
+def is_dot_geno_line(line: str) -> bool:
+    """Check whether the line is a custom .geno format line
+
+    :param line: The line to check for custom formatting
+    :type line: str
+    :rtype bool
+    """
+    return line.startswith("@")
+
+
+def file_statistics(filepath: Path, fileconfigs: dict) -> FileStats:
     """Compute file statistics from the lines in the file."""
+
+    is_comment_line = make_comment_line_checker(fileconfigs["comment.char"])
+    def _is_skippable_line_(line: str) -> bool:
+        return (is_blank_line(line)
+                or is_comment_line(line)
+                or is_dot_geno_line(line))
+    headers = read_csv_file_headers(filepath,
+                                    fileconfigs["transposed"],
+                                    fileconfigs["separator"],
+                                    _is_skippable_line_)
+    logger.debug("THE FILE HEADERS: %s", headers)
+    col_ids = identify_columns(headers, fileconfigs)
+    _errors = validate_headers(headers, fileconfigs)
     stats: dict = {
         "data_lines": 0,
         "total_lines": 0,
@@ -137,55 +164,23 @@ def file_statistics(lines: Iterator[str], fileconfigs: dict) -> FileStats:
         "comment_lines": 0,
         "dot_geno_lines": 0
     }
-    dotgeno = {}
-    headers: tuple[str, ...] = tuple()
     genosymbols: set[str] = set()
-    _errors: tuple[LineError, ...] = tuple()
-    for line in lines:
-        stats["total_lines"] = stats["total_lines"] + 1
-        _line = line.strip()
-        if _line == "":
-            stats["blank_lines"] = stats["blank_lines"] + 1
-            continue
+    # NOTES: Maybe do the .geno file custom fields check in a separate script?
 
-        if _line.startswith(fileconfigs["comment.char"]):
-            stats["comment_lines"] = stats["comment_lines"] + 1
-            continue
-
-        if _line.startswith("@"): # custom GeneNetwork Settings
-            stats["dot_geno_lines"] = stats["dot_geno_lines"] + 1
-            _fld = tuple(item.strip() for item in line.strip("@").split(":"))
-            dotgeno[_fld[0]] = _fld[1]
-            continue
-
-        stats["data_lines"] = stats["data_lines"] + 1
-        fields: tuple[str, ...] = tuple(
-            fld.strip() for fld in line.split(fileconfigs["separator"]))
-
-        # Basic QC:
-        # * Header row (normal) or column (transposed):
-        #   - **MUST** have values: no missing values
-        #   - **MUST** be strings
-        # * Mandatory Fields -- Fields that MUST have values:
-        #   - Marker/Locus
-        #   - Chromosome
-        #   - cM: Genetic distance
-        #   - Mb: Physical distance
-        if stats["data_lines"] == 1: # This is the first data line
-            if not fileconfigs["transposed"]: # File **IS NOT** transposed.
-                headers = fields # First line ***IS** headers' line!
-                continue
-            # ELSE: File **IS** transposed.
-            headers = headers + (fields[0],)
-            continue
+    lines = read_csv_file(
+        filepath, fileconfigs["separator"], _is_skippable_line_)
+    for line in lines:
+        stats["data_lines"] += 1
+        if line == headers:
+            continue# skip headers line
 
         line_dets = (
         process_transposed(
-            fields,
+            line,
             na_strings=fileconfigs["na_strings"])
         if fileconfigs["transposed"]
         else process_untransposed(
-                fields,
+                line,
                 fileconfigs,
                 headers=headers))
         _errors = _errors + tuple(line_dets.errors)
@@ -197,13 +192,9 @@ def file_statistics(lines: Iterator[str], fileconfigs: dict) -> FileStats:
         # TODO: do more processing
         continue # end of line processing
 
-    col_ids = identify_columns(headers, fileconfigs)
-    _errors = validate_headers(headers, fileconfigs) + _errors
-
     return FileStats(
         **stats,
         geno_symbols=tuple(genosymbols),
-        dot_geno_fields=(DotGenoFields(**dotgeno) if bool(dotgeno) else None),
         samples_list=col_ids.samples,
         errors=_errors)
 
@@ -303,7 +294,7 @@ if __name__ == "__main__":
             logger.debug("CLI Arguments: %s", args)
 
             stats = file_statistics(
-                filecontents(args.csv_file),
+                args.csv_file,
                 {
                     "markers": args.markers_field,
                     "chromosome": args.chr_field,
diff --git a/scripts/genotypes/types.py b/scripts/genotypes/types.py
index 18af7b8..472bb1b 100644
--- a/scripts/genotypes/types.py
+++ b/scripts/genotypes/types.py
@@ -4,7 +4,7 @@ from typing import Union, Sequence, Optional
 
 
 @dataclass
-class DotGenoFields:
+class DotGenoFields:# Still necessary??
     """Custom .geno file format values"""
     name: str
     mat: str
@@ -52,7 +52,6 @@ class FileStats:# pylint: disable=[too-many-instance-attributes]
     blank_lines: int = 0
     comment_lines: int = 0
     dot_geno_lines: int = 0
-    dot_geno_fields: Optional[DotGenoFields] = None
     samples_list: Sequence[str] = field(default_factory=tuple)
     markers_list: Sequence[str] = field(default_factory=tuple)
     errors: Sequence[LineError] = field(default_factory=tuple)