diff options
| author | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-02 11:58:40 -0500 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-02 11:58:40 -0500 |
| commit | 31437e9b7bed0fe2c2c499d91bc6308014b8421d (patch) | |
| tree | 3a2918f86ea6da71892e6132e6112e40b147cc6a /scripts/genotypes | |
| parent | 9652c118028ea30c423c3c08deb031c5fe2df76e (diff) | |
| download | gn-uploader-31437e9b7bed0fe2c2c499d91bc6308014b8421d.tar.gz | |
Refactor: Simplify code by reusing common functionality.
Diffstat (limited to 'scripts/genotypes')
| -rw-r--r-- | scripts/genotypes/preprocess_csv_collect_info.py | 87 | ||||
| -rw-r--r-- | scripts/genotypes/types.py | 3 |
2 files changed, 40 insertions, 50 deletions
diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py index 026936d..32ef1b2 100644 --- a/scripts/genotypes/preprocess_csv_collect_info.py +++ b/scripts/genotypes/preprocess_csv_collect_info.py @@ -8,6 +8,11 @@ from argparse import Namespace, ArgumentParser from gn_libs.cli import SysExit +from uploader.csvprocessing import (is_blank_line, + read_csv_file, + read_csv_file_headers, + make_comment_line_checker) + from scripts.cli.options import add_logging from scripts.cli.logging import setup_logging from scripts.cli.validators import file_exists @@ -15,7 +20,6 @@ from scripts.cli.validators import file_exists from .types import (FileStats, LineError, LineDetails, - DotGenoFields, FieldsIdentity) @@ -128,8 +132,31 @@ def validate_headers( return _errors -def file_statistics(lines: Iterator[str], fileconfigs: dict) -> FileStats: +def is_dot_geno_line(line: str) -> bool: + """Check whether the line is a custom .geno format line + + :param line: The line to check for custom formatting + :type line: str + :rtype bool + """ + return line.startswith("@") + + +def file_statistics(filepath: Path, fileconfigs: dict) -> FileStats: """Compute file statistics from the lines in the file.""" + + is_comment_line = make_comment_line_checker(fileconfigs["comment.char"]) + def _is_skippable_line_(line: str) -> bool: + return (is_blank_line(line) + or is_comment_line(line) + or is_dot_geno_line(line)) + headers = read_csv_file_headers(filepath, + fileconfigs["transposed"], + fileconfigs["separator"], + _is_skippable_line_) + logger.debug("THE FILE HEADERS: %s", headers) + col_ids = identify_columns(headers, fileconfigs) + _errors = validate_headers(headers, fileconfigs) stats: dict = { "data_lines": 0, "total_lines": 0, @@ -137,55 +164,23 @@ def file_statistics(lines: Iterator[str], fileconfigs: dict) -> FileStats: "comment_lines": 0, "dot_geno_lines": 0 } - dotgeno = {} - headers: tuple[str, ...] = tuple() genosymbols: set[str] = set() - _errors: tuple[LineError, ...] = tuple() - for line in lines: - stats["total_lines"] = stats["total_lines"] + 1 - _line = line.strip() - if _line == "": - stats["blank_lines"] = stats["blank_lines"] + 1 - continue + # NOTES: Maybe do the .geno file custom fields check in a separate script? - if _line.startswith(fileconfigs["comment.char"]): - stats["comment_lines"] = stats["comment_lines"] + 1 - continue - - if _line.startswith("@"): # custom GeneNetwork Settings - stats["dot_geno_lines"] = stats["dot_geno_lines"] + 1 - _fld = tuple(item.strip() for item in line.strip("@").split(":")) - dotgeno[_fld[0]] = _fld[1] - continue - - stats["data_lines"] = stats["data_lines"] + 1 - fields: tuple[str, ...] = tuple( - fld.strip() for fld in line.split(fileconfigs["separator"])) - - # Basic QC: - # * Header row (normal) or column (transposed): - # - **MUST** have values: no missing values - # - **MUST** be strings - # * Mandatory Fields -- Fields that MUST have values: - # - Marker/Locus - # - Chromosome - # - cM: Genetic distance - # - Mb: Physical distance - if stats["data_lines"] == 1: # This is the first data line - if not fileconfigs["transposed"]: # File **IS NOT** transposed. - headers = fields # First line ***IS** headers' line! - continue - # ELSE: File **IS** transposed. - headers = headers + (fields[0],) - continue + lines = read_csv_file( + filepath, fileconfigs["separator"], _is_skippable_line_) + for line in lines: + stats["data_lines"] += 1 + if line == headers: + continue# skip headers line line_dets = ( process_transposed( - fields, + line, na_strings=fileconfigs["na_strings"]) if fileconfigs["transposed"] else process_untransposed( - fields, + line, fileconfigs, headers=headers)) _errors = _errors + tuple(line_dets.errors) @@ -197,13 +192,9 @@ def file_statistics(lines: Iterator[str], fileconfigs: dict) -> FileStats: # TODO: do more processing continue # end of line processing - col_ids = identify_columns(headers, fileconfigs) - _errors = validate_headers(headers, fileconfigs) + _errors - return FileStats( **stats, geno_symbols=tuple(genosymbols), - dot_geno_fields=(DotGenoFields(**dotgeno) if bool(dotgeno) else None), samples_list=col_ids.samples, errors=_errors) @@ -303,7 +294,7 @@ if __name__ == "__main__": logger.debug("CLI Arguments: %s", args) stats = file_statistics( - filecontents(args.csv_file), + args.csv_file, { "markers": args.markers_field, "chromosome": args.chr_field, diff --git a/scripts/genotypes/types.py b/scripts/genotypes/types.py index 18af7b8..472bb1b 100644 --- a/scripts/genotypes/types.py +++ b/scripts/genotypes/types.py @@ -4,7 +4,7 @@ from typing import Union, Sequence, Optional @dataclass -class DotGenoFields: +class DotGenoFields:# Still necessary?? """Custom .geno file format values""" name: str mat: str @@ -52,7 +52,6 @@ class FileStats:# pylint: disable=[too-many-instance-attributes] blank_lines: int = 0 comment_lines: int = 0 dot_geno_lines: int = 0 - dot_geno_fields: Optional[DotGenoFields] = None samples_list: Sequence[str] = field(default_factory=tuple) markers_list: Sequence[str] = field(default_factory=tuple) errors: Sequence[LineError] = field(default_factory=tuple) |
