diff options
| author | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-14 16:23:41 -0500 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-14 16:23:41 -0500 |
| commit | ef45d22628bd06c0282bd586206bc3494773f132 (patch) | |
| tree | 13c191b9662d67dc52d77275762f930921d864da /scripts | |
| parent | b6d95c2ff301e572f024f525f6b5b81e73dafe5b (diff) | |
| download | gn-uploader-ef45d22628bd06c0282bd586206bc3494773f132.tar.gz | |
Refactor design to allow processing file with multiple processes.
If the file is larger (in bytes) than the value set in the `FILE_SIZE_THRESHOLD` constant, then the file will be chunked and processed with multiple processes, using the multiprocessing library, otherwise, it will be processed in a single-thread process.
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/genotypes/preprocess_csv_collect_info.py | 118 | ||||
| -rw-r--r-- | scripts/genotypes/types.py | 5 |
2 files changed, 87 insertions, 36 deletions
diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py index 06e7abd..2f687c4 100644 --- a/scripts/genotypes/preprocess_csv_collect_info.py +++ b/scripts/genotypes/preprocess_csv_collect_info.py @@ -1,16 +1,18 @@ """Preprocess the genotypes CSV files: Collect basic info.""" +import os import sys import json import logging from pathlib import Path -from dataclasses import asdict -from typing import Union, Iterator, Optional +from dataclasses import asdict, fields +from typing import Union, Iterator, Optional, Callable from argparse import Namespace, ArgumentParser from gn_libs.cli import SysExit from uploader.csvprocessing import (is_blank_line, read_csv_file, + read_text_file, read_csv_file_headers, make_comment_line_checker) @@ -20,6 +22,7 @@ from scripts.cli.validators import file_exists from .types import (FileStats, LineError, + DotGenoFields, FieldsIdentity, TransposedLineDetails, UntransposedLineDetails) @@ -27,6 +30,8 @@ from .types import (FileStats, logger = logging.getLogger(__name__) +FILE_SIZE_THRESHOLD = 100 * 1024 * 1024 # 100MB + def filecontents(path: Path) -> Iterator[str]: """Return an iterator to the file contents""" @@ -160,54 +165,95 @@ def is_dot_geno_line(line: str) -> bool: """ return line.startswith("@") - -def file_statistics(filepath: Path, fileconfigs: dict) -> FileStats: +def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats: """Compute file statistics from the lines in the file.""" - is_comment_line = make_comment_line_checker(fileconfigs["comment.char"]) - def _is_skippable_line_(line: str) -> bool: - return (is_blank_line(line) - or is_comment_line(line) - or is_dot_geno_line(line)) headers = read_csv_file_headers(filepath, fileconfigs["transposed"], fileconfigs["separator"], - _is_skippable_line_) + lambda line: (is_blank_line(line) + or is_comment_line(line) + or is_dot_geno_line(line))) + basic_stats = os.stat(filepath) + is_skippable_line = lambda line: (is_blank_line(line) + or is_comment_line(line) + or is_dot_geno_line(line) + or line == headers) + line_splitter = lambda line: tuple( + field.strip() for field in line.split(fileconfigs["separator"])) + lines = read_text_file(filepath) + return (file_stats_multiprocessing(headers, is_skippable_line, line_splitter, lines) + if (basic_stats.st_size/(1024*1024)) >= FILE_SIZE_THRESHOLD + else file_stats_singlethread( + fileconfigs, headers, is_skippable_line, line_splitter, lines)) + + +def parse_dot_geno_line(line) -> dict[str, str]: + """Parse the dot_geno format line.""" + assert line.startswith("@") + acceptable_fields = tuple(field.name for field in fields(DotGenoFields)) + parts = tuple(item.strip() for item in line[1:].split(":")) + if len(parts) == 2 and parts[0] in acceptable_fields: + return {parts[0]: parts[1]} + return {} + + +def file_stats_singlethread( + fileconfigs: dict, + headers: tuple[str, ...], + is_skippable_line: Callable[[str], bool], + line_splitter: Callable[[str], tuple[str, ...]], + lines: Iterator[str] +) -> FileStats: + """File stats from the lines in the file using a single-threaded process.""" logger.debug("THE FILE HEADERS: %s", headers) col_ids = identify_columns(headers, fileconfigs) _errors = validate_headers(headers, fileconfigs) _markers: tuple[str, ...] = tuple() - _data_lines = 0 + _total_lines: int = 0 + _data_lines: int = 0 + _comment_lines: int = 0 + _dot_geno_lines: int = 0 + _dot_geno_fields: dict = {} genosymbols: set[str] = set() # NOTES: Maybe do the .geno file custom fields check in a separate script? - lines = read_csv_file( - filepath, fileconfigs["separator"], _is_skippable_line_) for line in lines: - _data_lines += 1 - if line == headers: - continue# skip headers line - - if fileconfigs["transposed"]: - t_line_dets = process_transposed(line, fileconfigs) - headers = headers + (t_line_dets.line_header,) - genosymbols.update(t_line_dets.geno_symbols) - _errors = _errors + tuple(t_line_dets.errors) - if t_line_dets.is_markers_line: - _markers = tuple(t_line_dets.line_content[1:]) - else: - line_dets = process_untransposed( - line, - fileconfigs, - headers=headers) - genosymbols.update(line_dets.geno_symbols) - _errors = _errors + tuple(line_dets.errors) - if bool(line_dets.marker): - _markers = _markers + (str(line_dets.marker),) - continue # end of line processing + _total_lines += 1 + if not is_skippable_line(line): + _data_lines += 1 + _fields_values = line_splitter(line) + if fileconfigs["transposed"]: + t_line_dets = process_transposed(_fields_values, fileconfigs) + headers = headers + (t_line_dets.line_header,) + genosymbols.update(t_line_dets.geno_symbols) + _errors = _errors + tuple(t_line_dets.errors) + if t_line_dets.is_markers_line: + _markers = tuple(t_line_dets.line_content[1:]) + else: + line_dets = process_untransposed( + _fields_values, + fileconfigs, + headers=headers) + genosymbols.update(line_dets.geno_symbols) + _errors = _errors + tuple(line_dets.errors) + if bool(line_dets.marker): + _markers = _markers + (str(line_dets.marker),) + # TOD0: do more processing + continue # end of line processing + if line.startswith("@"): #handle special geno formats + _dot_geno_lines += 1 + _dot_geno_fields = {**_dot_geno_fields, **parse_dot_geno_line(line)} + if line.startswith("#"): #handle special geno formats + _comment_lines += 1 return FileStats( + total_lines=_total_lines, data_lines=_data_lines, + comment_lines=_comment_lines, + dot_geno_lines=_dot_geno_lines, + dot_geno_fields=(DotGenoFields(**_dot_geno_fields) + if bool(_dot_geno_fields) else None), geno_symbols=tuple(genosymbols), samples_list=col_ids.samples, markers_list=_markers, @@ -308,7 +354,7 @@ if __name__ == "__main__": setup_logging(logger, args.log_level.upper(), tuple()) logger.debug("CLI Arguments: %s", args) - stats = file_statistics( + stats = compute_statistics( args.csv_file, { "markers": args.markers_field, @@ -323,7 +369,7 @@ if __name__ == "__main__": print(json.dumps(asdict(stats))) return SysExit.OK except FileNotFoundError as _fnf: - logger.error(", ".join(_fnf.args)) + logger.error(", ".join(str(arg) for arg in _fnf.args), exc_info=True) return SysExit.FILENOTFOUND sys.exit(main().value) diff --git a/scripts/genotypes/types.py b/scripts/genotypes/types.py index 7f0e6cb..d765fe5 100644 --- a/scripts/genotypes/types.py +++ b/scripts/genotypes/types.py @@ -56,9 +56,14 @@ class UntransposedLineDetails: @dataclass(frozen=True) class FileStats:# pylint: disable=[too-many-instance-attributes] """Custom file statistics.""" + total_lines: int data_lines: int geno_symbols: Sequence[str] + comment_lines: int = 0 + blank_lines: int = 0 + dot_geno_lines: int = 0 + dot_geno_fields: Optional[DotGenoFields] = None samples_list: Sequence[str] = field(default_factory=tuple) markers_list: Sequence[str] = field(default_factory=tuple) errors: Sequence[LineError] = field(default_factory=tuple) |
