diff options
| author | pi coding agent <pi@localhost> | 2026-10-22 18:59:33 +0000 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-22 14:25:09 -0500 |
| commit | a897ceca02b78a9426652ac3092c79278cdc681c (patch) | |
| tree | 1ed499d3c583c93890d30864871740a276e4f718 /scripts/genotypes | |
| parent | e3859b3945606aed7bd6a3861af8a525e061a2b1 (diff) | |
| download | gn-uploader-a897ceca02b78a9426652ac3092c79278cdc681c.tar.gz | |
Move scripts into the uploader package
Move the top-level `scripts` package to `uploader/scripts` so that it becomes the `uploader.scripts` package. All files are moved unchanged; references are updated in a follow-up commit.
Diffstat (limited to 'scripts/genotypes')
| -rw-r--r-- | scripts/genotypes/__init__.py | 1 | ||||
| -rw-r--r-- | scripts/genotypes/preprocess_csv_collect_info.py | 509 | ||||
| -rw-r--r-- | scripts/genotypes/types.py | 69 |
3 files changed, 0 insertions, 579 deletions
diff --git a/scripts/genotypes/__init__.py b/scripts/genotypes/__init__.py deleted file mode 100644 index 74fa7a5..0000000 --- a/scripts/genotypes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Scripts handling genotype-specific tasks.""" diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py deleted file mode 100644 index f8cdeed..0000000 --- a/scripts/genotypes/preprocess_csv_collect_info.py +++ /dev/null @@ -1,509 +0,0 @@ -"""Preprocess the genotypes CSV files: Collect basic info.""" -import os -import sys -import json -import logging -import multiprocessing as mproc -from pathlib import Path -from dataclasses import asdict, dataclass, field, fields -from typing import Union, Iterable, Iterator, Optional, Callable, Sequence -from argparse import Namespace, ArgumentParser - -from gn_libs.cli import SysExit - -from uploader.csvprocessing import (is_blank_line, - read_csv_file, - read_text_file, - read_csv_file_headers, - make_comment_line_checker) - -from scripts.cli.options import add_logging -from scripts.cli.logging import setup_logging -from scripts.cli.validators import file_exists - -from .types import (FileStats, - LineError, - DotGenoFields, - FieldsIdentity, - TransposedLineDetails, - UntransposedLineDetails) - - -logger = logging.getLogger(__name__) - -FILE_SIZE_THRESHOLD = 100 * 1024 * 1024 # 100MB -MULTIPROCESSING_CHUNK_SIZE = 10_000 # lines - - -def filecontents(path: Path) -> Iterator[str]: - """Return an iterator to the file contents""" - with path.open("r") as fl: - yield from fl.readlines() - - -def identify_columns( - headers: tuple[str, ...], - settings: dict[str, str] -) -> FieldsIdentity: - """Identify columns using settings provided by user""" - _id: dict[str, Union[str, str]] = { - "marker": "", - "chromosome": "", - "genetic_distance": "", - "physical_distance": "" - } - _samples: tuple[str, ...] = tuple() - for header in headers: - if header.lower() == settings["markers"].lower(): - _id["marker"] = header - continue - if header.lower() == settings["chromosome"].lower(): - _id["chromosome"] = header - continue - if header.lower() == settings["cm"].lower(): - # linkage map: genetic distance - _id["genetic_distance"] = header - continue - if header.lower() == settings["mb"].lower(): - # physical map: physical distance - _id["physical_distance"] = header - continue - - _samples = _samples + (header,) - - return FieldsIdentity(**_id, samples=_samples) - - -def process_transposed( - line: tuple[str, ...], - settings: dict, -) -> TransposedLineDetails: - """Process a line in a transposed file.""" - return TransposedLineDetails( - line_content=line, - geno_symbols=tuple(set( - line[1:] if line[0] not in ( - settings["markers"], - settings["chromosome"], - settings["cm"], - settings["mb"]) - else tuple())), - line_header=line[0], - errors=tuple(), - is_markers_line=line[0]==settings["markers"]) - - -def process_untransposed( - line: tuple[str, ...], - settings: dict, - headers: tuple[str, ...] -) -> UntransposedLineDetails: - """Processed line in a non-transposed file.""" - _marker: Optional[str] = None - _errors: tuple[LineError, ...] = tuple() - _symbols = tuple(_symb for _head, _symb in zip(headers, line) - if _head not in ( - settings["markers"], - settings["chromosome"], - settings["cm"], - settings["mb"])) - - try: - _idx = headers.index(settings["markers"]) - _marker = line[_idx] - except ValueError as _verr: - _idx = 0 - - if len(headers) != len(line): - _errors = _errors + (LineError( - row_header=line[_idx], - column_header="-", - message=( - f"Columns mismatch: There are {len(headers)} headers but only " - f"{len(line)} field values in the line."), - line=line - ),) - - return UntransposedLineDetails( - line_content=line, - line_header=line[0], - geno_symbols=tuple(set(_symbols)), - errors=_errors, - marker=_marker) - - -def validate_headers( - headers: tuple[str, ...], - settings: dict -) -> tuple[LineError, ...]: - """Validate that the headers are correct.""" - _errors: tuple[LineError, ...] = tuple() - _header_descriptions = { - "markers": "marker", - "chromosome": "chromosome", - "cm": "genetic distance (cM)", - "mb": "physical location (Mb)"} - for _header, _desc in _header_descriptions.items(): - try: - headers.index(settings[_header]) - except ValueError as _verr: - _errors = _errors + (LineError( - row_header=headers[0], - column_header="-", - message=( - f"Header row does not have a '{_desc}' column named " - f"'{settings[_header]}'."), - line=headers),) - - return _errors - - -def is_dot_geno_line(line: str) -> bool: - """Check whether the line is a custom .geno format line - - :param line: The line to check for custom formatting - :type line: str - :rtype bool - """ - return line.startswith("@") - -def _make_line_splitter(separator: str) -> Callable[[str], tuple[str, ...]]: - """Create a function to split a line into its fields.""" - def _line_splitter(line: str) -> tuple[str, ...]: - return tuple(field.strip() for field in line.split(separator)) - return _line_splitter - - -def _make_is_skippable_line( - fileconfigs: dict, - headers: tuple[str, ...] -) -> Callable[[str], bool]: - """Create a function to check whether a line is skippable.""" - is_comment_line = make_comment_line_checker(fileconfigs["comment.char"]) - line_splitter = _make_line_splitter(fileconfigs["separator"]) - - def _is_skippable_line(line: str) -> bool: - return (is_blank_line(line) - or is_comment_line(line) - or is_dot_geno_line(line) - or line_splitter(line) == headers) - - return _is_skippable_line - - -@dataclass -class _ChunkStats:# pylint: disable=[too-many-instance-attributes] - """Statistics accumulated from a chunk of lines.""" - total_lines: int = 0 - data_lines: int = 0 - comment_lines: int = 0 - blank_lines: int = 0 - dot_geno_lines: int = 0 - dot_geno_fields: dict = field(default_factory=dict) - geno_symbols: set = field(default_factory=set) - markers: tuple = field(default_factory=tuple) - markers_seen: bool = False - errors: tuple = field(default_factory=tuple) - - -def _accumulate( - headers: tuple[str, ...], - fileconfigs: dict, - lines: Iterable[str] -) -> _ChunkStats: - """Accumulate statistics from the provided lines.""" - is_skippable_line = _make_is_skippable_line(fileconfigs, headers) - line_splitter = _make_line_splitter(fileconfigs["separator"]) - stats = _ChunkStats() - for line in (ln.strip() for ln in lines): - stats.total_lines += 1 - if not is_skippable_line(line): - stats.data_lines += 1 - fields_values = line_splitter(line) - if fileconfigs["transposed"]: - t_line_dets = process_transposed(fields_values, fileconfigs) - stats.geno_symbols.update(t_line_dets.geno_symbols) - stats.errors = stats.errors + tuple(t_line_dets.errors) - if t_line_dets.is_markers_line: - stats.markers = tuple(t_line_dets.line_content[1:]) - stats.markers_seen = True - else: - line_dets = process_untransposed( - fields_values, - fileconfigs, - headers=headers) - stats.geno_symbols.update(line_dets.geno_symbols) - stats.errors = stats.errors + tuple(line_dets.errors) - if bool(line_dets.marker): - stats.markers = stats.markers + (str(line_dets.marker),) - # TOD0: do more processing - continue # end of line processing - if line.startswith("@"): #handle special geno formats - stats.dot_geno_lines += 1 - stats.dot_geno_fields = { - **stats.dot_geno_fields, - **parse_dot_geno_line(line)} - if line.startswith("#"): #handle special geno formats - stats.comment_lines += 1 - if line == "": # Blank line - stats.blank_lines += 1 - return stats - - -def _merge_chunk_stats( - left: _ChunkStats, - right: _ChunkStats, - transposed: bool -) -> _ChunkStats: - """Merge two sets of chunk statistics, preserving file order. - - The two sets of statistics are expected to have been accumulated from - consecutive chunks of the same file, with ``left`` covering the earlier - chunk. - """ - if transposed: - # The markers come from the last markers line encountered. Keep the - # earlier markers when the later chunk did not see a markers line. - markers = right.markers if right.markers_seen else left.markers - else: - markers = left.markers + right.markers - return _ChunkStats( - total_lines=left.total_lines + right.total_lines, - data_lines=left.data_lines + right.data_lines, - comment_lines=left.comment_lines + right.comment_lines, - blank_lines=left.blank_lines + right.blank_lines, - dot_geno_lines=left.dot_geno_lines + right.dot_geno_lines, - dot_geno_fields={**left.dot_geno_fields, **right.dot_geno_fields}, - geno_symbols=left.geno_symbols | right.geno_symbols, - markers=markers, - markers_seen=left.markers_seen or right.markers_seen, - errors=left.errors + right.errors) - - -def _merge_all_chunk_stats( - chunks: Iterable[_ChunkStats], - transposed: bool -) -> _ChunkStats: - """Merge a sequence of chunk statistics into a single set of statistics.""" - merged = _ChunkStats() - for chunk_stats in chunks: - merged = _merge_chunk_stats(merged, chunk_stats, transposed) - return merged - - -def _read_chunks( - lines: Iterable[str], - chunk_size: int -) -> Iterator[list[str]]: - """Group the lines into chunks of at most ``chunk_size`` lines.""" - chunk: list[str] = [] - for line in lines: - chunk.append(line) - if len(chunk) >= chunk_size: - yield chunk - chunk = [] - if chunk: - yield chunk - - -def _process_chunk(args: tuple[tuple[str, ...], dict, list[str]]) -> _ChunkStats: - """Process a single chunk of lines. - - This is a module-level function so that it can be pickled and run in a - separate process. - """ - headers, fileconfigs, chunk = args - return _accumulate(headers, fileconfigs, chunk) - - -def file_stats_multiprocessing( - fileconfigs: dict, - headers: tuple[str, ...], - lines: Iterable[str], - chunk_size: int = MULTIPROCESSING_CHUNK_SIZE -) -> FileStats: - """File stats from the lines in the file using multiple processes.""" - with mproc.Pool(max(1, mproc.cpu_count() - 1)) as pool: - chunks = ((headers, fileconfigs, chunk) - for chunk in _read_chunks(lines, chunk_size)) - merged = _merge_all_chunk_stats( - pool.imap(_process_chunk, chunks), - fileconfigs["transposed"]) - return _as_file_stats( - merged, - identify_columns(headers, fileconfigs).samples, - validate_headers(headers, fileconfigs)) - - -def _as_file_stats( - stats: _ChunkStats, - samples: Sequence[str], - header_errors: tuple[LineError, ...] -) -> FileStats: - """Convert accumulated statistics into file statistics.""" - return FileStats( - total_lines=stats.total_lines, - data_lines=stats.data_lines, - comment_lines=stats.comment_lines, - blank_lines=stats.blank_lines, - dot_geno_lines=stats.dot_geno_lines, - dot_geno_fields=(DotGenoFields(**stats.dot_geno_fields) - if bool(stats.dot_geno_fields) else None), - geno_symbols=tuple(sorted(stats.geno_symbols)), - samples_list=tuple(samples), - markers_list=stats.markers, - errors=tuple(header_errors) + tuple(stats.errors)) - - -def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats: - """Compute file statistics from the lines in the file.""" - is_comment_line = make_comment_line_checker(fileconfigs["comment.char"]) - headers = read_csv_file_headers(filepath, - fileconfigs["transposed"], - fileconfigs["separator"], - lambda line: (is_blank_line(line) - or is_comment_line(line) - or is_dot_geno_line(line))) - logger.debug("THE FILE HEADERS: %s", headers) - basic_stats = os.stat(filepath) - lines = read_text_file(filepath) - return (file_stats_multiprocessing(fileconfigs, headers, lines) - if basic_stats.st_size >= FILE_SIZE_THRESHOLD - else file_stats_singlethread(fileconfigs, headers, lines)) - - -def parse_dot_geno_line(line) -> dict[str, str]: - """Parse the dot_geno format line.""" - assert line.startswith("@") - acceptable_fields = tuple(field.name for field in fields(DotGenoFields)) - parts = tuple(item.strip() for item in line[1:].split(":")) - if len(parts) == 2 and parts[0] in acceptable_fields: - return {parts[0]: parts[1]} - return {} - - -def file_stats_singlethread( - fileconfigs: dict, - headers: tuple[str, ...], - lines: Iterable[str] -) -> FileStats: - """File stats from the lines in the file using a single-threaded process.""" - return _as_file_stats( - _accumulate(headers, fileconfigs, lines), - identify_columns(headers, fileconfigs).samples, - validate_headers(headers, fileconfigs)) - - -if __name__ == "__main__": - - def parse_args() -> Namespace: - """Parse the command-line arguments.""" - parser = add_logging( - ArgumentParser( - prog="preprocess-csv-collect-info", - description=( - "Pre-process the CSV file, with a focus on collecting " - "basic file information and statistics."))) - - parser.add_argument( - "csv_file", - metavar="CSV-FILE", - type=file_exists, - help="The CSV file to process.") - - ## format-specific arguments - parser.add_argument( - "--is-transposed", - action="store_true", - default=False, - help="Are the rows and columns in the CSV file flipped?") - - parser.add_argument( - "--separator", - metavar="FIELDS-SEPARATOR", - type=str, - default=",", - help=( - "The character acting as the fields' separator in the CSV file.")) - - parser.add_argument( - "--comment_char", - metavar="COMMENTS-CHARACTER", - type=str, - default="#", - help="The character indicating a comment line.") - - parser.add_argument( - "--na_strings", - metavar="MISSING-VALUES-STRING", - action="append", - type=str, - default=["-", "NA", "N/A"], - help="The character indicating a comment line.") - - ## content-specific arguments - parser.add_argument( - "--markers_field", - metavar="MARKERS-FIELD", - type=str, - default="Locus", - help=( - "The name of the column (or row, when transposed) containing " - "the names of the genetic markers.")) - - parser.add_argument( - "--chr_field", - metavar="CHROMOSOMES-FIELD", - type=str, - default="Chr", - help=( - "The name of the column (or row, when transposed) containing " - "the names of the names of the chromosomes.")) - - parser.add_argument( - "--cm_field", - metavar="GENETIC-DISTANCES-FIELD", - type=str, - default="cM", - help=( - "The name of the column (or row, when transposed) containing " - "the genetic distance values in centimorgans.")) - - parser.add_argument( - "--mb_field", - metavar="PHYSICAL-DISTANCES-FIELD", - type=str, - default="Mb", - help=( - "The name of the column (or row, when transposed) containing " - "the physical distance values in megabase pairs.")) - - return parser.parse_args() - - - def main() -> SysExit: - """Entry-point to the pre-processor.""" - try: - args = parse_args() - setup_logging(logger, args.log_level.upper(), tuple()) - logger.debug("CLI Arguments: %s", args) - - stats = compute_statistics( - args.csv_file, - { - "markers": args.markers_field, - "chromosome": args.chr_field, - "cm": args.cm_field, - "mb": args.mb_field, - "separator": args.separator, - "comment.char": args.comment_char, - "na_strings": args.na_strings, - "transposed": args.is_transposed - }) - print(json.dumps(asdict(stats))) - return SysExit.OK - except FileNotFoundError as _fnf: - logger.error(", ".join(str(arg) for arg in _fnf.args), exc_info=True) - return SysExit.FILENOTFOUND - - sys.exit(main().value) diff --git a/scripts/genotypes/types.py b/scripts/genotypes/types.py deleted file mode 100644 index d765fe5..0000000 --- a/scripts/genotypes/types.py +++ /dev/null @@ -1,69 +0,0 @@ -"""The datatypes this package will use.""" -from typing import Sequence, Optional -from dataclasses import field, dataclass - - -@dataclass -class DotGenoFields:# Still necessary?? - """Custom .geno file format values""" - name: str - mat: str - pat: str - het: str - type: Optional[str] = None - unk: Optional[str] = None - - -@dataclass(frozen=True) -class FieldsIdentity: - """Data type for identity of the fields.""" - marker: str - chromosome: str - genetic_distance: str # cM - physical_distance: str # Mb - samples: Sequence[str] - - -@dataclass -class LineError: - """An error in a specific line of the genotype file.""" - row_header: str - column_header: str - message: str - line: Sequence[str] - - -@dataclass(frozen=True) -class TransposedLineDetails: - """Data type for processed line details""" - line_content: Sequence[str] - geno_symbols: Sequence[str] - line_header: str - errors: Sequence[LineError] - is_markers_line: bool = False - - -@dataclass(frozen=True) -class UntransposedLineDetails: - """Data type for processed line details""" - line_content: Sequence[str] - geno_symbols: Sequence[str] - line_header: str - errors: Sequence[LineError] - marker: Optional[str] = None - - -@dataclass(frozen=True) -class FileStats:# pylint: disable=[too-many-instance-attributes] - """Custom file statistics.""" - total_lines: int - data_lines: int - geno_symbols: Sequence[str] - - comment_lines: int = 0 - blank_lines: int = 0 - dot_geno_lines: int = 0 - dot_geno_fields: Optional[DotGenoFields] = None - samples_list: Sequence[str] = field(default_factory=tuple) - markers_list: Sequence[str] = field(default_factory=tuple) - errors: Sequence[LineError] = field(default_factory=tuple) |
