about summary refs log tree commit diff
path: root/scripts/genotypes
diff options
context:
space:
mode:
authorpi coding agent <pi@localhost>2026-10-22 18:59:33 +0000
committerFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-10-22 14:25:09 -0500
commita897ceca02b78a9426652ac3092c79278cdc681c (patch)
tree1ed499d3c583c93890d30864871740a276e4f718 /scripts/genotypes
parente3859b3945606aed7bd6a3861af8a525e061a2b1 (diff)
downloadgn-uploader-a897ceca02b78a9426652ac3092c79278cdc681c.tar.gz
Move scripts into the uploader package
Move the top-level `scripts` package to `uploader/scripts` so that it becomes
the `uploader.scripts` package. All files are moved unchanged; references are
updated in a follow-up commit.
Diffstat (limited to 'scripts/genotypes')
-rw-r--r--scripts/genotypes/__init__.py1
-rw-r--r--scripts/genotypes/preprocess_csv_collect_info.py509
-rw-r--r--scripts/genotypes/types.py69
3 files changed, 0 insertions, 579 deletions
diff --git a/scripts/genotypes/__init__.py b/scripts/genotypes/__init__.py
deleted file mode 100644
index 74fa7a5..0000000
--- a/scripts/genotypes/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"""Scripts handling genotype-specific tasks."""
diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py
deleted file mode 100644
index f8cdeed..0000000
--- a/scripts/genotypes/preprocess_csv_collect_info.py
+++ /dev/null
@@ -1,509 +0,0 @@
-"""Preprocess the genotypes CSV files: Collect basic info."""
-import os
-import sys
-import json
-import logging
-import multiprocessing as mproc
-from pathlib import Path
-from dataclasses import asdict, dataclass, field, fields
-from typing import Union, Iterable, Iterator, Optional, Callable, Sequence
-from argparse import Namespace, ArgumentParser
-
-from gn_libs.cli import SysExit
-
-from uploader.csvprocessing import (is_blank_line,
-                                    read_csv_file,
-                                    read_text_file,
-                                    read_csv_file_headers,
-                                    make_comment_line_checker)
-
-from scripts.cli.options import add_logging
-from scripts.cli.logging import setup_logging
-from scripts.cli.validators import file_exists
-
-from .types import (FileStats,
-                    LineError,
-                    DotGenoFields,
-                    FieldsIdentity,
-                    TransposedLineDetails,
-                    UntransposedLineDetails)
-
-
-logger = logging.getLogger(__name__)
-
-FILE_SIZE_THRESHOLD = 100 * 1024 * 1024 # 100MB
-MULTIPROCESSING_CHUNK_SIZE = 10_000 # lines
-
-
-def filecontents(path: Path) -> Iterator[str]:
-    """Return an iterator to the file contents"""
-    with path.open("r") as fl:
-        yield from fl.readlines()
-
-
-def identify_columns(
-        headers: tuple[str, ...],
-        settings: dict[str, str]
-) -> FieldsIdentity:
-    """Identify columns using settings provided by user"""
-    _id: dict[str, Union[str, str]] = {
-        "marker": "",
-        "chromosome": "",
-        "genetic_distance": "",
-        "physical_distance": ""
-    }
-    _samples: tuple[str, ...] = tuple()
-    for header in headers:
-        if header.lower() == settings["markers"].lower():
-            _id["marker"] = header
-            continue
-        if header.lower() == settings["chromosome"].lower():
-            _id["chromosome"] = header
-            continue
-        if header.lower() == settings["cm"].lower():
-            # linkage map: genetic distance
-            _id["genetic_distance"] = header
-            continue
-        if header.lower() == settings["mb"].lower():
-            # physical map: physical distance
-            _id["physical_distance"] = header
-            continue
-
-        _samples = _samples + (header,)
-
-    return FieldsIdentity(**_id, samples=_samples)
-
-
-def process_transposed(
-        line: tuple[str, ...],
-        settings: dict,
-) -> TransposedLineDetails:
-    """Process a line in a transposed file."""
-    return TransposedLineDetails(
-        line_content=line,
-        geno_symbols=tuple(set(
-            line[1:] if line[0] not in (
-                settings["markers"],
-                settings["chromosome"],
-                settings["cm"],
-                settings["mb"])
-            else tuple())),
-        line_header=line[0],
-        errors=tuple(),
-        is_markers_line=line[0]==settings["markers"])
-
-
-def process_untransposed(
-        line: tuple[str, ...],
-        settings: dict,
-        headers: tuple[str, ...]
-) -> UntransposedLineDetails:
-    """Processed line in a non-transposed file."""
-    _marker: Optional[str] = None
-    _errors: tuple[LineError, ...] = tuple()
-    _symbols = tuple(_symb for _head, _symb in zip(headers, line)
-                if _head not in (
-                        settings["markers"],
-                        settings["chromosome"],
-                        settings["cm"],
-                        settings["mb"]))
-
-    try:
-        _idx = headers.index(settings["markers"])
-        _marker = line[_idx]
-    except ValueError as _verr:
-        _idx = 0
-
-    if len(headers) != len(line):
-        _errors = _errors + (LineError(
-            row_header=line[_idx],
-            column_header="-",
-            message=(
-                f"Columns mismatch: There are {len(headers)} headers but only "
-                f"{len(line)} field values in the line."),
-            line=line
-        ),)
-
-    return UntransposedLineDetails(
-        line_content=line,
-        line_header=line[0],
-        geno_symbols=tuple(set(_symbols)),
-        errors=_errors,
-        marker=_marker)
-
-
-def validate_headers(
-        headers: tuple[str, ...],
-        settings: dict
-) -> tuple[LineError, ...]:
-    """Validate that the headers are correct."""
-    _errors: tuple[LineError, ...] = tuple()
-    _header_descriptions = {
-        "markers": "marker",
-        "chromosome": "chromosome",
-        "cm": "genetic distance (cM)",
-        "mb": "physical location (Mb)"}
-    for _header, _desc in _header_descriptions.items():
-        try:
-            headers.index(settings[_header])
-        except ValueError as _verr:
-            _errors = _errors + (LineError(
-                row_header=headers[0],
-                column_header="-",
-                message=(
-                    f"Header row does not have a '{_desc}' column named "
-                    f"'{settings[_header]}'."),
-                line=headers),)
-
-    return _errors
-
-
-def is_dot_geno_line(line: str) -> bool:
-    """Check whether the line is a custom .geno format line
-
-    :param line: The line to check for custom formatting
-    :type line: str
-    :rtype bool
-    """
-    return line.startswith("@")
-
-def _make_line_splitter(separator: str) -> Callable[[str], tuple[str, ...]]:
-    """Create a function to split a line into its fields."""
-    def _line_splitter(line: str) -> tuple[str, ...]:
-        return tuple(field.strip() for field in line.split(separator))
-    return _line_splitter
-
-
-def _make_is_skippable_line(
-        fileconfigs: dict,
-        headers: tuple[str, ...]
-) -> Callable[[str], bool]:
-    """Create a function to check whether a line is skippable."""
-    is_comment_line = make_comment_line_checker(fileconfigs["comment.char"])
-    line_splitter = _make_line_splitter(fileconfigs["separator"])
-
-    def _is_skippable_line(line: str) -> bool:
-        return (is_blank_line(line)
-                or is_comment_line(line)
-                or is_dot_geno_line(line)
-                or line_splitter(line) == headers)
-
-    return _is_skippable_line
-
-
-@dataclass
-class _ChunkStats:# pylint: disable=[too-many-instance-attributes]
-    """Statistics accumulated from a chunk of lines."""
-    total_lines: int = 0
-    data_lines: int = 0
-    comment_lines: int = 0
-    blank_lines: int = 0
-    dot_geno_lines: int = 0
-    dot_geno_fields: dict = field(default_factory=dict)
-    geno_symbols: set = field(default_factory=set)
-    markers: tuple = field(default_factory=tuple)
-    markers_seen: bool = False
-    errors: tuple = field(default_factory=tuple)
-
-
-def _accumulate(
-        headers: tuple[str, ...],
-        fileconfigs: dict,
-        lines: Iterable[str]
-) -> _ChunkStats:
-    """Accumulate statistics from the provided lines."""
-    is_skippable_line = _make_is_skippable_line(fileconfigs, headers)
-    line_splitter = _make_line_splitter(fileconfigs["separator"])
-    stats = _ChunkStats()
-    for line in (ln.strip() for ln in lines):
-        stats.total_lines += 1
-        if not is_skippable_line(line):
-            stats.data_lines += 1
-            fields_values = line_splitter(line)
-            if fileconfigs["transposed"]:
-                t_line_dets = process_transposed(fields_values, fileconfigs)
-                stats.geno_symbols.update(t_line_dets.geno_symbols)
-                stats.errors = stats.errors + tuple(t_line_dets.errors)
-                if t_line_dets.is_markers_line:
-                    stats.markers = tuple(t_line_dets.line_content[1:])
-                    stats.markers_seen = True
-            else:
-                line_dets = process_untransposed(
-                    fields_values,
-                    fileconfigs,
-                    headers=headers)
-                stats.geno_symbols.update(line_dets.geno_symbols)
-                stats.errors = stats.errors + tuple(line_dets.errors)
-                if bool(line_dets.marker):
-                    stats.markers = stats.markers + (str(line_dets.marker),)
-            # TOD0: do more processing
-            continue # end of line processing
-        if line.startswith("@"): #handle special geno formats
-            stats.dot_geno_lines += 1
-            stats.dot_geno_fields = {
-                **stats.dot_geno_fields,
-                **parse_dot_geno_line(line)}
-        if line.startswith("#"): #handle special geno formats
-            stats.comment_lines += 1
-        if line == "": # Blank line
-            stats.blank_lines += 1
-    return stats
-
-
-def _merge_chunk_stats(
-        left: _ChunkStats,
-        right: _ChunkStats,
-        transposed: bool
-) -> _ChunkStats:
-    """Merge two sets of chunk statistics, preserving file order.
-
-    The two sets of statistics are expected to have been accumulated from
-    consecutive chunks of the same file, with ``left`` covering the earlier
-    chunk.
-    """
-    if transposed:
-        # The markers come from the last markers line encountered. Keep the
-        # earlier markers when the later chunk did not see a markers line.
-        markers = right.markers if right.markers_seen else left.markers
-    else:
-        markers = left.markers + right.markers
-    return _ChunkStats(
-        total_lines=left.total_lines + right.total_lines,
-        data_lines=left.data_lines + right.data_lines,
-        comment_lines=left.comment_lines + right.comment_lines,
-        blank_lines=left.blank_lines + right.blank_lines,
-        dot_geno_lines=left.dot_geno_lines + right.dot_geno_lines,
-        dot_geno_fields={**left.dot_geno_fields, **right.dot_geno_fields},
-        geno_symbols=left.geno_symbols | right.geno_symbols,
-        markers=markers,
-        markers_seen=left.markers_seen or right.markers_seen,
-        errors=left.errors + right.errors)
-
-
-def _merge_all_chunk_stats(
-        chunks: Iterable[_ChunkStats],
-        transposed: bool
-) -> _ChunkStats:
-    """Merge a sequence of chunk statistics into a single set of statistics."""
-    merged = _ChunkStats()
-    for chunk_stats in chunks:
-        merged = _merge_chunk_stats(merged, chunk_stats, transposed)
-    return merged
-
-
-def _read_chunks(
-        lines: Iterable[str],
-        chunk_size: int
-) -> Iterator[list[str]]:
-    """Group the lines into chunks of at most ``chunk_size`` lines."""
-    chunk: list[str] = []
-    for line in lines:
-        chunk.append(line)
-        if len(chunk) >= chunk_size:
-            yield chunk
-            chunk = []
-    if chunk:
-        yield chunk
-
-
-def _process_chunk(args: tuple[tuple[str, ...], dict, list[str]]) -> _ChunkStats:
-    """Process a single chunk of lines.
-
-    This is a module-level function so that it can be pickled and run in a
-    separate process.
-    """
-    headers, fileconfigs, chunk = args
-    return _accumulate(headers, fileconfigs, chunk)
-
-
-def file_stats_multiprocessing(
-        fileconfigs: dict,
-        headers: tuple[str, ...],
-        lines: Iterable[str],
-        chunk_size: int = MULTIPROCESSING_CHUNK_SIZE
-) -> FileStats:
-    """File stats from the lines in the file using multiple processes."""
-    with mproc.Pool(max(1, mproc.cpu_count() - 1)) as pool:
-        chunks = ((headers, fileconfigs, chunk)
-                  for chunk in _read_chunks(lines, chunk_size))
-        merged = _merge_all_chunk_stats(
-            pool.imap(_process_chunk, chunks),
-            fileconfigs["transposed"])
-    return _as_file_stats(
-        merged,
-        identify_columns(headers, fileconfigs).samples,
-        validate_headers(headers, fileconfigs))
-
-
-def _as_file_stats(
-        stats: _ChunkStats,
-        samples: Sequence[str],
-        header_errors: tuple[LineError, ...]
-) -> FileStats:
-    """Convert accumulated statistics into file statistics."""
-    return FileStats(
-        total_lines=stats.total_lines,
-        data_lines=stats.data_lines,
-        comment_lines=stats.comment_lines,
-        blank_lines=stats.blank_lines,
-        dot_geno_lines=stats.dot_geno_lines,
-        dot_geno_fields=(DotGenoFields(**stats.dot_geno_fields)
-                         if bool(stats.dot_geno_fields) else None),
-        geno_symbols=tuple(sorted(stats.geno_symbols)),
-        samples_list=tuple(samples),
-        markers_list=stats.markers,
-        errors=tuple(header_errors) + tuple(stats.errors))
-
-
-def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats:
-    """Compute file statistics from the lines in the file."""
-    is_comment_line = make_comment_line_checker(fileconfigs["comment.char"])
-    headers = read_csv_file_headers(filepath,
-                                    fileconfigs["transposed"],
-                                    fileconfigs["separator"],
-                                    lambda line: (is_blank_line(line)
-                                                  or is_comment_line(line)
-                                                  or is_dot_geno_line(line)))
-    logger.debug("THE FILE HEADERS: %s", headers)
-    basic_stats = os.stat(filepath)
-    lines = read_text_file(filepath)
-    return (file_stats_multiprocessing(fileconfigs, headers, lines)
-            if basic_stats.st_size >= FILE_SIZE_THRESHOLD
-            else file_stats_singlethread(fileconfigs, headers, lines))
-
-
-def parse_dot_geno_line(line) -> dict[str, str]:
-    """Parse the dot_geno format line."""
-    assert line.startswith("@")
-    acceptable_fields = tuple(field.name for field in fields(DotGenoFields))
-    parts = tuple(item.strip() for item in line[1:].split(":"))
-    if len(parts) == 2 and parts[0] in acceptable_fields:
-        return {parts[0]: parts[1]}
-    return {}
-
-
-def file_stats_singlethread(
-        fileconfigs: dict,
-        headers: tuple[str, ...],
-        lines: Iterable[str]
-) -> FileStats:
-    """File stats from the lines in the file using a single-threaded process."""
-    return _as_file_stats(
-        _accumulate(headers, fileconfigs, lines),
-        identify_columns(headers, fileconfigs).samples,
-        validate_headers(headers, fileconfigs))
-
-
-if __name__ == "__main__":
-
-    def parse_args() -> Namespace:
-        """Parse the command-line arguments."""
-        parser = add_logging(
-            ArgumentParser(
-                prog="preprocess-csv-collect-info",
-                description=(
-                    "Pre-process the CSV file, with a focus on collecting "
-                    "basic file information and statistics.")))
-
-        parser.add_argument(
-            "csv_file",
-            metavar="CSV-FILE",
-            type=file_exists,
-            help="The CSV file to process.")
-
-        ## format-specific arguments
-        parser.add_argument(
-            "--is-transposed",
-            action="store_true",
-            default=False,
-            help="Are the rows and columns in the CSV file flipped?")
-
-        parser.add_argument(
-            "--separator",
-            metavar="FIELDS-SEPARATOR",
-            type=str,
-            default=",",
-            help=(
-                "The character acting as the fields' separator in the CSV file."))
-
-        parser.add_argument(
-            "--comment_char",
-            metavar="COMMENTS-CHARACTER",
-            type=str,
-            default="#",
-            help="The character indicating a comment line.")
-
-        parser.add_argument(
-            "--na_strings",
-            metavar="MISSING-VALUES-STRING",
-            action="append",
-            type=str,
-            default=["-", "NA", "N/A"],
-            help="The character indicating a comment line.")
-
-        ## content-specific arguments
-        parser.add_argument(
-            "--markers_field",
-            metavar="MARKERS-FIELD",
-            type=str,
-            default="Locus",
-            help=(
-                "The name of the column (or row, when transposed) containing "
-                "the names of the genetic markers."))
-
-        parser.add_argument(
-            "--chr_field",
-            metavar="CHROMOSOMES-FIELD",
-            type=str,
-            default="Chr",
-            help=(
-                "The name of the column (or row, when transposed) containing "
-                "the names of the names of the chromosomes."))
-
-        parser.add_argument(
-            "--cm_field",
-            metavar="GENETIC-DISTANCES-FIELD",
-            type=str,
-            default="cM",
-            help=(
-                "The name of the column (or row, when transposed) containing "
-                "the genetic distance values in centimorgans."))
-
-        parser.add_argument(
-            "--mb_field",
-            metavar="PHYSICAL-DISTANCES-FIELD",
-            type=str,
-            default="Mb",
-            help=(
-                "The name of the column (or row, when transposed) containing "
-                "the physical distance values in megabase pairs."))
-
-        return parser.parse_args()
-
-
-    def main() -> SysExit:
-        """Entry-point to the pre-processor."""
-        try:
-            args = parse_args()
-            setup_logging(logger, args.log_level.upper(), tuple())
-            logger.debug("CLI Arguments: %s", args)
-
-            stats = compute_statistics(
-                args.csv_file,
-                {
-                    "markers": args.markers_field,
-                    "chromosome": args.chr_field,
-                    "cm": args.cm_field,
-                    "mb": args.mb_field,
-                    "separator": args.separator,
-                    "comment.char": args.comment_char,
-                    "na_strings": args.na_strings,
-                    "transposed": args.is_transposed
-                })
-            print(json.dumps(asdict(stats)))
-            return SysExit.OK
-        except FileNotFoundError as _fnf:
-            logger.error(", ".join(str(arg) for arg in _fnf.args), exc_info=True)
-            return SysExit.FILENOTFOUND
-
-    sys.exit(main().value)
diff --git a/scripts/genotypes/types.py b/scripts/genotypes/types.py
deleted file mode 100644
index d765fe5..0000000
--- a/scripts/genotypes/types.py
+++ /dev/null
@@ -1,69 +0,0 @@
-"""The datatypes this package will use."""
-from typing import Sequence, Optional
-from dataclasses import field, dataclass
-
-
-@dataclass
-class DotGenoFields:# Still necessary??
-    """Custom .geno file format values"""
-    name: str
-    mat: str
-    pat: str
-    het: str
-    type: Optional[str] = None
-    unk: Optional[str] = None
-
-
-@dataclass(frozen=True)
-class FieldsIdentity:
-    """Data type for identity of the fields."""
-    marker: str
-    chromosome: str
-    genetic_distance: str # cM
-    physical_distance: str # Mb
-    samples: Sequence[str]
-
-
-@dataclass
-class LineError:
-    """An error in a specific line of the genotype file."""
-    row_header: str
-    column_header: str
-    message: str
-    line: Sequence[str]
-
-
-@dataclass(frozen=True)
-class TransposedLineDetails:
-    """Data type for processed line details"""
-    line_content: Sequence[str]
-    geno_symbols: Sequence[str]
-    line_header: str
-    errors: Sequence[LineError]
-    is_markers_line: bool = False
-
-
-@dataclass(frozen=True)
-class UntransposedLineDetails:
-    """Data type for processed line details"""
-    line_content: Sequence[str]
-    geno_symbols: Sequence[str]
-    line_header: str
-    errors: Sequence[LineError]
-    marker: Optional[str] = None
-
-
-@dataclass(frozen=True)
-class FileStats:# pylint: disable=[too-many-instance-attributes]
-    """Custom file statistics."""
-    total_lines: int
-    data_lines: int
-    geno_symbols: Sequence[str]
-
-    comment_lines: int = 0
-    blank_lines: int = 0
-    dot_geno_lines: int = 0
-    dot_geno_fields: Optional[DotGenoFields] = None
-    samples_list: Sequence[str] = field(default_factory=tuple)
-    markers_list: Sequence[str] = field(default_factory=tuple)
-    errors: Sequence[LineError] = field(default_factory=tuple)