diff options
| author | pi coding agent <pi@localhost> | 2026-10-22 18:59:33 +0000 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-22 14:25:09 -0500 |
| commit | a897ceca02b78a9426652ac3092c79278cdc681c (patch) | |
| tree | 1ed499d3c583c93890d30864871740a276e4f718 /scripts | |
| parent | e3859b3945606aed7bd6a3861af8a525e061a2b1 (diff) | |
| download | gn-uploader-a897ceca02b78a9426652ac3092c79278cdc681c.tar.gz | |
Move scripts into the uploader package
Move the top-level `scripts` package to `uploader/scripts` so that it becomes the `uploader.scripts` package. All files are moved unchanged; references are updated in a follow-up commit.
Diffstat (limited to 'scripts')
30 files changed, 0 insertions, 4337 deletions
diff --git a/scripts/__init__.py b/scripts/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/scripts/__init__.py +++ /dev/null diff --git a/scripts/cli/__init__.py b/scripts/cli/__init__.py deleted file mode 100644 index 45bbda9..0000000 --- a/scripts/cli/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Package to hold CLI-specific utilities.""" - -from . import options diff --git a/scripts/cli/logging.py b/scripts/cli/logging.py deleted file mode 100644 index 30ecf17..0000000 --- a/scripts/cli/logging.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Logging for scripts.""" -import logging - -def setup_logging( - script_logger: logging.Logger, - loglevel: str, - modules: tuple[str, ...] = tuple() -): - """Setup module-level loggers to the same log-level as the application.""" - logging.basicConfig( - encoding="utf-8", - format=("%(asctime)s — %(filename)s:%(lineno)s — %(levelname)s: " - "%(message)s"), - level=logging.INFO) - script_logger.setLevel(getattr(logging, loglevel.upper())) - effective_loglevel = logging.getLevelName(script_logger.getEffectiveLevel()) - for module in modules: - logging.getLogger(module).setLevel(effective_loglevel) diff --git a/scripts/cli/options.py b/scripts/cli/options.py deleted file mode 100644 index 58d3df4..0000000 --- a/scripts/cli/options.py +++ /dev/null @@ -1,56 +0,0 @@ -"""General options to be added to CLI scripts.""" -from argparse import ArgumentParser - - -def add_logging(parser: ArgumentParser) -> ArgumentParser: - """Add optional log-level option""" - loglevels = ("debug", "info", "warning", "error", "critical") - parser.add_argument( - "--log_level", - "--log-level", - "--loglevel", - metavar="LOG-LEVEL", - type=str, - default="INFO", - choices=loglevels, - help=("Controls the severity of events to log. Valid values are: " + - ", ".join(f"'{level}'" for level in loglevels))) - return parser - - -def add_mariadb_uri(parser: ArgumentParser) -> ArgumentParser: - """Add the MySQL/MariaDB URI argument.""" - parser.add_argument("db_uri", - metavar="DB-URI", - type=str, - help="MariaDB/MySQL connection URL") - return parser - - -def add_species_id(parser: ArgumentParser) -> ArgumentParser: - """Add species-id as a mandatory argument.""" - parser.add_argument("species_id", - metavar="SPECIES-ID", - type=int, - help="The species to operate on.") - return parser - - -def add_population_id(parser: ArgumentParser) -> ArgumentParser: - """Add population-id as a mandatory argument.""" - parser = add_species_id(parser) - parser.add_argument("population_id", - metavar="POPULATION-ID", - type=int, - help="The ID for the population to operate on.") - return parser - - -def add_dataset_id(parser: ArgumentParser) -> ArgumentParser: - """Add dataset-id as a mandatory argument.""" - parser = add_population_id(parser) - parser.add_argument("dataset_id", - metavar="DATASET-ID", - type=int, - help="The ID for the dataset to operate on.") - return parser diff --git a/scripts/cli/validators.py b/scripts/cli/validators.py deleted file mode 100644 index dc198f5..0000000 --- a/scripts/cli/validators.py +++ /dev/null @@ -1,18 +0,0 @@ -"""CLI options validators.""" -from pathlib import Path - - -def file_exists(val: str) -> Path: - """Ensure the file path specified actually exists.""" - _file = Path(val) - if _file.exists() and _file.is_file(): - return _file - raise FileNotFoundError(f"Could not find the file '{_file}'.") - - -def directory_exists(val: str) -> Path: - """Check that directory path specified actually exists.""" - _dir = Path(val).absolute() - if _dir.is_dir() and _dir.exists(): - return _dir - raise FileNotFoundError(f"The path '{_dir}' MUST exist and be a directory.") diff --git a/scripts/cli_parser.py b/scripts/cli_parser.py deleted file mode 100644 index bf39731..0000000 --- a/scripts/cli_parser.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Common utilities for CLI parsers""" -from uuid import UUID -from typing import Optional -from argparse import ArgumentParser - - -def add_logging_option(parser: ArgumentParser) -> ArgumentParser: - """Add optional log-level option""" - parser.add_argument( - "--log-level", - "--loglevel", - type=str, - default="INFO", - choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", - "debug", "info", "warning", "error", "critical"], - help="The severity of events to track with the logger.") - return parser - - -def init_cli_parser(program: str, description: Optional[str] = None) -> ArgumentParser: - """Initialise the CLI arguments parser.""" - parser = ArgumentParser(prog=program, description=description) - - parser.add_argument( - "databaseuri", type=str, help="URI to connect to MariaDB") - parser.add_argument( - "redisuri", type=str, help="URI to connect to the redis server.") - parser.add_argument( - "redisprefix", type=str, help="A prefix to the job ID.") - parser.add_argument("jobid", type=UUID, help="Job ID that this belongs to") - parser.add_argument( - "--redisexpiry", - type=int, - default=86400, - help="How long to keep any redis keys around.") - return add_logging_option(parser) - - -def add_global_data_arguments(parser: ArgumentParser) -> ArgumentParser: - """Add the global (present in nearly ALL scripts) CLI arguments.""" - parser.add_argument("speciesid", - type=int, - help="Species to which bundle relates.") - parser.add_argument("populationid", - type=int, - help="Population to group data under") - return parser diff --git a/scripts/compute_phenotype_means.py b/scripts/compute_phenotype_means.py deleted file mode 100644 index 6d39ace..0000000 --- a/scripts/compute_phenotype_means.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Compute phenotype means.""" -import sys -import logging -from pathlib import Path -from typing import TypeVar -from argparse import Namespace, ArgumentParser - -import MySQLdb - -from gn_libs import mysqldb -from uploader import setup_modules_logging - -from .cli_parser import add_logging_option -from .load_phenotypes_to_db import update_means - -logger = logging.getLogger(__name__) -logging.basicConfig( - encoding="utf-8", - format="%(asctime)s - %(name)s - %(levelname)s — %(message)s", - level=logging.INFO) - - -def fetch_xref_id(conn: mysqldb.Connection, population_id: int) -> tuple[int, ...]: - """Fetch a population's cross-reference IDs.""" - logger.debug("Fetching the xref IDs.") - with conn.cursor(cursorclass=MySQLdb.cursors.DictCursor) as cursor: - query = "SELECT Id FROM PublishXRef WHERE InbredSetId=%(population_id)s" - cursor.execute(query, {"population_id": population_id}) - return tuple(int(row["Id"]) for row in cursor.fetchall()) - - -def run(args) -> int: - """Run the script.""" - logger.debug("Running the script!") - with mysqldb.database_connection(args.db_uri) as mariadb_conn: - xref_ids = args.cross_ref_ids or fetch_xref_id(mariadb_conn, args.population_id) - if len(xref_ids): - update_means(mariadb_conn, - args.population_id, - xref_ids) - logger.debug("Successfully computed means for %02d phenotypes.", - len(xref_ids)) - return 0 - _reasons = ( - f"no population exists with the ID {args.population_id}", - "the population exists but it has no phenotypes linked to it yet") - logger.error( - "No cross-reference IDs to run against. Likely causes are: %s", - " OR ".join(_reasons) + ".") - return 1 - - -T = TypeVar("T") -def comma_separated_list(val: str, itemstype: type = str) -> tuple[T, ...]: - """Convert val into a list of items of type 'itemstype'.""" - return tuple(itemstype(item.strip()) for item in val.split(",")) - - -def comma_separated_list_of_integers(val: str) -> tuple[int, ...]: - """Convert 'val' into list of items of type 'int'.""" - return comma_separated_list(val, int) - - -if __name__ == "__main__": - def parse_args() -> Namespace: - """Define and parse the CLI parsers accepted by this script.""" - parser = ArgumentParser( - "compute-phenotype-means", - description="Compute/Recompute the phenotype means.") - parser.add_argument("db_uri", - metavar="db-uri", - type=str, - help="MariaDB/MySQL connection URL") - parser.add_argument("jobs_db_path", - metavar="jobs-db-path", - type=Path, - help="Path to jobs' SQLite database.") - parser.add_argument("population_id", - metavar="population-id", - type=int, - help=("Identifier for the InbredSet group/" - "population to run means against.")) - ## Optional arguments - parser = add_logging_option(parser) - parser.add_argument( - "--cross-ref-ids", - type=comma_separated_list_of_integers, - help=("Provide cross-reference IDs to narrow the number of " - "phenotypes that the means are computed against."), - default=[]) - - return parser.parse_args() - - def main() -> int: - """compute-phenotype-means: Entry-point function.""" - args = parse_args() - logger.setLevel(getattr(logging, args.log_level.upper())) - setup_modules_logging(logger, ("scripts.load_phenotypes_to_db",)) - return run(args) - - sys.exit(main()) diff --git a/scripts/genotypes/__init__.py b/scripts/genotypes/__init__.py deleted file mode 100644 index 74fa7a5..0000000 --- a/scripts/genotypes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Scripts handling genotype-specific tasks.""" diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py deleted file mode 100644 index f8cdeed..0000000 --- a/scripts/genotypes/preprocess_csv_collect_info.py +++ /dev/null @@ -1,509 +0,0 @@ -"""Preprocess the genotypes CSV files: Collect basic info.""" -import os -import sys -import json -import logging -import multiprocessing as mproc -from pathlib import Path -from dataclasses import asdict, dataclass, field, fields -from typing import Union, Iterable, Iterator, Optional, Callable, Sequence -from argparse import Namespace, ArgumentParser - -from gn_libs.cli import SysExit - -from uploader.csvprocessing import (is_blank_line, - read_csv_file, - read_text_file, - read_csv_file_headers, - make_comment_line_checker) - -from scripts.cli.options import add_logging -from scripts.cli.logging import setup_logging -from scripts.cli.validators import file_exists - -from .types import (FileStats, - LineError, - DotGenoFields, - FieldsIdentity, - TransposedLineDetails, - UntransposedLineDetails) - - -logger = logging.getLogger(__name__) - -FILE_SIZE_THRESHOLD = 100 * 1024 * 1024 # 100MB -MULTIPROCESSING_CHUNK_SIZE = 10_000 # lines - - -def filecontents(path: Path) -> Iterator[str]: - """Return an iterator to the file contents""" - with path.open("r") as fl: - yield from fl.readlines() - - -def identify_columns( - headers: tuple[str, ...], - settings: dict[str, str] -) -> FieldsIdentity: - """Identify columns using settings provided by user""" - _id: dict[str, Union[str, str]] = { - "marker": "", - "chromosome": "", - "genetic_distance": "", - "physical_distance": "" - } - _samples: tuple[str, ...] = tuple() - for header in headers: - if header.lower() == settings["markers"].lower(): - _id["marker"] = header - continue - if header.lower() == settings["chromosome"].lower(): - _id["chromosome"] = header - continue - if header.lower() == settings["cm"].lower(): - # linkage map: genetic distance - _id["genetic_distance"] = header - continue - if header.lower() == settings["mb"].lower(): - # physical map: physical distance - _id["physical_distance"] = header - continue - - _samples = _samples + (header,) - - return FieldsIdentity(**_id, samples=_samples) - - -def process_transposed( - line: tuple[str, ...], - settings: dict, -) -> TransposedLineDetails: - """Process a line in a transposed file.""" - return TransposedLineDetails( - line_content=line, - geno_symbols=tuple(set( - line[1:] if line[0] not in ( - settings["markers"], - settings["chromosome"], - settings["cm"], - settings["mb"]) - else tuple())), - line_header=line[0], - errors=tuple(), - is_markers_line=line[0]==settings["markers"]) - - -def process_untransposed( - line: tuple[str, ...], - settings: dict, - headers: tuple[str, ...] -) -> UntransposedLineDetails: - """Processed line in a non-transposed file.""" - _marker: Optional[str] = None - _errors: tuple[LineError, ...] = tuple() - _symbols = tuple(_symb for _head, _symb in zip(headers, line) - if _head not in ( - settings["markers"], - settings["chromosome"], - settings["cm"], - settings["mb"])) - - try: - _idx = headers.index(settings["markers"]) - _marker = line[_idx] - except ValueError as _verr: - _idx = 0 - - if len(headers) != len(line): - _errors = _errors + (LineError( - row_header=line[_idx], - column_header="-", - message=( - f"Columns mismatch: There are {len(headers)} headers but only " - f"{len(line)} field values in the line."), - line=line - ),) - - return UntransposedLineDetails( - line_content=line, - line_header=line[0], - geno_symbols=tuple(set(_symbols)), - errors=_errors, - marker=_marker) - - -def validate_headers( - headers: tuple[str, ...], - settings: dict -) -> tuple[LineError, ...]: - """Validate that the headers are correct.""" - _errors: tuple[LineError, ...] = tuple() - _header_descriptions = { - "markers": "marker", - "chromosome": "chromosome", - "cm": "genetic distance (cM)", - "mb": "physical location (Mb)"} - for _header, _desc in _header_descriptions.items(): - try: - headers.index(settings[_header]) - except ValueError as _verr: - _errors = _errors + (LineError( - row_header=headers[0], - column_header="-", - message=( - f"Header row does not have a '{_desc}' column named " - f"'{settings[_header]}'."), - line=headers),) - - return _errors - - -def is_dot_geno_line(line: str) -> bool: - """Check whether the line is a custom .geno format line - - :param line: The line to check for custom formatting - :type line: str - :rtype bool - """ - return line.startswith("@") - -def _make_line_splitter(separator: str) -> Callable[[str], tuple[str, ...]]: - """Create a function to split a line into its fields.""" - def _line_splitter(line: str) -> tuple[str, ...]: - return tuple(field.strip() for field in line.split(separator)) - return _line_splitter - - -def _make_is_skippable_line( - fileconfigs: dict, - headers: tuple[str, ...] -) -> Callable[[str], bool]: - """Create a function to check whether a line is skippable.""" - is_comment_line = make_comment_line_checker(fileconfigs["comment.char"]) - line_splitter = _make_line_splitter(fileconfigs["separator"]) - - def _is_skippable_line(line: str) -> bool: - return (is_blank_line(line) - or is_comment_line(line) - or is_dot_geno_line(line) - or line_splitter(line) == headers) - - return _is_skippable_line - - -@dataclass -class _ChunkStats:# pylint: disable=[too-many-instance-attributes] - """Statistics accumulated from a chunk of lines.""" - total_lines: int = 0 - data_lines: int = 0 - comment_lines: int = 0 - blank_lines: int = 0 - dot_geno_lines: int = 0 - dot_geno_fields: dict = field(default_factory=dict) - geno_symbols: set = field(default_factory=set) - markers: tuple = field(default_factory=tuple) - markers_seen: bool = False - errors: tuple = field(default_factory=tuple) - - -def _accumulate( - headers: tuple[str, ...], - fileconfigs: dict, - lines: Iterable[str] -) -> _ChunkStats: - """Accumulate statistics from the provided lines.""" - is_skippable_line = _make_is_skippable_line(fileconfigs, headers) - line_splitter = _make_line_splitter(fileconfigs["separator"]) - stats = _ChunkStats() - for line in (ln.strip() for ln in lines): - stats.total_lines += 1 - if not is_skippable_line(line): - stats.data_lines += 1 - fields_values = line_splitter(line) - if fileconfigs["transposed"]: - t_line_dets = process_transposed(fields_values, fileconfigs) - stats.geno_symbols.update(t_line_dets.geno_symbols) - stats.errors = stats.errors + tuple(t_line_dets.errors) - if t_line_dets.is_markers_line: - stats.markers = tuple(t_line_dets.line_content[1:]) - stats.markers_seen = True - else: - line_dets = process_untransposed( - fields_values, - fileconfigs, - headers=headers) - stats.geno_symbols.update(line_dets.geno_symbols) - stats.errors = stats.errors + tuple(line_dets.errors) - if bool(line_dets.marker): - stats.markers = stats.markers + (str(line_dets.marker),) - # TOD0: do more processing - continue # end of line processing - if line.startswith("@"): #handle special geno formats - stats.dot_geno_lines += 1 - stats.dot_geno_fields = { - **stats.dot_geno_fields, - **parse_dot_geno_line(line)} - if line.startswith("#"): #handle special geno formats - stats.comment_lines += 1 - if line == "": # Blank line - stats.blank_lines += 1 - return stats - - -def _merge_chunk_stats( - left: _ChunkStats, - right: _ChunkStats, - transposed: bool -) -> _ChunkStats: - """Merge two sets of chunk statistics, preserving file order. - - The two sets of statistics are expected to have been accumulated from - consecutive chunks of the same file, with ``left`` covering the earlier - chunk. - """ - if transposed: - # The markers come from the last markers line encountered. Keep the - # earlier markers when the later chunk did not see a markers line. - markers = right.markers if right.markers_seen else left.markers - else: - markers = left.markers + right.markers - return _ChunkStats( - total_lines=left.total_lines + right.total_lines, - data_lines=left.data_lines + right.data_lines, - comment_lines=left.comment_lines + right.comment_lines, - blank_lines=left.blank_lines + right.blank_lines, - dot_geno_lines=left.dot_geno_lines + right.dot_geno_lines, - dot_geno_fields={**left.dot_geno_fields, **right.dot_geno_fields}, - geno_symbols=left.geno_symbols | right.geno_symbols, - markers=markers, - markers_seen=left.markers_seen or right.markers_seen, - errors=left.errors + right.errors) - - -def _merge_all_chunk_stats( - chunks: Iterable[_ChunkStats], - transposed: bool -) -> _ChunkStats: - """Merge a sequence of chunk statistics into a single set of statistics.""" - merged = _ChunkStats() - for chunk_stats in chunks: - merged = _merge_chunk_stats(merged, chunk_stats, transposed) - return merged - - -def _read_chunks( - lines: Iterable[str], - chunk_size: int -) -> Iterator[list[str]]: - """Group the lines into chunks of at most ``chunk_size`` lines.""" - chunk: list[str] = [] - for line in lines: - chunk.append(line) - if len(chunk) >= chunk_size: - yield chunk - chunk = [] - if chunk: - yield chunk - - -def _process_chunk(args: tuple[tuple[str, ...], dict, list[str]]) -> _ChunkStats: - """Process a single chunk of lines. - - This is a module-level function so that it can be pickled and run in a - separate process. - """ - headers, fileconfigs, chunk = args - return _accumulate(headers, fileconfigs, chunk) - - -def file_stats_multiprocessing( - fileconfigs: dict, - headers: tuple[str, ...], - lines: Iterable[str], - chunk_size: int = MULTIPROCESSING_CHUNK_SIZE -) -> FileStats: - """File stats from the lines in the file using multiple processes.""" - with mproc.Pool(max(1, mproc.cpu_count() - 1)) as pool: - chunks = ((headers, fileconfigs, chunk) - for chunk in _read_chunks(lines, chunk_size)) - merged = _merge_all_chunk_stats( - pool.imap(_process_chunk, chunks), - fileconfigs["transposed"]) - return _as_file_stats( - merged, - identify_columns(headers, fileconfigs).samples, - validate_headers(headers, fileconfigs)) - - -def _as_file_stats( - stats: _ChunkStats, - samples: Sequence[str], - header_errors: tuple[LineError, ...] -) -> FileStats: - """Convert accumulated statistics into file statistics.""" - return FileStats( - total_lines=stats.total_lines, - data_lines=stats.data_lines, - comment_lines=stats.comment_lines, - blank_lines=stats.blank_lines, - dot_geno_lines=stats.dot_geno_lines, - dot_geno_fields=(DotGenoFields(**stats.dot_geno_fields) - if bool(stats.dot_geno_fields) else None), - geno_symbols=tuple(sorted(stats.geno_symbols)), - samples_list=tuple(samples), - markers_list=stats.markers, - errors=tuple(header_errors) + tuple(stats.errors)) - - -def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats: - """Compute file statistics from the lines in the file.""" - is_comment_line = make_comment_line_checker(fileconfigs["comment.char"]) - headers = read_csv_file_headers(filepath, - fileconfigs["transposed"], - fileconfigs["separator"], - lambda line: (is_blank_line(line) - or is_comment_line(line) - or is_dot_geno_line(line))) - logger.debug("THE FILE HEADERS: %s", headers) - basic_stats = os.stat(filepath) - lines = read_text_file(filepath) - return (file_stats_multiprocessing(fileconfigs, headers, lines) - if basic_stats.st_size >= FILE_SIZE_THRESHOLD - else file_stats_singlethread(fileconfigs, headers, lines)) - - -def parse_dot_geno_line(line) -> dict[str, str]: - """Parse the dot_geno format line.""" - assert line.startswith("@") - acceptable_fields = tuple(field.name for field in fields(DotGenoFields)) - parts = tuple(item.strip() for item in line[1:].split(":")) - if len(parts) == 2 and parts[0] in acceptable_fields: - return {parts[0]: parts[1]} - return {} - - -def file_stats_singlethread( - fileconfigs: dict, - headers: tuple[str, ...], - lines: Iterable[str] -) -> FileStats: - """File stats from the lines in the file using a single-threaded process.""" - return _as_file_stats( - _accumulate(headers, fileconfigs, lines), - identify_columns(headers, fileconfigs).samples, - validate_headers(headers, fileconfigs)) - - -if __name__ == "__main__": - - def parse_args() -> Namespace: - """Parse the command-line arguments.""" - parser = add_logging( - ArgumentParser( - prog="preprocess-csv-collect-info", - description=( - "Pre-process the CSV file, with a focus on collecting " - "basic file information and statistics."))) - - parser.add_argument( - "csv_file", - metavar="CSV-FILE", - type=file_exists, - help="The CSV file to process.") - - ## format-specific arguments - parser.add_argument( - "--is-transposed", - action="store_true", - default=False, - help="Are the rows and columns in the CSV file flipped?") - - parser.add_argument( - "--separator", - metavar="FIELDS-SEPARATOR", - type=str, - default=",", - help=( - "The character acting as the fields' separator in the CSV file.")) - - parser.add_argument( - "--comment_char", - metavar="COMMENTS-CHARACTER", - type=str, - default="#", - help="The character indicating a comment line.") - - parser.add_argument( - "--na_strings", - metavar="MISSING-VALUES-STRING", - action="append", - type=str, - default=["-", "NA", "N/A"], - help="The character indicating a comment line.") - - ## content-specific arguments - parser.add_argument( - "--markers_field", - metavar="MARKERS-FIELD", - type=str, - default="Locus", - help=( - "The name of the column (or row, when transposed) containing " - "the names of the genetic markers.")) - - parser.add_argument( - "--chr_field", - metavar="CHROMOSOMES-FIELD", - type=str, - default="Chr", - help=( - "The name of the column (or row, when transposed) containing " - "the names of the names of the chromosomes.")) - - parser.add_argument( - "--cm_field", - metavar="GENETIC-DISTANCES-FIELD", - type=str, - default="cM", - help=( - "The name of the column (or row, when transposed) containing " - "the genetic distance values in centimorgans.")) - - parser.add_argument( - "--mb_field", - metavar="PHYSICAL-DISTANCES-FIELD", - type=str, - default="Mb", - help=( - "The name of the column (or row, when transposed) containing " - "the physical distance values in megabase pairs.")) - - return parser.parse_args() - - - def main() -> SysExit: - """Entry-point to the pre-processor.""" - try: - args = parse_args() - setup_logging(logger, args.log_level.upper(), tuple()) - logger.debug("CLI Arguments: %s", args) - - stats = compute_statistics( - args.csv_file, - { - "markers": args.markers_field, - "chromosome": args.chr_field, - "cm": args.cm_field, - "mb": args.mb_field, - "separator": args.separator, - "comment.char": args.comment_char, - "na_strings": args.na_strings, - "transposed": args.is_transposed - }) - print(json.dumps(asdict(stats))) - return SysExit.OK - except FileNotFoundError as _fnf: - logger.error(", ".join(str(arg) for arg in _fnf.args), exc_info=True) - return SysExit.FILENOTFOUND - - sys.exit(main().value) diff --git a/scripts/genotypes/types.py b/scripts/genotypes/types.py deleted file mode 100644 index d765fe5..0000000 --- a/scripts/genotypes/types.py +++ /dev/null @@ -1,69 +0,0 @@ -"""The datatypes this package will use.""" -from typing import Sequence, Optional -from dataclasses import field, dataclass - - -@dataclass -class DotGenoFields:# Still necessary?? - """Custom .geno file format values""" - name: str - mat: str - pat: str - het: str - type: Optional[str] = None - unk: Optional[str] = None - - -@dataclass(frozen=True) -class FieldsIdentity: - """Data type for identity of the fields.""" - marker: str - chromosome: str - genetic_distance: str # cM - physical_distance: str # Mb - samples: Sequence[str] - - -@dataclass -class LineError: - """An error in a specific line of the genotype file.""" - row_header: str - column_header: str - message: str - line: Sequence[str] - - -@dataclass(frozen=True) -class TransposedLineDetails: - """Data type for processed line details""" - line_content: Sequence[str] - geno_symbols: Sequence[str] - line_header: str - errors: Sequence[LineError] - is_markers_line: bool = False - - -@dataclass(frozen=True) -class UntransposedLineDetails: - """Data type for processed line details""" - line_content: Sequence[str] - geno_symbols: Sequence[str] - line_header: str - errors: Sequence[LineError] - marker: Optional[str] = None - - -@dataclass(frozen=True) -class FileStats:# pylint: disable=[too-many-instance-attributes] - """Custom file statistics.""" - total_lines: int - data_lines: int - geno_symbols: Sequence[str] - - comment_lines: int = 0 - blank_lines: int = 0 - dot_geno_lines: int = 0 - dot_geno_fields: Optional[DotGenoFields] = None - samples_list: Sequence[str] = field(default_factory=tuple) - markers_list: Sequence[str] = field(default_factory=tuple) - errors: Sequence[LineError] = field(default_factory=tuple) diff --git a/scripts/insert_data.py b/scripts/insert_data.py deleted file mode 100644 index aec0251..0000000 --- a/scripts/insert_data.py +++ /dev/null @@ -1,368 +0,0 @@ -"""Insert means/averages or standard-error data into the database.""" -import sys -import string -import random -import logging -import argparse -from functools import reduce -from typing import Tuple, Iterator - -import MySQLdb as mdb -from redis import Redis -from MySQLdb.cursors import DictCursor -from gn_libs.mysqldb import database_connection - -from functional_tools import take - -from quality_control.file_utils import open_file -from uploader.check_connections import check_db, check_redis - -# Set up logging -stderr_handler = logging.StreamHandler(stream=sys.stderr) -root_logger = logging.getLogger() -root_logger.addHandler(stderr_handler) -root_logger.setLevel("WARNING") - -def random_string(count: int = 10) -> str: - """Generate a random, alphanumeric string.""" - return "".join(random.choices( - string.digits + string.ascii_letters, k=count)) - -def translate_alias(heading): - "Translate strain aliases into canonical names" - translations = {"B6": "C57BL/6J", "D2": "DBA/2J"} - return translations.get(heading, heading) - -def read_file_headings(filepath) -> Tuple[str, ...]: - "Get the file headings" - with open_file(filepath) as input_file: - for line_contents in input_file: - headings = tuple( - translate_alias(heading.strip()) - for heading in line_contents.split("\t")) - break - - return headings - -def read_file_contents(filepath): - "Get the file contents" - with open_file(filepath) as input_file: - for line_number, line_contents in enumerate(input_file): - if line_number == 0: - continue - if line_number > 0: - yield tuple( - field.strip() for field in line_contents.split("\t")) - -def strains_info( - dbconn: mdb.Connection, strain_names: Tuple[str, ...], - speciesid: int) -> dict: - "Retrieve information for the strains" - with dbconn.cursor(cursorclass=DictCursor) as cursor: - query = ( - "SELECT * FROM Strain WHERE Name IN " - f"({', '.join(['%s']*len(strain_names))}) " - "AND SpeciesId = %s") - cursor.execute(query, tuple(strain_names) + (speciesid,)) - return {strain["Name"]: strain for strain in cursor.fetchall()} - -def read_datavalues(filepath, headings, strain_info): - """Read numerical, data values from the file.""" - id_key = headings[0] - return { - str(row[id_key]): tuple({ - "ProbeSetName": str(row[id_key]), - "StrainId": strain_info[sname]["Id"], - "DataValue": float(row[sname]) - } for sname in headings[1:]) - for row in - (dict(zip(headings, line)) for line in read_file_contents(filepath)) - } - -def read_probesets(filepath, headings): - """Read the ProbeSet names.""" - id_key = headings[0] - for row in (dict(zip(headings, line)) - for line in read_file_contents(filepath)): - yield {"Name": str(row[id_key])} - -def last_data_id(dbconn: mdb.Connection) -> int: - "Get the last id from the database" - with dbconn.cursor() as cursor: - cursor.execute("SELECT MAX(Id) FROM ProbeSetData") - return int(cursor.fetchone()[0]) - -def check_strains(headings_strains, db_strains): - "Check strains in headings exist in database" - from_db = tuple(db_strains.keys()) - not_in_db = tuple( - strain for strain in headings_strains if strain not in from_db) - if len(not_in_db) == 0: - return True - - str_not_in_db = "', '".join(not_in_db) - print( - (f"ERROR: The strain(s) '{str_not_in_db}' w(as|ere) not found in the " - "database."), - file=sys.stderr) - sys.exit(1) - - -def annotationinfo( - dbconn: mdb.Connection, - platformid: int, - datasetid: int -) -> dict[str, dict]: - "Get annotation information from the database." - # This is somewhat slow. Look into optimising the behaviour - def __organise_annotations__(accm, item): - names_dict = ( - {**accm[0], item["Name"]: item} if bool(item["Name"]) else accm[0]) - targs_dict = ( - {**accm[1], item["TargetId"]: item} - if bool(item["TargetId"]) else accm[1]) - return (names_dict, targs_dict) - - query = ( - "SELECT ProbeSet.Name, ProbeSet.ChipId, ProbeSet.TargetId, " - "ProbeSetXRef.DataId, ProbeSetXRef.ProbeSetFreezeId " - "FROM ProbeSet INNER JOIN ProbeSetXRef " - "ON ProbeSet.Id=ProbeSetXRef.ProbeSetId " - "WHERE ProbeSet.ChipId=%s AND ProbeSetXRef.ProbeSetFreezeId=%s") - with dbconn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute(query, (platformid, datasetid)) - annot_dicts = reduce(# type: ignore[var-annotated] - __organise_annotations__, cursor.fetchall(), ({}, {})) - return {**annot_dicts[0], **annot_dicts[1]} - return {} - -def __format_query__(query, params): - "Format the query for output" - def __param_str__(param): - return "', '".join(str(elt) for elt in param) - idx = query.find("VALUES") - idx = query.find("%") - fields = tuple( - elt.replace("%(", "").replace(")s", "").replace(")", "").strip() - for elt in query[idx:-1].split(",")) - values = (tuple(param[field] for field in fields) for param in params) - values_str = ", ".join( - f"('{__param_str__(value_tup)}')" for value_tup in values) - insert_str = query[:idx].replace( - "INSERT INTO ", "INSERT INTO\n\t") - return f"{insert_str}\nVALUES\n\t{values_str};" - -def insert_probesets(filepath: str, - dbconn: mdb.Connection, - platform_id: int, - headings: tuple[str, ...], - session_rand_str: str) -> tuple[str, ...]: - """Save new ProbeSets into the database.""" - probeset_query = ( - "INSERT INTO ProbeSet(ChipId, Name) " - "VALUES (%(ChipId)s, %(Name)s) ") - the_probesets = ({ - **row, - "Name": f"{row['Name']}{session_rand_str}", - "ChipId": platform_id - } for row in read_probesets(filepath, headings)) - probeset_names: tuple[str, ...] = tuple() - with dbconn.cursor(cursorclass=DictCursor) as cursor: - while True: - probeset_params = tuple(take(the_probesets, 10000)) - if not bool(probeset_params): - break - print(__format_query__(probeset_query, probeset_params)) - print() - cursor.executemany(probeset_query, probeset_params) - probeset_names = probeset_names + tuple( - row["Name"] for row in probeset_params) - - return probeset_names - -def probeset_ids(dbconn: mdb.Connection, - chip_id: int, - probeset_names: tuple[str, ...]) -> Iterator[tuple[str, int]]: - """Fetch the IDs of the probesets with the given names.""" - with dbconn.cursor() as cursor: - params_str = ", ".join(["%s"] * len(probeset_names)) - cursor.execute( - "SELECT Name, Id FROM ProbeSet " - "WHERE ChipId=%s " - f"AND Name IN ({params_str})", - (chip_id,) + probeset_names) - while True: - row = cursor.fetchone() - if not bool(row): - break - yield row - -def insert_means(# pylint: disable=[too-many-locals, too-many-arguments, too-many-positional-arguments] - filepath: str, speciesid: int, platform_id: int, datasetid: int, - dbconn: mdb.Connection, rconn: Redis) -> int: # pylint: disable=[unused-argument] - "Insert the means/averages data into the database" - headings = read_file_headings(filepath) - strains = strains_info(dbconn, headings[1:], speciesid) - check_strains(headings[1:], strains) - means_query = ( - "INSERT INTO ProbeSetData " - "VALUES(%(ProbeSetDataId)s, %(StrainId)s, %(DataValue)s)") - xref_query = ( - "INSERT INTO ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) " - "VALUES(%(ProbeSetFreezeId)s, %(ProbeSetId)s, %(ProbeSetDataId)s)") - - # A random string to avoid over-write chances. - # This is needed because the `ProbeSet` table is defined with - # UNIQUE KEY `ProbeSetId` (`ChipId`,`Name`) - # which means that we cannot have 2 (or more) ProbeSets which share both - # the name and chip_id (platform) at the same time. - rand_str = f"::RAND_{random_string()}" - pset_ids = { - name[0:name.index("::RAND_")]: pset_id - for name, pset_id in probeset_ids( - dbconn, - platform_id, - insert_probesets( - filepath, dbconn, platform_id, headings, rand_str)) - } - the_means = ({ - **mean, "ProbeSetFreezeId": datasetid, "ProbeSetDataId": data_id, - "ChipId": platform_id, "ProbeSetId": pset_ids[mean["ProbeSetName"]] - } for data_id, mean in enumerate(( - item for sublist in - read_datavalues(filepath, headings, strains).values() - for item in sublist), - start=last_data_id(dbconn)+1)) - with dbconn.cursor(cursorclass=DictCursor) as cursor: - while True: - means = tuple(take(the_means, 10000)) - if not bool(means): - break - print(__format_query__(means_query, means)) - print() - print(__format_query__(xref_query, means)) - cursor.executemany(means_query, means) - cursor.executemany(xref_query, means) - return 0 - -def insert_se(# pylint: disable = [too-many-arguments,too-many-locals, too-many-positional-arguments] - filepath: str, speciesid: int, platformid: int, datasetid: int, - dbconn: mdb.Connection, rconn: Redis) -> int: # pylint: disable=[unused-argument] - "Insert the standard-error data into the database" - headings = read_file_headings(filepath) - strains = strains_info(dbconn, headings[1:], speciesid) - check_strains(headings[1:], strains) - se_query = ( - "INSERT INTO ProbeSetSE " - "VALUES(%(DataId)s, %(StrainId)s, %(DataValue)s)") - annotations = annotationinfo(dbconn, platformid, datasetid) - if not bool(annotations): - with dbconn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - ("SELECT " - "gc.GeneChipName AS platformname, pf.Name AS studyname, " - "psf.FullName AS datasetname " - "FROM GeneChip AS gc INNER JOIN ProbeFreeze AS pf " - "ON gc.Id=pf.ChipId INNER JOIN ProbeSetFreeze AS psf " - "ON pf.Id=psf.ProbeFreezeId " - "WHERE gc.Id=%s AND psf.Id=%s"), - (platformid, datasetid)) - - errorinfo = cursor.fetchone() - print(("ERROR: No annotations found for the " - f"'{errorinfo['datasetname']}' dataset (Id: {datasetid}) " - f"under the '{errorinfo['studyname']}' study linked to the " - f"'{errorinfo['platformname']}' platform (Id: {platformid})." - "\n\n" - " Please verify you selected the correct platform, " - "study and dataset for the standard-error file(s) you were " - "trying to upload."), - file=sys.stderr) - return 1 - - namemappings = { - _key[0:_key.find("::RAND_")]: _key for _key in annotations.keys() - } - se_values = ( - {"DataId": annotations[namemappings[str(item["ProbeSetName"])]]["DataId"], **item} - for item in ( - row for psrows in - read_datavalues(filepath, headings, strains).values() - for row in psrows)) - with dbconn.cursor(cursorclass=DictCursor) as cursor: - try: - while True: - serrors = tuple(take(se_values, 1000)) - if not bool(serrors): - break - print(__format_query__(se_query, serrors)) - cursor.executemany(se_query, serrors) - except KeyError as kerr: - print( - ( - f"The following name(s) or identifier(s) do(es) not " - "exist in the database and did not exist in the original " - f"values file: {', '.join(kerr.args)}"), - file=sys.stderr) - return 1 - return 0 - -if __name__ == "__main__": - def cli_args(): - "Compute the CLI arguments" - parser = argparse.ArgumentParser( - prog="InsertData", description=( - "Script to insert data from an 'averages' file into the " - "database.")) - parser.add_argument( - "filetype", help="type of data to insert.", - choices=("average", "standard-error")) - parser.add_argument( - "filepath", help="path to the file with the 'averages' data.") - parser.add_argument( - "speciesid", help="Identifier for the species in the database.", - type=int) - parser.add_argument( - "platformid", help="Identifier for the platform in the database.", - type=int) - parser.add_argument( - "datasetid", help="Identifier for the dataset in the database.", - type=int) - parser.add_argument( - "database_uri", - help="URL to be used to initialise the connection to the database") - parser.add_argument( - "redisuri", - help="URL to initialise connection to redis", - default="redis:///") - - args = parser.parse_args() - check_db(args.database_uri) - check_redis(args.redisuri) - return args - - insert_fns = { - "average": insert_means, - "standard-error": insert_se - } - - extract_args = { - "average": lambda args, dbconn, rconn: ( - args.filepath, args.speciesid, args.platformid, args.datasetid, dbconn, - rconn), - "standard-error": lambda args, dbconn, rconn: ( - args.filepath, args.speciesid, args.platformid, args.datasetid, - dbconn, rconn), - } - - def main(): - "Main entry point" - args = cli_args() - with Redis.from_url(args.redisuri, decode_responses=True) as rconn: - with database_connection(args.database_uri) as dbconn: - return insert_fns[args.filetype]( - *extract_args[args.filetype](args, dbconn, rconn)) - - return 2 - - sys.exit(main()) diff --git a/scripts/insert_samples.py b/scripts/insert_samples.py deleted file mode 100644 index 96ae8e2..0000000 --- a/scripts/insert_samples.py +++ /dev/null @@ -1,147 +0,0 @@ -"""Insert samples into the database.""" -import sys -import logging -import pathlib -import argparse -import traceback - -import MySQLdb as mdb - -from gn_libs.mysqldb import database_connection - -from uploader.check_connections import check_db -from uploader.species.models import species_by_id -from uploader.population.models import population_by_id -from uploader.samples.models import ( - save_samples_data, - read_samples_file, - cross_reference_samples) - -stderr_handler = logging.StreamHandler(stream=sys.stderr) -root_logger = logging.getLogger() -root_logger.addHandler(stderr_handler) -root_logger.setLevel("INFO") - -class SeparatorAction(argparse.Action): - """Action to handle the separator values.""" - def __init__(self, option_strings, dest, nargs=None, **kwargs): - """Init the action""" - if nargs is not None: - raise ValueError("nargs not allowed.") - super().__init__(option_strings, dest, nargs, **kwargs) - - def __call__(self, parser, namespace, values, option_string=None): - """Process the value passed in.""" - setattr(namespace, self.dest, (chr(9) if values == "\\t" else values)) - -def insert_samples(conn: mdb.Connection,# pylint: disable=[too-many-arguments, too-many-positional-arguments] - speciesid: int, - populationid: int, - samplesfile: pathlib.Path, - separator: str, - firstlineheading: bool, - quotechar: str): - """Insert the samples into the database.""" - print("Checking for errors:") - species = species_by_id(conn, speciesid) - if not bool(species): - logging.error("Species with id '%s' does not exist.", str(speciesid)) - return 1 - print(f"\tSpecies with ID '{speciesid}' found") - population = population_by_id(conn, populationid) - if not bool(population): - logging.error("Population with id '%s' does not exist.", - str(populationid)) - return 1 - print(f"\tPopulations with ID '{populationid}' found") - print("No errors found. Continuing...") - print("\nInserting samples ...") - save_samples_data( - conn, - speciesid, - read_samples_file(samplesfile, separator, firstlineheading)) - print("Cross-referencing samples with their populations.") - cross_reference_samples( - conn, - speciesid, - populationid, - (row["Name"] for row in - read_samples_file(samplesfile, - separator, - firstlineheading, - quotechar=quotechar))) - print("Samples upload successfully completed.") - return 0 - - -if __name__ == "__main__": - - def cli_args(): - """Process the command-line arguments.""" - # - parser = argparse.ArgumentParser( - prog="insert_samples", - description = ( - "Script to parse and insert sample data from a file into the " - "database.")) - - # == Mandatory Arguments == - parser.add_argument( - "databaseuri", - help="URL to be used to initialise the connection to the database") - parser.add_argument("speciesid", - type=int, - help="The species identifier in the database.") - parser.add_argument( - "populationid", - type=int, - help="The grouping/population identifier in the database.") - parser.add_argument( - "samplesfile", - type=pathlib.Path, - help="Path to the CSV file containing the samples data.") - parser.add_argument( - "separator", - action=SeparatorAction, - help="The 'character' in the CSV file that separates the fields.", - default=chr(9)) - - # == Optional Arguments == - parser.add_argument( - "--firstlineheading", - action="store_true", - help=("If the first line of the file is a header row, invoke the " - "program with this flag.")) - parser.add_argument( - "--quotechar", - default='"', - help=("The character used to delimit (surround?) the value in " - "each column.")) - - args = parser.parse_args() - return args - - def main(): - """Run script to insert samples into the database.""" - status_code = 1 # Exit with an Exception - args = cli_args() - check_db(args.databaseuri) - if not args.samplesfile.exists(): - logging.error("File not found: '%s'.", args.samplesfile) - return 2 - - with database_connection(args.databaseuri) as dbconn: - try: - status_code = insert_samples(dbconn, - args.speciesid, - args.populationid, - args.samplesfile, - args.separator, - args.firstlineheading, - args.quotechar) - except Exception as _exc:# pylint: disable=[broad-exception-caught] - print(traceback.format_exc(), file=sys.stderr) - - return status_code - - sys.exit(main()) diff --git a/scripts/load_phenotypes_to_db.py b/scripts/load_phenotypes_to_db.py deleted file mode 100644 index addda99..0000000 --- a/scripts/load_phenotypes_to_db.py +++ /dev/null @@ -1,560 +0,0 @@ -"""Load phenotypes and their data provided in files into the database.""" -import sys -import uuid -import json -import time -import logging -import argparse -from pathlib import Path -from zipfile import ZipFile -from datetime import datetime -from typing import Any, Iterable -from urllib.parse import urljoin -from functools import reduce, partial - -from MySQLdb.cursors import DictCursor - -from gn_libs import jobs, mysqldb, sqlite3, monadic_requests as mrequests - -from r_qtl import r_qtl2 as rqtl2 -from uploader.species.models import species_by_id -from uploader.population.models import population_by_species_and_id -from uploader.csvprocessing import read_csv_file, read_csv_file_headers -from uploader.samples.models import samples_by_species_and_population -from uploader.phenotypes.models import ( - dataset_by_id, - save_phenotypes_data, - create_new_phenotypes, - quick_save_phenotypes_data) -from uploader.publications.models import fetch_publication_by_id - -from scripts.rqtl2.bundleutils import build_line_joiner, build_line_splitter - -from functional_tools import take - -logging.basicConfig( - format="%(asctime)s — %(filename)s:%(lineno)s — %(levelname)s: %(message)s") -logger = logging.getLogger(__name__) - - - -def __replace_na_strings__(line, na_strings): - return ((None if value in na_strings else value) for value in line) - - -def save_phenotypes( - conn: mysqldb.Connection, - control_data: dict[str, Any], - population_id, - publication_id, - filesdir: Path -) -> tuple[dict, ...]: - """Read `phenofiles` and save the phenotypes therein.""" - phenofiles = tuple(filesdir.joinpath(_file) for _file in control_data["phenocovar"]) - if len(phenofiles) <= 0: - return tuple() - - def _is_skippable_line_(line): - return line.startswith(control_data["comment.char"]) - - if control_data["phenocovar_transposed"]: - logger.info("Undoing transposition of the files rows and columns.") - phenofiles = tuple( - rqtl2.transpose_csv_with_rename( - _file, - build_line_splitter(control_data), - build_line_joiner(control_data)) - for _file in phenofiles) - - _headers = read_csv_file_headers(phenofiles[0], - control_data["phenocovar_transposed"], - control_data["sep"], - _is_skippable_line_) - return create_new_phenotypes( - conn, - population_id, - publication_id, - (dict(zip(_headers, - __replace_na_strings__(line, control_data["na.strings"]))) - for filecontent - in (read_csv_file(path, - separator=control_data["sep"], - is_skippable_line=_is_skippable_line_) - for path in phenofiles) - for idx, line in enumerate(filecontent) - if idx != 0)) - - -def __row_to_dataitems__( - sample_row: dict, - dataidmap: dict, - pheno_name2id: dict[str, int], - samples: dict -) -> Iterable[dict]: - samplename = sample_row["id"] - - return ({ - "phenotype_id": dataidmap[pheno_name2id[phenoname]]["phenotype_id"], - "data_id": dataidmap[pheno_name2id[phenoname]]["data_id"], - "sample_name": samplename, - "sample_id": samples[samplename]["Id"], - "value": phenovalue - } for phenoname, phenovalue in sample_row.items() if phenoname != "id") - - -def __build_dataitems__( - phenofiles, - control_data, - samples, - dataidmap, - pheno_name2id -): - def _is_skippable_line_(line): - return line.startswith(control_data["comment.char"]) - - _headers = read_csv_file_headers( - phenofiles[0], - False, # Any transposed files have been un-transposed by this point - control_data["sep"], - _is_skippable_line_) - _filescontents = ( - read_csv_file(path, - separator=control_data["sep"], - is_skippable_line=_is_skippable_line_) - for path in phenofiles) - _linescontents = ( - __row_to_dataitems__( - dict(zip(("id",) + _headers[1:], - __replace_na_strings__(line, control_data["na.strings"]))), - dataidmap, - pheno_name2id, - samples) - for linenum, line in (enumline for filecontent in _filescontents - for enumline in enumerate(filecontent)) - if linenum > 0) - return (item for items in _linescontents - for item in items - if item["value"] is not None) - - -def save_numeric_data(# pylint: disable=[too-many-positional-arguments,too-many-arguments] - conn: mysqldb.Connection, - dataidmap: dict, - pheno_name2id: dict[str, int], - samples: dict, - control_data: dict, - filesdir: Path, - filetype: str, - table: str -): - """Read data from files and save to the database.""" - phenofiles = tuple( - filesdir.joinpath(_file) for _file in control_data[filetype]) - if len(phenofiles) <= 0: - return tuple() - - if control_data[f"{filetype}_transposed"]: - logger.info("Undoing transposition of the files rows and columns.") - phenofiles = tuple( - rqtl2.transpose_csv_with_rename( - _file, - build_line_splitter(control_data), - build_line_joiner(control_data)) - for _file in phenofiles) - - try: - logger.debug("Attempt quick save with `LOAD … INFILE`.") - return quick_save_phenotypes_data( - conn, - table, - __build_dataitems__( - phenofiles, - control_data, - samples, - dataidmap, - pheno_name2id), - filesdir) - except Exception as _exc:# pylint: disable=[broad-exception-caught] - logger.debug("Could not use `LOAD … INFILE`, using raw query", - exc_info=True) - time.sleep(60) - return save_phenotypes_data( - conn, - table, - __build_dataitems__( - phenofiles, - control_data, - samples, - dataidmap, - pheno_name2id)) - - -save_pheno_data = partial(save_numeric_data, - filetype="pheno", - table="PublishData") - - -save_phenotypes_se = partial(save_numeric_data, - filetype="phenose", - table="PublishSE") - - -save_phenotypes_n = partial(save_numeric_data, - filetype="phenonum", - table="NStrain") - - -def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments,too-many-arguments] - auth_details, - resource_details, - species, - population, - dataset, - xrefdata): - """Grant the user access to their data.""" - logger.info("Updating authorisation for the data.") - logger.debug("Resource details for the authorisation: %s", resource_details) - authserver, token = auth_details - _tries = 0 - _delay = 1 - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - "Accept": "application/json" - } - def authserveruri(endpoint): - return urljoin(authserver, endpoint) - - def __fetch_user_details__(): - logger.info("… Fetching user details") - return mrequests.get( - authserveruri("/auth/user/"), - headers=headers - ) - - def __link_data__(user): - logger.info("… linking uploaded data to user's group") - return mrequests.post( - authserveruri("/auth/data/link/phenotype"), - headers=headers, - json={ - "species_name": species["Name"], - "group_id": user["group"]["group_id"], - "selected": [ - { - "SpeciesId": species["SpeciesId"], - "InbredSetId": population["Id"], - "PublishFreezeId": dataset["Id"], - "dataset_name": dataset["Name"], - "dataset_fullname": dataset["FullName"], - "dataset_shortname": dataset["ShortName"], - "PublishXRefId": item["xref_id"] - } - for item in xrefdata - ], - "using-raw-ids": "on" - }).then(lambda ld_results: (user, ld_results)) - - def __fetch_phenotype_category_details__(user, linkeddata): - logger.info("… fetching phenotype category details") - return mrequests.get( - authserveruri("/auth/resource/categories"), - headers=headers - ).then( - lambda categories: ( - user, - linkeddata, - next(category for category in categories - if category["resource_category_key"] == "phenotype")) - ) - - def __create_resource__(user, linkeddata, category): - logger.info("… creating authorisation resource object") - return mrequests.post( - authserveruri("/auth/resource/create"), - headers=headers, - json={ - **resource_details, - "resource_category": category["resource_category_id"], - "public": "off" - }).then(lambda cr_results: (user, linkeddata, cr_results)) - - def __attach_data_to_resource__(user, linkeddata, resource): - logger.info("… attaching data to authorisation resource object") - return mrequests.post( - authserveruri("/auth/resource/data/link"), - headers=headers, - json={ - "dataset_type": "phenotype", - "resource_id": resource["resource_id"], - "data_link_ids": [ - item["data_link_id"] for item in linkeddata["traits"]] - }).then(lambda attc: (user, linkeddata, resource, attc)) - - def __handle_error__(resp): - logger.error("ERROR OCCURRED: %s", resp.response.text) - error = resp.json() - if error.get("error") == "IntegrityError": - # This is hacky. If the auth already exists, something went wrong - # somewhere. - # This needs investigation to recover correctly. - logger.error( - "Error: The authorisation for the data was already set up.") - return 0 - logger.error("ERROR: Updating the authorisation for the data failed.") - logger.debug( - "ERROR: The response from the authorisation server was:\n\t%s", - error) - return 1 - - def __handle_success__(_val): - logger.info( - "The authorisation for the data has been updated successfully.") - return 0 - - return __fetch_user_details__().then(__link_data__).then( - lambda result: __fetch_phenotype_category_details__(*result) - ).then( - lambda result: __create_resource__(*result) - ).then( - lambda result: __attach_data_to_resource__(*result) - ).either(__handle_error__, __handle_success__) - - -def load_data(# pylint: disable=[too-many-locals] - conn: mysqldb.Connection, job: dict -) -> tuple[dict, dict, dict, tuple[int, ...]]: - """Load the data attached in the given job.""" - _job_metadata = job["metadata"] - # Steps - # 0. Read data from the files: can be multiple files per type - # - _species = species_by_id(conn, int(_job_metadata["species_id"])) - _population = population_by_species_and_id( - conn, - _species["SpeciesId"], - int(_job_metadata["population_id"])) - _dataset = dataset_by_id( - conn, - _species["SpeciesId"], - _population["Id"], - int(_job_metadata["dataset_id"])) - # 1. Just retrive the publication: Don't create publications for now. - _publication = fetch_publication_by_id( - conn, int(_job_metadata.get("publication_id", "0"))) or {"Id": 0} - # 2. Save all new phenotypes: - # -> return phenotype IDs - bundle = Path(_job_metadata["bundle_file"]) - _control_data = rqtl2.control_data(bundle) - logger.info("Extracting the zipped bundle of files.") - _outdir = Path(bundle.parent, f"bundle_{bundle.stem}") - with ZipFile(str(bundle), "r") as zfile: - _files = rqtl2.extract(zfile, _outdir) - logger.info("Saving new phenotypes.") - _phenos = save_phenotypes(conn, - _control_data, - _population["Id"], - _publication["Id"], - _outdir) - - def __build_phenos_maps__(accumulator, row): - return ({ - **accumulator[0], - row["phenotype_id"]: { - "population_id": _population["Id"], - "phenotype_id": row["phenotype_id"], - "data_id": row["data_id"], - "publication_id": row["publication_id"], - } - }, { - **accumulator[1], - row["pre_publication_abbreviation"]: row["phenotype_id"] - }, ( - accumulator[2] + ({ - "xref_id": row["xref_id"], - "population_id": row["population_id"], - "phenotype_id": row["phenotype_id"], - "publication_id": row["publication_id"], - "data_id": row["data_id"] - },))) - dataidmap, pheno_name2id, _xrefs = reduce(# type: ignore[var-annotated] - __build_phenos_maps__, _phenos, ({},{}, tuple())) - # 3. a. Fetch the strain names and IDS: create name->ID map - samples = { - row["Name"]: row - for row in samples_by_species_and_population( - conn, _species["SpeciesId"], _population["Id"])} - # b. Save all the data items (DataIds are vibes), return new IDs - logger.info("Saving new phenotypes data.") - _num_data_rows = save_pheno_data(conn=conn, - dataidmap=dataidmap, - pheno_name2id=pheno_name2id, - samples=samples, - control_data=_control_data, - filesdir=_outdir) - logger.info("Saved %s new phenotype data rows.", _num_data_rows) - - # 4. If standard errors and N exist, save them too - # (use IDs returned in `3. b.` above). - if _control_data.get("phenose"): - logger.info("Saving new phenotypes standard errors.") - _num_se_rows = save_phenotypes_se(conn=conn, - dataidmap=dataidmap, - pheno_name2id=pheno_name2id, - samples=samples, - control_data=_control_data, - filesdir=_outdir) - logger.info("Saved %s new phenotype standard error rows.", _num_se_rows) - - if _control_data.get("phenonum"): - logger.info("Saving new phenotypes sample counts.") - _num_n_rows = save_phenotypes_n(conn=conn, - dataidmap=dataidmap, - pheno_name2id=pheno_name2id, - samples=samples, - control_data=_control_data, - filesdir=_outdir) - logger.info("Saved %s new phenotype sample counts rows.", _num_n_rows) - - return (_species, _population, _dataset, _xrefs) - - -def update_means( - conn: mysqldb.Connection, - population_id: int, - xref_ids: tuple[int, ...] -): - """Compute the means from the data and update them in the database.""" - logger.info("Computing means for %02d phenotypes.", len(xref_ids)) - query = ( - "UPDATE PublishXRef SET mean = " - "(SELECT AVG(value) FROM PublishData" - " WHERE PublishData.Id=PublishXRef.DataId) " - "WHERE PublishXRef.Id=%(xref_id)s " - "AND PublishXRef.InbredSetId=%(population_id)s") - _xref_iterator = (_xref_id for _xref_id in xref_ids) - with conn.cursor(cursorclass=DictCursor) as cursor: - while True: - batch = take(_xref_iterator, 10000) - if len(batch) == 0: - break - logger.info("\tComputing means for batch of %02d phenotypes.", len(batch)) - cursor.executemany( - query, - tuple({ - "population_id": population_id, - "xref_id": _xref_id - } for _xref_id in batch)) - - -if __name__ == "__main__": - def parse_args(): - """Setup command-line arguments.""" - parser = argparse.ArgumentParser( - prog="load_phenotypes_to_db", - description="Process the phenotypes' data and load it into the database.") - parser.add_argument("db_uri", type=str, help="MariaDB/MySQL connection URL") - parser.add_argument( - "jobs_db_path", type=Path, help="Path to jobs' SQLite database.") - parser.add_argument("job_id", type=uuid.UUID, help="ID of the running job") - parser.add_argument( - "--log-level", - type=str, - help="Determines what is logged out.", - choices=("debug", "info", "warning", "error", "critical"), - default="info") - return parser.parse_args() - - def setup_logging(log_level: str): - """Setup logging for the script.""" - logger.setLevel(log_level) - logging.getLogger("uploader.phenotypes.models").setLevel(log_level) - - - def __parse_resource_details__(meta) -> dict: - """Parse out details regarding the wrapper resource from the metadata.""" - _key_mappings_ = { - # allow both 'data_*' and 'data*' for the metadata. - "data_description": "description", - "datadescription": "description" - } - return { - "resource_name": meta.get( - "dataname", - meta.get("data_name", - "Unnamed phenotypes - " + datetime.now().isoformat())), - "resource_metadata": { - rkey: meta[mkey] - for mkey, rkey in _key_mappings_.items() if mkey in meta - } - } - - - def main(): - """Entry-point for this script.""" - args = parse_args() - setup_logging(args.log_level.upper()) - - with (mysqldb.database_connection(args.db_uri) as conn, - conn.cursor(cursorclass=DictCursor) as cursor, - sqlite3.connection(args.jobs_db_path) as jobs_conn): - job = jobs.job(jobs_conn, args.job_id) - - # Lock the PublishXRef/PublishData/PublishSE/NStrain here: Why? - # The `DataId` values are sequential, but not auto-increment - # Can't convert `PublishXRef`.`DataId` to AUTO_INCREMENT. - # `SELECT MAX(DataId) FROM PublishXRef;` - # How do you check for a table lock? - # https://oracle-base.com/articles/mysql/mysql-identify-locked-tables - # `SHOW OPEN TABLES LIKE 'Publish%';` - _db_tables_ = ( - "Species", - "InbredSet", - "Strain", - "StrainXRef", - "Publication", - "Phenotype", - "PublishXRef", - "PublishFreeze", - "PublishData", - "PublishSE", - "NStrain") - - logger.debug( - ("Locking database tables for the connection:" + - "".join("\n\t- %s" for _ in _db_tables_) + "\n"), - *_db_tables_) - cursor.execute(# Lock the tables to avoid race conditions - "LOCK TABLES " + ", ".join( - f"{_table} WRITE" for _table in _db_tables_)) - - db_results = load_data(conn, job) - _xref_ids = tuple(xref["xref_id"] for xref in db_results[3]) - jobs.update_metadata( - jobs_conn, - args.job_id, - "xref_ids", - json.dumps(_xref_ids)) - - logger.info("Unlocking all database tables.") - cursor.execute("UNLOCK TABLES") - - logger.info("Updating means.") - update_means(conn, db_results[1]["Id"], _xref_ids) - - # Update authorisations (break this down) — maybe loop until it works? - logger.info("Updating authorisation.") - _job_metadata = job["metadata"] - - return update_auth((_job_metadata["authserver"], - _job_metadata["token"]), - __parse_resource_details__(_job_metadata), - *db_results) - - - try: - sys.exit(main()) - except Exception as _exc:# pylint: disable=[broad-exception-caught] - logger.debug("Data loading failed… Halting!", - exc_info=True) - sys.exit(1) diff --git a/scripts/phenotypes/__init__.py b/scripts/phenotypes/__init__.py deleted file mode 100644 index 73ad839..0000000 --- a/scripts/phenotypes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"Scripts for dealing with phenotypes." diff --git a/scripts/phenotypes/delete_phenotypes.py b/scripts/phenotypes/delete_phenotypes.py deleted file mode 100644 index 461f3ec..0000000 --- a/scripts/phenotypes/delete_phenotypes.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Delete phenotypes.""" -import sys -import logging -from pathlib import Path -from typing import Optional -from urllib.parse import urljoin -from argparse import Namespace, ArgumentParser - -import requests -from MySQLdb.cursors import DictCursor, BaseCursor - -from gn_libs.mysqldb import database_connection - -from uploader.phenotypes.models import delete_phenotypes -from scripts.cli.logging import setup_logging -from scripts.cli.options import (add_logging, - add_mariadb_uri, - add_population_id) - -logger = logging.getLogger(__name__) - -def read_xref_ids_file(filepath: Optional[Path]) -> tuple[int, ...]: - """Read the phenotypes' cross-reference IDS from file.""" - if filepath is None: - return tuple() - - logger.debug("Using file '%s' to retrieve XREF IDs for deletion.", - filepath.name) - _ids: tuple[int, ...] = tuple() - with filepath.open(mode="r") as infile: - for line in infile.readlines(): - try: - _ids += (int(line.strip()),) - except TypeError: - pass - - return _ids - - -def fetch_all_xref_ids( - cursor: BaseCursor, population_id: int) -> tuple[int, ...]: - """Fetch all cross-reference IDs.""" - cursor.execute("SELECT Id FROM PublishXRef WHERE InbredSetId=%s", - (population_id,)) - return tuple(int(row["Id"]) for row in cursor.fetchall()) - - -def update_auth( - auth_details: tuple[str, str], - species_id: int, - population_id: int, - dataset_id: int, - xref_ids: tuple[int, ...] = tuple() -): - """Update the authorisation server: remove items to delete.""" - authserver, token = auth_details - resp = requests.post( - urljoin(authserver, - (f"/auth/data/phenotypes/{species_id}/{population_id}" - f"/{dataset_id}/delete")), - timeout=(9.13, 20), - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json" - }, - json={"xref_ids": xref_ids}) - resp.raise_for_status() - - -def delete_the_phenotypes( - cursor: BaseCursor, - population_id: int, - xref_ids: tuple[int, ...] = tuple()) -> int: - """Process and delete the phenotypes.""" - delete_phenotypes(cursor, population_id, xref_ids) - - return 0 - -if __name__ == "__main__": - def parse_args() -> Namespace: - """Parse CLI arguments.""" - parser = add_logging( - add_population_id( - add_mariadb_uri( - ArgumentParser( - prog="delete-phenotypes", - description=( - "Script to delete phenotypes from the database."))))) - parser.add_argument( - "dataset_id", - metavar="DATASET-ID", - type=int, - help="The dataset identifier for phenotypes to delete.") - parser.add_argument( - "auth_server_uri", - metavar="AUTH-SERVER-URI", - type=str, - help="URI to the authorisation server.") - parser.add_argument( - "auth_token", - metavar="AUTH-TOKEN", - type=str, - help=("Token to use to update the authorisation system with the " - "deletions done.")) - parser.add_argument( - "--xref_ids_file", - metavar="XREF-IDS-FILE", - type=Path, - help=("Path to a file with phenotypes cross-reference IDs to " - "delete.")) - parser.add_argument( - "--delete-all", - action="store_true", - help=("If no 'XREF-IDS-FILE' is provided, this flag determines " - "whether or not all the phenotypes for the given population " - "will be deleted.")) - return parser.parse_args() - - - def main(): - """The `delete-phenotypes` script's entry point.""" - args = parse_args() - setup_logging(logger, args.log_level.upper(), tuple()) - with (database_connection(args.db_uri) as conn, - conn.cursor(cursorclass=DictCursor) as cursor): - xref_ids = read_xref_ids_file(args.xref_ids_file) - try: - assert not (len(xref_ids) > 0 and args.delete_all) - xref_ids = (fetch_all_xref_ids(cursor, args.population_id) - if args.delete_all else xref_ids) - logger.debug("Will delete %s phenotypes and related data", - len(xref_ids)) - if len(xref_ids) == 0: - print("No cross-reference IDs were provided. Aborting.") - return 0 - - print("Updating authorisations: ", end="") - update_auth((args.auth_server_uri, args.auth_token), - args.species_id, - args.population_id, - args.dataset_id, - xref_ids) - print("OK.") - print("Deleting the data: ", end="") - delete_phenotypes(cursor, args.population_id, xref_ids=xref_ids) - print("OK.") - if args.xref_ids_file is not None: - print("Deleting temporary file: ", end="") - args.xref_ids_file.unlink() - print("OK.") - - return 0 - except AssertionError: - logger.error( - "'DELETE-ALL' and 'XREF-IDS' are mutually exclusive. " - "If you specify the list of XREF-IDS (in a file) to delete " - "and also specify to 'DELETE-ALL' phenotypes in the " - "population, we have no way of knowing what it is you want.") - return 1 - except requests.exceptions.HTTPError as _exc: - resp = _exc.response - resp_data = resp.json() - logger.debug("%s: %s", - resp_data["error"], - resp_data["error_description"], - exc_info=True) - return 1 - except Exception as _exc:# pylint: disable=[broad-exception-caught] - logger.debug("Failed while attempting to delete phenotypes.", - exc_info=True) - return 1 - - sys.exit(main()) diff --git a/scripts/process_rqtl2_bundle.py b/scripts/process_rqtl2_bundle.py deleted file mode 100644 index e2ce420..0000000 --- a/scripts/process_rqtl2_bundle.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Entry point for processing R/qtl2 bundles.""" -import sys -import uuid -import json -import argparse -import traceback -from typing import Any -from pathlib import Path -from zipfile import ZipFile -from logging import Logger, getLogger, StreamHandler - -import MySQLdb as mdb -from redis import Redis -from gn_libs.mysqldb import database_connection - -from functional_tools import take - -import r_qtl.r_qtl2 as rqtl2 -import r_qtl.r_qtl2_qc as rqc -import r_qtl.exceptions as rqe - -from uploader import jobs -from uploader.check_connections import check_db, check_redis - -from scripts.cli_parser import init_cli_parser -from scripts.redis_logger import setup_redis_logger - -from scripts.rqtl2.install_phenos import install_pheno_files -from scripts.rqtl2.install_genotypes import install_genotypes - -def safe_json_decode(value: str) -> Any: - """Safely decode the string values into JSON.""" - try: - return json.loads(value) - except json.decoder.JSONDecodeError: - return value - -def parse_job(rconn: Redis, rprefix: str, jobid: uuid.UUID) -> dict: - """Load and parse job from Redis""" - return { - key: safe_json_decode(value) - for key, value - in jobs.job(rconn, rprefix, jobid).items() - } - -def has_geno_file(job: dict) -> bool: - """Check whether to trigger processing of geno file(s).""" - return bool(job.get("bundle-metadata", {}).get("geno-dataset-id")) - -def has_pheno_file(job: dict) -> bool: - """Check whether to trigger processing of pheno file(s).""" - meta = job.get("bundle-metadata", {}) - return (bool(meta.get("probe-study-id")) - and bool(meta.get("probe-dataset-id"))) - -def percent_completion(geno: float, pheno: float) -> float: - """Compute the total completion percent.""" - return 0.5 * (geno + pheno) - -def qc_r_qtl2_bundle(bundlefilepath, logger): - """Run QC checks on the R/qtl2 bundle.""" - with ZipFile(bundlefilepath, "r") as zfile: - logger.info("Validating the bundle ...") - rqc.validate_bundle(zfile) - logger.info( - "Bundle successfully validated. All listed files are present.") - - cdata = rqtl2.control_data(zfile) - if "geno" in cdata: - gerr = False - logger.info("Validating 'geno' file.") - for error in take(rqc.geno_errors(zfile), 10): - gerr = True - logger.error("%s: [Line %s, Field %s]", - error[2], error[0], error[1]) - if gerr: - logger.error("... more") - raise rqe.InvalidFormat("'geno' file content contains errors.") - logger.info("'geno' file validation was successful.") - -def process_bundle(dbconn: mdb.Connection, - rconn: Redis, - rprefix: str, - jobid: uuid.UUID, - logger: Logger) -> int: - """Process the R/qtl2 bundle.""" - try: - thejob = parse_job(rconn, rprefix, jobid) - meta = thejob["bundle-metadata"] - qc_r_qtl2_bundle(meta["rqtl2-bundle-file"], logger) - rconn.hset(jobs.job_key(rprefix, jobid), "geno-percent", "0") - rconn.hset(jobs.job_key(rprefix, jobid), "pheno-percent", "0") - - if has_geno_file(thejob): - logger.info("Processing geno files.") - genoexit = install_genotypes( - rconn, - dbconn, - f"{rprefix}:{jobid}", - argparse.Namespace( - speciesid=meta["speciesid"], - populationid=meta["populationid"], - datasetid=meta["geno-dataset-id"], - rqtl2bundle=Path(meta["rqtl2-bundle-file"])), - logger) - if genoexit != 0: - raise Exception("Processing 'geno' file failed.")# pylint: disable=[broad-exception-raised] - logger.debug( - "geno file processing completed successfully. (ExitCode: %s)", - genoexit) - rconn.hset(jobs.job_key(rprefix, jobid), "geno-percent", "100") - - if has_pheno_file(thejob): - phenoexit = install_pheno_files( - rconn, - dbconn, - f"{rprefix}:{jobid}", - argparse.Namespace( - speciesid=meta["speciesid"], - platformid=meta["platformid"], - dataset_id=meta["probe-dataset-id"], - rqtl2bundle=Path(meta["rqtl2-bundle-file"])), - logger) - if phenoexit != 0: - raise Exception("Processing 'pheno' file failed.")# pylint: disable=[broad-exception-raised] - logger.debug( - "pheno file processing completed successfully. (ExitCode: %s)", - phenoexit) - rconn.hset(jobs.job_key(rprefix, jobid), "pheno-percent", "100") - - return 0 - except jobs.JobNotFound as exc: - logger.error("%s", exc.args) - except Exception as _exc:#pylint: disable=[broad-except] - logger.error("Exiting with generic error: %s", traceback.format_exc()) - - return 1 - -if __name__ == "__main__": - def main(): - """Run the `process_rqtl2_bundle` script.""" - args = init_cli_parser( - "upload_rqtl2_bundle", - "Launch this to control the processing of R/qtl2 zip bundles." - ).parse_args() - check_db(args.databaseuri) - check_redis(args.redisuri) - - logger = getLogger("process_rqtl2_bundle") - logger.addHandler(StreamHandler(stream=sys.stderr)) - logger.setLevel("DEBUG") - - fqjobid = jobs.job_key(args.redisprefix, args.jobid) - with (database_connection(args.databaseuri) as dbconn, - Redis.from_url(args.redisuri, decode_responses=True) as rconn): - logger.addHandler(setup_redis_logger( - rconn, fqjobid, f"{fqjobid}:log-messages", - args.redisexpiry)) - - exitcode = process_bundle( - dbconn, rconn, args.redisprefix, args.jobid, logger) - rconn.hset( - jobs.job_key(args.redisprefix, args.jobid), "percent", "100") - return exitcode - - sys.exit(main()) diff --git a/scripts/qc.py b/scripts/qc.py deleted file mode 100644 index b00f4c1..0000000 --- a/scripts/qc.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Implements the command-line interface for the qc application""" -import os -import sys -import mimetypes -from typing import Union, Callable -from argparse import ArgumentParser - -from gn_libs.mysqldb import database_connection - -from functional_tools import take - -from quality_control.utils import make_progress_calculator -from quality_control.errors import InvalidValue, DuplicateHeading -from quality_control.parsing import FileType, strain_names, collect_errors - -from .cli_parser import init_cli_parser - - -def is_file_mime(filepath:str, mimetype:str) -> bool: - """Check that `filepath` has a mimetype of `mimetype` or `text/plain`""" - the_type = mimetypes.guess_type(filepath)[0] - return the_type in ("text/plain", mimetype) - -def add_file_validation_arguments(parser: ArgumentParser) -> ArgumentParser: - """File validation specific CLI arguments.""" - parser.add_argument( - "filetype", - help="The type of file to check", - choices=("average", "standard-error")) - parser.add_argument( - "filepath", - help=( - "The path to the file to be checked." - "If an absolute path is not provided, then the file will be relative to" - f"\t'{os.getcwd()}'")) - - parser.add_argument( - "-c", "--count", type=int, - help=( - "Number of errors to display. " - "A negative number means display all errors."), - default=20) - - parser.add_argument( - "-v", "--verbose", - help="Controls whether to show extra output", - default=False, action="store_true") - return parser - -def cli_argument_parser(): - """Create the parser for the CLI arguments""" - theparser = init_cli_parser( - "qc", - "Command-Line Interface program for quality control of data files") - theparser.add_argument("speciesid", type=int, help="ID of the species.") - return add_file_validation_arguments(theparser) - -def make_progress_indicator( - verbose: bool, progress_calc_fn: Callable) -> Union[Callable, None]: - """Utility to display the progress""" - if not verbose: - return None - - def __indicator__(linenumber, linetext): - msg = progress_calc_fn(linenumber, linetext) - print(f"LINE: {msg.currentline} ({msg.percent:.2f}%)", end="\r") - return msg - - return __indicator__ - -def print_errors(errors, verbose): - """Print out the errors""" - errors_exist = False - starter = "\n" if verbose else "" - print(f"{starter}line(s)\tcolumn(s)\terrors") - for error in errors: - cols = ( - error.column if isinstance(error, InvalidValue) - else (", ".join(str(col) for col in error.columns) - if isinstance(error, DuplicateHeading) - else "-")) - errors_exist = True - print(f"{starter}{error.line}\t{cols}\t{error.message}") - - if not errors_exist: - print("No errors were found!") - - return errors - -def check(filepath, filetype, strains, count, verbose=False): - """Check the file and print out results""" - updater = make_progress_indicator( - verbose, make_progress_calculator(os.stat(filepath).st_size)) - - if count > 0: - return print_errors( - take(collect_errors(filepath, filetype, strains, updater), count), - verbose) - return print_errors( - collect_errors(filepath, filetype, strains, updater), verbose) - -def main(): - """Entry point function""" - argparser = cli_argument_parser() - args = argparser.parse_args() - if not os.path.exists(args.filepath): - print(f"The file '{args.filepath}' does not exist.", file=sys.stderr) - return 1 - - if not is_file_mime(args.filepath, "text/tab-separated-values"): - print( - f"The file '{args.filepath}' MUST be a tab-separated file.", - file=sys.stderr) - return 3 - - with database_connection(args.databaseuri) as dbconn: - strains = strain_names(dbconn, args.speciesid) - - filepath = os.path.realpath(args.filepath) - if args.verbose: - print(f"Checking '{filepath}' for errors") - - return check( - filepath, ( - FileType.AVERAGE if args.filetype == "average" - else FileType.STANDARD_ERROR), - strains, args.count, verbose=args.verbose) - -if __name__ == "__main__": - main() diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py deleted file mode 100644 index 4e6ef00..0000000 --- a/scripts/qc_on_rqtl2_bundle.py +++ /dev/null @@ -1,330 +0,0 @@ -"""Run Quality Control checks on R/qtl2 bundle.""" -import sys -import json -from time import sleep -from pathlib import Path -from zipfile import ZipFile -from functools import partial -from argparse import Namespace -import multiprocessing as mproc -from logging import Logger, getLogger, StreamHandler -from typing import Union, Sequence, Callable, Iterator - -import MySQLdb as mdb -from redis import Redis -from gn_libs.mysqldb import database_connection - -from quality_control.errors import InvalidValue -from quality_control.checks import decimal_points_error - -from uploader import jobs -from uploader.check_connections import check_db, check_redis - -from r_qtl import r_qtl2 as rqtl2 -from r_qtl import r_qtl2_qc as rqc -from r_qtl import exceptions as rqe -from r_qtl import fileerrors as rqfe - -from scripts.process_rqtl2_bundle import parse_job -from scripts.redis_logger import setup_redis_logger -from scripts.cli_parser import init_cli_parser, add_global_data_arguments - -def dict2tuple(dct: dict) -> tuple: - """Utility to convert items in dicts to pairs of tuples.""" - return tuple((key, val) for key,val in dct.items()) - -def add_to_errors(rconn: Redis, - fqjobid: str, - key: str, - errors: Sequence[Union[InvalidValue, rqfe.MissingFile]]): - """Add `errors` to a given list of errors""" - errs = tuple(dict(item) for item in set( - [dict2tuple(old) for old in - json.loads(rconn.hget(fqjobid, key) or "[]")] +# type: ignore[arg-type] - [dict2tuple({"type": type(error).__name__, **error._asdict()}) - for error in errors])) - rconn.hset(fqjobid, key, json.dumps(errs)) - -def qc_missing_files(rconn: Redis, - fqjobid: str, - zfile: ZipFile, - logger: Logger) -> bool: - """Run QC for files listed in control file that don't exist in bundle.""" - logger.info("Checking for missing files…") - missing = rqc.missing_files(zfile) - add_to_errors(rconn, fqjobid, "errors-generic", tuple( - rqfe.MissingFile( - mfile[0], mfile[1], ( - f"File '{mfile[1]}' is listed in the control file under " - f"the '{mfile[0]}' key, but it does not actually exist in " - "the bundle.")) - for mfile in missing)) - if len(missing) > 0: - logger.error("Missing files in the bundle!") - return True - return False - -def compute_filesize(zfile: ZipFile, filetype: str) -> int: - """Compute the total file size.""" - cdata = rqtl2.control_data(zfile) - if isinstance(cdata[filetype], str): - return zfile.getinfo(cdata[filetype]).file_size - - return sum(zfile.getinfo(afile).file_size for afile in cdata[filetype]) - -def retrieve_errors_with_progress(rconn: Redis,#pylint: disable=[too-many-locals] - fqjobid: str, - zfile: ZipFile, - filetype: str, - checkers: tuple[Callable]) -> Iterator[Union[ - InvalidValue, rqfe.MissingFile]]: - """Filter the errors while also counting the number of lines in the file.""" - assert filetype in rqtl2.FILE_TYPES, f"Invalid file type {filetype}." - count = 0 - checked = 0 - cdata = rqtl2.control_data(zfile) - rconn.hset( - fqjobid, f"{filetype}-filesize", str(compute_filesize(zfile, filetype))) - def __update_processed__(value): - nonlocal checked - checked = checked + len(value) - rconn.hset(fqjobid, f"{filetype}-checked", checked) - - try:# pylint: disable=[too-many-nested-blocks] - for lineno, row in enumerate( - rqtl2.file_data(zfile, filetype, cdata), start=1): - count = count + 1 - for field, value in row.items(): - if field == "id": - __update_processed__(value) - continue - if value is not None: - for checker in checkers: - error = checker(lineno=lineno, field=field, value=value) - if bool(error): - yield error - __update_processed__(value) - - rconn.hset(fqjobid, f"{filetype}-linecount", count)# type: ignore[arg-type] - except rqe.MissingFileException: - fname = cdata.get(filetype) - yield rqfe.MissingFile(filetype, fname, ( - f"The file '{fname}' does not exist in the bundle despite it being " - f"listed under '{filetype}' in the control file.")) - -def qc_geno_errors(rconn, fqjobid, _dburi, _speciesid, zfile, logger) -> bool: - """Check for errors in `geno` file(s).""" - cdata = rqtl2.control_data(zfile) - if "geno" in cdata: - logger.info("Checking for errors in the 'geno' file…") - gerrs = tuple(retrieve_errors_with_progress( - rconn, fqjobid, zfile, "geno", - (rqc.make_genocode_checker(cdata.get("genotypes", {}), "geno"),))) - add_to_errors(rconn, fqjobid, "errors-generic", tuple( - err for err in gerrs if isinstance(err, rqfe.MissingFile))) - add_to_errors(rconn, fqjobid, "errors-geno", tuple( - err for err in gerrs if not isinstance(err, rqfe.MissingFile))) - if len(gerrs) > 0: - logger.error("The 'geno' file has errors.") - return True - logger.info("No errors found in the 'geno' file.") - - return False - - -def fetch_db_geno_samples(conn: mdb.Connection, speciesid: int) -> tuple[str, ...]: - """Fetch samples/cases/individuals from the database.""" - samples = set()# type: ignore[var-annotated] - with conn.cursor() as cursor: - cursor.execute("SELECT Name, Name2 from Strain WHERE SpeciesId=%s", - (speciesid,)) - rows = cursor.fetchall() or tuple() - for row in rows: - samples.update(tuple(row)) - - return tuple(item.strip() for item in samples if bool(item)) - - -def check_pheno_samples( - conn: mdb.Connection, - speciesid: int, - zipfilepath: Union[str, Path], - logger: Logger -) -> tuple[Union[InvalidValue, rqfe.MissingFile], ...]: - """Check that samples in 'pheno' file exist in geno file.""" - cdata = rqtl2.read_control_file(zipfilepath) - genosamples = tuple( - sample for perfilesamples in ( - rqtl2.load_samples(zipfilepath, member, cdata["geno_transposed"]) - for member in cdata["geno"]) - for sample in perfilesamples) + fetch_db_geno_samples(conn, speciesid) - - def __check_file__(member) -> tuple[InvalidValue, ...]: - logger.info("Checking samples/cases in member file '%s' …", member) - sampledata = rqtl2.load_samples( - zipfilepath, member, cdata["pheno_transposed"]) - errors: tuple[InvalidValue, ...] = tuple() - for sample in sampledata: - if sample not in genosamples: - errors = errors + (InvalidValue( - member, "-", "-", sample, - f"The individual/case/sample '{sample}' in file " - f"{member} does not exist in either, any of the 'geno' " - "files provided in the bundle or the GeneNetwork database." - ),) - - logger.info("Found %s missing samples in member file '%s'.", - len(errors), - member) - return errors - - allerrors: tuple[Union[InvalidValue, rqfe.MissingFile], ...] = tuple() - for afile in cdata["pheno"]: - try: - allerrors = allerrors + __check_file__(afile) - except KeyError: - allerrors = allerrors + (rqfe.MissingFile( - "pheno", - afile, - (f"The file '{afile}' does not exist in the zipfile despite " - "being listed in the control file.")),) - - return allerrors - - -def qc_pheno_errors(# pylint: disable=[too-many-arguments, too-many-positional-arguments] - rconn, fqjobid, dburi, speciesid, zfile, logger) -> bool: - """Check for errors in `pheno` file(s).""" - cdata = rqtl2.control_data(zfile) - if "pheno" in cdata: - logger.info("Checking for errors in the 'pheno' file…") - perrs = tuple()# type: ignore[var-annotated] - with database_connection(dburi) as dbconn: - perrs = check_pheno_samples( - dbconn, speciesid, zfile.filename, logger) + tuple( - retrieve_errors_with_progress( - rconn,fqjobid, zfile, "pheno", - (partial(decimal_points_error, - filename="pheno", - mini=3),))) - add_to_errors(rconn, fqjobid, "errors-generic", tuple( - err for err in perrs if isinstance(err, rqfe.MissingFile))) - add_to_errors(rconn, fqjobid, "errors-pheno", tuple( - err for err in perrs if not isinstance(err, rqfe.MissingFile))) - if len(perrs) > 0: - logger.error("The 'pheno' file has errors.") - return True - logger.info("No errors found in the 'pheno' file.") - - return False - -def qc_phenose_errors(# pylint: disable=[too-many-arguments] - rconn, fqjobid, _dburi, _speciesid, zfile, logger) -> bool: - """Check for errors in `phenose` file(s).""" - cdata = rqtl2.control_data(zfile) - if "phenose" in cdata: - logger.info("Checking for errors in the 'phenose' file…") - perrs = tuple(retrieve_errors_with_progress( - rconn,fqjobid, zfile, "phenose", - (partial(decimal_points_error, mini=6),))) - add_to_errors(rconn, fqjobid, "errors-generic", tuple( - err for err in perrs if isinstance(err, rqfe.MissingFile))) - add_to_errors(rconn, fqjobid, "errors-phenose", tuple( - err for err in perrs if not isinstance(err, rqfe.MissingFile))) - if len(perrs) > 0: - logger.error("The 'phenose' file has errors.") - return True - logger.info("No errors found in the 'phenose' file.") - - return False - -def qc_phenocovar_errors( - _rconn, - _fqjobid, - _dburi, - _speciesid, - _zfile, - _logger -) -> bool: - """Check for errors in `phenocovar` file(s).""" - return False - -def run_qc(rconn: Redis, - args: Namespace, - logger: Logger) -> int: - """Run the QC programs.""" - fqjobid = jobs.job_key(args.redisprefix, args.jobid) - thejob = parse_job(rconn, args.redisprefix, args.jobid) - jobmeta = thejob["job-metadata"] - - with ZipFile(jobmeta["rqtl2-bundle-file"], "r") as zfile: - if qc_missing_files(rconn, fqjobid, zfile, logger): - return 1 - - def with_zipfile(# pylint: disable=[too-many-arguments,too-many-positional-arguments] - rconn, fqjobid, dbconn, speciesid, filename, logger, func - ): - with ZipFile(filename, "r") as zfile: - return func(rconn, fqjobid, dbconn, speciesid, zfile, logger) - - def buildargs(func): - return (rconn, - fqjobid, - args.databaseuri, - args.speciesid, - jobmeta["rqtl2-bundle-file"], - logger, - func) - processes = [ - mproc.Process(target=with_zipfile, args=buildargs(qc_geno_errors,)), - mproc.Process(target=with_zipfile, args=buildargs(qc_pheno_errors,)), - mproc.Process(target=with_zipfile, args=buildargs(qc_phenose_errors,)), - mproc.Process(target=with_zipfile, args=buildargs(qc_phenocovar_errors,)) - ] - for process in processes: - process.start() - - while True: - processes_running = any( - (process.is_alive() for process in processes)) - if not processes_running: - break - sleep(2) - - if any((process.exitcode for process in processes)): - # at least one process failed for some reason... - return 1 - - def __fetch_errors__(rkey: str) -> tuple: - return tuple(json.loads(rconn.hget(fqjobid, rkey) or "[]")) # type: ignore[arg-type] - - return (1 if any(( - bool(__fetch_errors__(key)) - for key in - ("errors-geno", "errors-pheno", "errors-phenos", "errors-phenocovar"))) - else 0) - -if __name__ == "__main__": - def main(): - """Enter R/qtl2 bundle QC runner.""" - args = add_global_data_arguments(init_cli_parser( - "qc-on-rqtl2-bundle", "Run QC on R/qtl2 bundle.")).parse_args() - check_redis(args.redisuri) - check_db(args.databaseuri) - - logger = getLogger("qc-on-rqtl2-bundle") - logger.addHandler(StreamHandler(stream=sys.stderr)) - logger.setLevel("DEBUG") - - fqjobid = jobs.job_key(args.redisprefix, args.jobid) - with Redis.from_url(args.redisuri, decode_responses=True) as rconn: - logger.addHandler(setup_redis_logger( - rconn, fqjobid, f"{fqjobid}:log-messages", - args.redisexpiry)) - - exitcode = run_qc(rconn, args, logger) - rconn.hset( - jobs.job_key(args.redisprefix, args.jobid), "exitcode", exitcode) - return exitcode - - sys.exit(main()) diff --git a/scripts/qcapp_wsgi.py b/scripts/qcapp_wsgi.py deleted file mode 100644 index fe77031..0000000 --- a/scripts/qcapp_wsgi.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Run the application""" -import os -import sys -from logging import getLogger, StreamHandler - -from flask import Flask - -from uploader import create_app -from uploader.check_connections import check_db, check_redis - -def setup_logging(appl: Flask) -> Flask: - """Setup appropriate logging paradigm depending on environment.""" - # https://datatracker.ietf.org/doc/html/draft-coar-cgi-v11-03#section-4.1.17 - # https://wsgi.readthedocs.io/en/latest/proposals-2.0.html#making-some-keys-required - # https://peps.python.org/pep-3333/#id4 - software, *_version_and_comments = os.environ.get( - "SERVER_SOFTWARE", "").split('/') - if bool(software): - gunicorn_logger = getLogger("gunicorn.error") - appl.logger.handlers = gunicorn_logger.handlers - appl.logger.setLevel(gunicorn_logger.level)#pylint: disable=[no-member] - else: - loglevel = appl.config["LOG_LEVEL"].upper() - # Maybe call `logging.dictConfig(…)` here instead of all this stuff below - handler_stderr = StreamHandler(stream=sys.stderr) - appl.logger.addHandler(handler_stderr) - rootlogger = getLogger() - rootlogger.addHandler(handler_stderr) - rootlogger.setLevel(loglevel) - appl.logger.setLevel(loglevel) - - return appl - - -def check_and_build_app() -> Flask: - """Setup the application for running.""" - # Setup the app - appl = create_app() - - # Check connections - check_db(appl.config["SQL_URI"]) - check_redis(appl.config["REDIS_URL"]) - return setup_logging(appl) - - -app = check_and_build_app() - -if __name__ == "__main__": - # Run the app - app.run() diff --git a/scripts/redis_logger.py b/scripts/redis_logger.py deleted file mode 100644 index a74e5e4..0000000 --- a/scripts/redis_logger.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Utilities to log to redis for our worker scripts.""" -import logging -from typing import Optional - -from redis import Redis - -class RedisLogger(logging.Handler): - """Log out to redis for our worker scripts""" - def __init__(self,#pylint: disable=[too-many-arguments, too-many-positional-arguments] - rconn: Redis, - fullyqualifiedjobid: str, - messageslistname: str, - level:int = logging.NOTSET, - expiry: int = 86400): - """Initialise the handler.""" - super().__init__(level) - self.redisconnection = rconn - self.jobid = fullyqualifiedjobid - self.messageslistname = messageslistname - self.expiry = expiry - rconn.hset(name=fullyqualifiedjobid, - key="log-messagelist", - value=messageslistname) - - def emit(self, record): - """Log to the messages list for a given worker.""" - self.redisconnection.rpush(self.messageslistname, self.format(record)) - self.redisconnection.expire(self.messageslistname, self.expiry) - -class RedisMessageListHandler(logging.Handler): - """Send messages to specified redis list.""" - def __init__(self, - rconn: Redis, - fullyqualifiedkey: str, - loglevel: int = logging.NOTSET, - expiry: Optional[int] = 86400): - super().__init__(loglevel) - self.redisconnection = rconn - self.fullyqualifiedkey = fullyqualifiedkey - self.expiry = expiry - - def emit(self, record): - """Log out to specified `fullyqualifiedkey`.""" - self.redisconnection.rpush(self.fullyqualifiedkey, self.format(record)) - if bool(self.expiry): - self.redisconnection.expire(self.fullyqualifiedkey, self.expiry) - else: - self.redisconnection.persist(self.fullyqualifiedkey) - -def setup_redis_logger(rconn: Redis, - fullyqualifiedjobid: str, - job_messagelist: str, - expiry: int = 86400) -> RedisLogger: - """Setup a default RedisLogger logger.""" - formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s: %(message)s") - rconn.hset( - name=fullyqualifiedjobid, key="log-messagelist", value=job_messagelist) - redislogger = RedisLogger( - rconn, fullyqualifiedjobid, job_messagelist, expiry=expiry) - redislogger.setFormatter(formatter) - return redislogger diff --git a/scripts/rqtl2/__init__.py b/scripts/rqtl2/__init__.py deleted file mode 100644 index 62bc52a..0000000 --- a/scripts/rqtl2/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""A package of scripts for use in parsing R/qtl2 bundles""" diff --git a/scripts/rqtl2/bundleutils.py b/scripts/rqtl2/bundleutils.py deleted file mode 100644 index 17faa7c..0000000 --- a/scripts/rqtl2/bundleutils.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Common utilities to operate in R/qtl2 bundles.""" -from typing import Union, Callable - -def build_line_splitter(cdata: dict) -> Callable[[str], tuple[Union[str, None], ...]]: - """Build and return a function to use to split data in the files. - - Parameters - ---------- - cdata: A dict holding the control information included with the R/qtl2 - bundle. - - Returns - ------- - A function that takes a string and return a tuple of strings. - """ - separator = cdata["sep"] - na_strings = cdata["na.strings"] - def __splitter__(line: str) -> tuple[Union[str, None], ...]: - return tuple( - item if item not in na_strings else None - for item in - (field.strip() for field in line.strip().split(separator))) - return __splitter__ - - -def build_line_joiner(cdata: dict) -> Callable[[tuple[Union[str, None], ...]], str]: - """Build and return a function to use to split data in the files. - - Parameters - ---------- - cdata: A dict holding the control information included with the R/qtl2 - bundle. - - Returns - ------- - A function that takes a string and return a tuple of strings. - """ - separator = cdata["sep"] - na_strings = cdata["na.strings"] - def __joiner__(row: tuple[Union[str, None], ...]) -> str: - return separator.join( - (na_strings[0] if item is None else item) - for item in row) - return __joiner__ diff --git a/scripts/rqtl2/cli_parser.py b/scripts/rqtl2/cli_parser.py deleted file mode 100644 index 9bb60a3..0000000 --- a/scripts/rqtl2/cli_parser.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Utilities for CLI parsers specific to R/qtl2 parsing scripts.""" -from pathlib import Path -from argparse import ArgumentParser - -def add_bundle_argument(parser: ArgumentParser) -> ArgumentParser: - """Add the `rqtl2bundle` argument.""" - parser.add_argument("rqtl2bundle", - type=Path, - help="Path to R/qtl2 bundle zip file.") - return parser - - -def add_datasetid_argument(parser: ArgumentParser) -> ArgumentParser: - """Add the `datasetid` argument.""" - parser.add_argument("datasetid", - type=int, - help="The dataset to which the data belongs.") - return parser - - -def add_common_arguments(parser: ArgumentParser) -> ArgumentParser: - """Add common arguments to the CLI parser.""" - return add_bundle_argument(add_datasetid_argument(parser)) diff --git a/scripts/rqtl2/entry.py b/scripts/rqtl2/entry.py deleted file mode 100644 index 7423a4b..0000000 --- a/scripts/rqtl2/entry.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Build common script-entry structure.""" -import sys -import logging -from typing import Callable -from argparse import Namespace -from logging import StreamHandler - -from redis import Redis -from MySQLdb import Connection -from gn_libs.mysqldb import database_connection - -from uploader import jobs -from uploader.check_connections import check_db, check_redis - -from scripts.redis_logger import setup_redis_logger - -def build_main( - args: Namespace, - run_fn: Callable[ - [Redis, Connection, str, Namespace, logging.Logger], - int - ], - logger: logging.Logger -) -> Callable[[],int]: - """Build a function to be used as an entry-point for scripts.""" - def main(): - with (Redis.from_url(args.redisuri, decode_responses=True) as rconn, - database_connection(args.databaseuri) as dbconn): - logger.setLevel(args.log_level.upper()) - fqjobid = jobs.job_key(args.redisprefix, args.jobid) - - try: - rconn.hset(fqjobid, "status", "started") - logger.addHandler(setup_redis_logger( - rconn, - fqjobid, - f"{fqjobid}:log-messages", - args.redisexpiry)) - logger.addHandler(StreamHandler(stream=sys.stderr)) - - check_db(args.databaseuri) - check_redis(args.redisuri) - if not args.rqtl2bundle.exists(): - logger.error("File not found: '%s'.", args.rqtl2bundle) - return 2 - - returncode = run_fn(rconn, dbconn, fqjobid, args) - if returncode == 0: - rconn.hset(fqjobid, "status", "completed:success") - return returncode - rconn.hset(fqjobid, "status", "completed:error") - return returncode - except Exception as _exc:# pylint: disable=[broad-except] - logger.error("The process failed!", exc_info=True) - rconn.hset(fqjobid, "status", "completed:error") - return 4 - - return main diff --git a/scripts/rqtl2/install_genotypes.py b/scripts/rqtl2/install_genotypes.py deleted file mode 100644 index 5e6abb0..0000000 --- a/scripts/rqtl2/install_genotypes.py +++ /dev/null @@ -1,261 +0,0 @@ -"""Load genotypes from R/qtl2 bundle into the database.""" -import sys -import argparse -import traceback -from zipfile import ZipFile -from functools import reduce -from typing import Iterator, Optional -from logging import Logger, getLogger - -from redis import Redis -import MySQLdb as mdb -from MySQLdb.cursors import DictCursor - -from r_qtl import r_qtl2 as rqtl2 -from r_qtl import r_qtl2_qc as rqc - -from functional_tools import take - -from scripts.rqtl2.entry import build_main -from scripts.rqtl2.cli_parser import add_common_arguments -from scripts.cli_parser import init_cli_parser, add_global_data_arguments - -logger = getLogger(__name__) - -def insert_markers( - dbconn: mdb.Connection, - speciesid: int, - markers: tuple[str, ...], - pmapdata: Optional[Iterator[dict]], - _logger: Logger -) -> int: - """Insert genotype and genotype values into the database.""" - mdata = reduce(#type: ignore[var-annotated] - lambda acc, row: ({#type: ignore[arg-type, return-value] - **acc, row["id"]: { - key: val - for key,val in row.items() - if key != "id" - } - }), - (pmapdata or tuple()), - {}) - with dbconn.cursor() as cursor: - cursor.executemany( - "INSERT INTO Geno(SpeciesId, Name, Marker_Name, Chr, Mb) " - "VALUES (%(speciesid)s, %(marker)s, %(marker)s, %(chr)s, %(pos)s) " - "ON DUPLICATE KEY UPDATE SpeciesId=SpeciesId", - tuple({ - (speciesid, marker): { - "speciesid": speciesid, - "marker": marker, - "chr": mdata.get(marker, {}).get("chr"), - "pos": mdata.get(marker, {}).get("pos") - } for marker in markers}.values())) - return cursor.rowcount - -def insert_individuals( - dbconn: mdb.Connection, - speciesid: int, - individuals: tuple[str, ...], - _logger: Logger -) -> int: - """Insert individuals/samples into the database.""" - with dbconn.cursor() as cursor: - cursor.executemany( - "INSERT INTO Strain(Name, Name2, SpeciesId) " - "VALUES(%(id)s, %(id)s, %(speciesid)s) " - "ON DUPLICATE KEY UPDATE Name=Name", - tuple({"speciesid": speciesid, "id": individual} - for individual in individuals)) - return cursor.rowcount - -def cross_reference_individuals( - dbconn: mdb.Connection, - speciesid: int, - populationid: int, - individuals: tuple[str, ...], - _logger: Logger -) -> int: - """Cross reference any inserted individuals.""" - with dbconn.cursor(cursorclass=DictCursor) as cursor: - paramstr = ", ".join(["%s"] * len(individuals)) - cursor.execute("SELECT Id FROM Strain " - f"WHERE SpeciesId=%s AND Name IN ({paramstr})", - (speciesid,) + individuals) - ids = ({"popid": populationid, "indid": row["Id"]} - for row in cursor.fetchall()) - cursor.executemany( - "INSERT INTO StrainXRef(InbredSetId, StrainId) " - "VALUES(%(popid)s, %(indid)s) " - "ON DUPLICATE KEY UPDATE InbredSetId=InbredSetId", - tuple(ids)) - return cursor.rowcount - -def insert_genotype_data( - dbconn: mdb.Connection, - speciesid: int, - genotypes: tuple[dict, ...], - individuals: tuple[str, ...], - _logger: Logger -) -> tuple[int, tuple[dict, ...]]: - """Insert the genotype data values into the database.""" - with dbconn.cursor(cursorclass=DictCursor) as cursor: - paramstr = ", ".join(["%s"] * len(individuals)) - cursor.execute( - f"SELECT Id, Name FROM Strain WHERE Name IN ({paramstr})", - individuals) - indids = {row["Name"]: row["Id"] for row in cursor.fetchall()} - markers = tuple(key for key in genotypes[0].keys() if key != "id") - paramstr = ", ".join(["%s"] * len(markers)) - cursor.execute( - f"SELECT Id, Name FROM Geno WHERE SpeciesId=%s AND Name IN ({paramstr})", - (speciesid,)+markers) - markerids = {row["Name"]: row["Id"] for row in cursor.fetchall()} - cursor.execute("SELECT MAX(Id) AS lastid FROM GenoData") - lastid = cursor.fetchone()["lastid"] - data = tuple(item for item in ( - {**row, "gid": gid} - for gid, row in enumerate( - ({ - "value": innerrow[marker], - "indid": indids[innerrow["id"]], - "markerid": markerids[marker] - } for innerrow in genotypes for marker in - (key for key in innerrow.keys() if key != "id")), - start=lastid+1)) - if item["value"] is not None) - cursor.executemany( - "INSERT INTO GenoData(Id, StrainId, value) " - "VALUES(%(gid)s, %(indid)s, %(value)s)", - data) - return cursor.rowcount, tuple({ - "dataid": row["gid"], - "markerid": row["markerid"] - } for row in data) - -def cross_reference_genotypes( - dbconn: mdb.Connection, - speciesid: int, - datasetid: int, - dataids: tuple[dict, ...], - gmapdata: Optional[Iterator[dict]], - _logger: Logger -) -> int: - """Cross-reference the data to the relevant dataset.""" - _rows, markers, mdata = reduce(#type: ignore[var-annotated] - lambda acc, row: (#type: ignore[return-value,arg-type] - (acc[0] + (row,)), (acc[1] + (row["id"],)), - { - **acc[2], row["id"]: { - key: val - for key,val in row.items() - if key != "id" - } - }), - (gmapdata or tuple()), - (tuple(), tuple(), {})) - - with dbconn.cursor(cursorclass=DictCursor) as cursor: - markersdict = {} - if len(markers) > 0: - paramstr = ", ".join(["%s"] * len(markers)) - insertparams = (speciesid,) + markers - selectquery = ("SELECT Id, Name FROM Geno " - f"WHERE SpeciesId=%s AND Name IN ({paramstr})") - _logger.debug( - "The select query was\n\t%s\n\nwith the parameters\n\t%s", - selectquery, - (speciesid,) + markers) - cursor.execute(selectquery, insertparams) - markersdict = {row["Id"]: row["Name"] for row in cursor.fetchall()} - - insertquery = ( - "INSERT INTO GenoXRef(GenoFreezeId, GenoId, DataId, cM) " - "VALUES(%(datasetid)s, %(markerid)s, %(dataid)s, %(pos)s) " - "ON DUPLICATE KEY UPDATE GenoFreezeId=GenoFreezeId") - insertparams = tuple({ - **row, - "datasetid": datasetid, - "pos": mdata.get(markersdict.get( - row.get("markerid"), "nosuchkey"), {}).get("pos") - } for row in dataids) - _logger.debug( - "The insert query was\n\t%s\n\nwith the parameters\n\t%s", - insertquery, insertparams) - cursor.executemany(insertquery, insertparams) - return cursor.rowcount - -def install_genotypes(#pylint: disable=[too-many-locals] - rconn: Redis,#pylint: disable=[unused-argument] - dbconn: mdb.Connection, - fullyqualifiedjobid: str,#pylint: disable=[unused-argument] - args: argparse.Namespace, - logger: Logger = logger # pylint: disable=[redefined-outer-name] -) -> int: - """Load any existing genotypes into the database.""" - (speciesid, populationid, datasetid, rqtl2bundle) = ( - args.speciesid, args.populationid, args.datasetid, args.rqtl2bundle) - count = 0 - with ZipFile(str(rqtl2bundle.absolute()), "r") as zfile: - try: - logger.info("Validating bundle") - rqc.validate_bundle(zfile) - logger.info("Bundle validated successfully.") - logger.info(("Loading genotypes. This could take a while. " - "Please be patient.")) - - cdata = rqtl2.control_data(zfile) - genotypes = rqtl2.genotype_data(zfile) - while True: - batch = tuple(take(genotypes, 1000)) - if len(batch) == 0: - logger.info("Loading Genotypes complete!") - logger.info( - "Total rows processed: %s", count) - break - - insert_markers( - dbconn, - speciesid, - tuple(key for key in batch[0].keys() if key != "id"), - (rqtl2.file_data(zfile, "pmap", cdata) if "pmap" in cdata - else None), - logger) - individuals = tuple(row["id"] for row in batch) - insert_individuals(dbconn, speciesid, individuals, logger) - cross_reference_individuals( - dbconn, speciesid, populationid, individuals, logger) - _num_rows, dataids = insert_genotype_data( - dbconn, speciesid, batch, individuals, logger) - cross_reference_genotypes( - dbconn, - speciesid, - datasetid, - dataids, - (rqtl2.file_data(zfile, "gmap", cdata) - if "gmap" in cdata else None), - logger) - count = count + len(batch) - except rqtl2.InvalidFormat as exc: - logger.error(str(exc)) - logger.info("There are no genotypes to load.") - except Exception as _exc:# pylint: disable=[broad-except] - # Narrow this exception with time. - logger.error("Failing with exception: %s", traceback.format_exc()) - return 3 - - return 0 - -if __name__ == "__main__": - - def cli_args(): - """Process command-line arguments for install_genotypes""" - parser = add_common_arguments(add_global_data_arguments(init_cli_parser( - "install_genotypes", - "Parse genotypes from R/qtl2 bundle into the database."))) - - return parser.parse_args() - - main = build_main(cli_args(), install_genotypes, logger) - sys.exit(main()) diff --git a/scripts/rqtl2/install_phenos.py b/scripts/rqtl2/install_phenos.py deleted file mode 100644 index 11ac8a4..0000000 --- a/scripts/rqtl2/install_phenos.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Load pheno from R/qtl2 bundle into the database.""" -import sys -import argparse -import traceback -from zipfile import ZipFile -from functools import reduce -from logging import Logger, getLogger - -from redis import Redis -import MySQLdb as mdb -from MySQLdb.cursors import DictCursor - -from scripts.rqtl2.entry import build_main -from scripts.rqtl2.cli_parser import add_common_arguments -from scripts.cli_parser import init_cli_parser, add_global_data_arguments - -from r_qtl import r_qtl2 as rqtl2 -from r_qtl import r_qtl2_qc as rqc - -from functional_tools import take - -logger = getLogger(__name__) - -def insert_probesets(dbconn: mdb.Connection, - platformid: int, - phenos: tuple[str, ...]) -> int: - """Insert any new phenos into the database.""" - with dbconn.cursor() as cursor: - cursor.executemany( - "INSERT INTO ProbeSet(ChipId, Name) VALUES(%(chipid)s, %(pheno)s) " - "ON DUPLICATE KEY UPDATE Name=Name", - tuple({"pheno": pheno, "chipid": platformid} for pheno in phenos)) - return cursor.rowcount - -def insert_probeset_data(dbconn: mdb.Connection, - speciesid: int, - phenos: tuple[dict, ...]) -> tuple[ - int, tuple[dict, ...]]: - """Insert numeric (averages) data into the database.""" - with dbconn.cursor(cursorclass=DictCursor) as cursor: - _other, samplenames = reduce(#type: ignore[var-annotated] - lambda acc, row: (#type: ignore[arg-type, return-value] - (acc[0] + (row,)), - (acc[1] + (row["id"],))), - phenos, - (tuple(), tuple())) - paramstr = ", ".join(["%s"] * len(samplenames)) - cursor.execute( - "SELECT Id, Name FROM Strain WHERE SpeciesId=%s " - f"AND Name IN ({paramstr})", - (speciesid,) + samplenames) - sampleids = {row["Name"]: row["Id"] for row in cursor.fetchall()} - cursor.execute("SELECT MAX(Id) AS lastid FROM ProbeSetData") - lastid = (cursor.fetchone() or {}).get("lastid", 0) - data = tuple( - {**row, "psdid": lastid + idx} - for idx, row in enumerate( - (item for item in ({ - "sampleid": sampleids[innerrow["id"]], - "value": innerrow[pheno], - "pheno": pheno - } for innerrow in phenos for pheno in - (key for key in innerrow.keys() if key != "id")) - if item["value"] is not None), - start=1)) - cursor.executemany( - "INSERT INTO ProbeSetData(Id, StrainId, value) " - "VALUES(%(psdid)s, %(sampleid)s, %(value)s)", - data) - return cursor.rowcount, tuple({ - "dataid": row["psdid"], - "pheno": row["pheno"] - } for row in data) - -def cross_reference_probeset_data(dbconn: mdb.Connection, - platformid: int, - datasetid: int, - phenos: tuple[str, ...], - dataids: tuple[dict, ...]) -> int: - """ - Cross-reference the data with the dataset and related ProbeSet phenotype. - """ - with dbconn.cursor(cursorclass=DictCursor) as cursor: - paramstr = ", ".join(["%s"] * len(phenos)) - cursor.execute("SELECT Id, Name FROM ProbeSet WHERE ChipId=%s " - f"AND Name IN ({paramstr})", - (platformid,) + phenos) - phenoids = {row["Name"]: row["Id"] for row in cursor.fetchall()} - cursor.executemany( - "INSERT INTO ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) " - "VALUES(%(datasetid)s, %(phenoid)s, %(dataid)s)", - tuple({ - **row, - "datasetid": datasetid, - "phenoid": phenoids.get(row["pheno"]) - } for row in dataids)) - return cursor.rowcount - -def install_pheno_files(#pylint: disable=[too-many-locals] - rconn: Redis,#pylint: disable=[unused-argument] - dbconn: mdb.Connection, - fullyqualifiedjobid: str,#pylint: disable=[unused-argument] - args: argparse.Namespace, - logger: Logger = logger # pylint: disable=[redefined-outer-name] -) -> int: - """Load data in `pheno` files and other related files into the database.""" - (speciesid, platformid, datasetid, rqtl2bundle) = ( - args.speciesid, args.platformid, args.datasetid, args.rqtl2bundle) - with ZipFile(str(rqtl2bundle), "r") as zfile: - try: - rqc.validate_bundle(zfile) - logger.info("R/qtl2 bundle validated successfully.") - - cdata = rqtl2.control_data(zfile) - phenodata = rqtl2.file_data(zfile, "pheno", cdata) - phenorows = 0 - datarows = 0 - while True: - batch = tuple(take(phenodata, 1000)) - if len(batch) == 0: - logger.info("pheno data loading complete!") - logger.info("Inserted a total of %s new ProbeSets and %s " - "new data rows.", - phenorows, - datarows) - break - - phenos = tuple(key for key in batch[0].keys() if key != "id") - count = insert_probesets(dbconn, platformid, phenos) - phenorows += count - logger.info("Inserted %s new ProbeSets", count) - - count, dataids = insert_probeset_data(dbconn, speciesid, batch) - datarows += count - logger.info("Inserted %s new data rows", count) - - cross_reference_probeset_data( - dbconn, platformid, datasetid, phenos, dataids) - except rqtl2.InvalidFormat as _exc: - logger.error(traceback.format_exc()) - logger.debug("There is no pheno file in the R/qtl2 bundle.") - except Exception as _exc:# pylint: disable=[broad-except] - logger.error("Failing with the exception: %s", traceback.format_exc()) - return 3 - return 0 - -if __name__ == "__main__": - - def cli_args(): - """Process command-line arguments for `install_phenos`""" - parser = add_global_data_arguments(init_cli_parser( - "install_genotypes", - "Parse genotypes from R/qtl2 bundle into the database.")) - - parser.add_argument( - "platformid", - help="The platform from which the data was generated.") - - parser = add_common_arguments(parser) - - return parser.parse_args() - - main = build_main(cli_args(), install_pheno_files, logger) - sys.exit(main()) diff --git a/scripts/rqtl2/phenotypes_qc.py b/scripts/rqtl2/phenotypes_qc.py deleted file mode 100644 index 943cd68..0000000 --- a/scripts/rqtl2/phenotypes_qc.py +++ /dev/null @@ -1,528 +0,0 @@ -"""Run quality control on phenotypes-specific files in the bundle.""" -import sys -import uuid -import json -import shutil -import logging -import tempfile -import contextlib -from pathlib import Path -from logging import Logger -from zipfile import ZipFile -from argparse import Namespace -import multiprocessing as mproc -from functools import reduce, partial -from typing import Union, Iterator, Callable, Optional, Sequence - -import MySQLdb as mdb -from redis import Redis - -from r_qtl import r_qtl2 as rqtl2 -from r_qtl import r_qtl2_qc as rqc -from r_qtl import exceptions as rqe -from r_qtl.fileerrors import InvalidValue - -from functional_tools import chain - -from quality_control.checks import decimal_places_pattern - -from uploader.csvprocessing import read_csv_file -from uploader.files import sha256_digest_over_file -from uploader.samples.models import samples_by_species_and_population - -from scripts.rqtl2.entry import build_main -from scripts.redis_logger import RedisMessageListHandler -from scripts.rqtl2.cli_parser import add_bundle_argument -from scripts.cli_parser import init_cli_parser, add_global_data_arguments -from scripts.rqtl2.bundleutils import build_line_joiner, build_line_splitter - -__MODULE__ = "scripts.rqtl2.phenotypes_qc" -logging.basicConfig( - format=("%(asctime)s - %(levelname)s %(name)s: " - "(%(pathname)s: %(lineno)d) %(message)s")) -logger = logging.getLogger(__MODULE__) - -def validate( - phenobundle: Path, - logger: Logger# pylint: disable=[redefined-outer-name] -) -> dict: - """Check that the bundle is generally valid""" - try: - rqc.validate_bundle(phenobundle) - except rqe.RQTLError as rqtlerr: - # logger.error("Bundle file validation failed!", exc_info=True) - return { - "skip": True, - "logger": logger, - "phenobundle": phenobundle, - "errors": (" ".join(rqtlerr.args),) - } - return { - "errors": tuple(), - "skip": False, - "phenobundle": phenobundle, - "logger": logger - } - - -def check_for_mandatory_pheno_keys( - phenobundle: Path, - logger: Logger,# pylint: disable=[redefined-outer-name] - **kwargs -) -> dict: - """Check that the mandatory keys exist for phenotypes.""" - if kwargs.get("skip", False): - return { - **kwargs, - "logger": logger, - "phenobundle": phenobundle - } - - _mandatory_keys = ("pheno", "phenocovar") - _cdata = rqtl2.read_control_file(phenobundle) - _errors = kwargs.get("errors", tuple()) + tuple( - f"Expected '{key}' file(s) are not declared in the bundle." - for key in _mandatory_keys if key not in _cdata.keys()) - return { - **kwargs, - "logger": logger, - "phenobundle": phenobundle, - "errors": _errors, - "skip": len(_errors) > 0 - } - - -def check_for_averages_files( - phenobundle: Path, - logger: Logger,# pylint: disable=[redefined-outer-name] - **kwargs -) -> dict: - """Check that averages files appear together""" - if kwargs.get("skip", False): - return { - **kwargs, - "logger": logger, - "phenobundle": phenobundle - } - - _together = (("phenose", "phenonum"), ("phenonum", "phenose")) - _cdata = rqtl2.read_control_file(phenobundle) - _errors = kwargs.get("errors", tuple()) + tuple( - f"'{first}' is defined in the control file but there is no " - f"corresponding '{second}'" - for first, second in _together - if ((first in _cdata.keys()) and (second not in _cdata.keys()))) - return { - **kwargs, - "logger": logger, - "phenobundle": phenobundle, - "errors": _errors, - "skip": len(_errors) > 0 - } - - -def extract_bundle( - bundle: Path, workdir: Path, jobid: uuid.UUID -) -> tuple[Path, tuple[Path, ...]]: - """Extract the bundle.""" - with ZipFile(bundle) as zfile: - extractiondir = workdir.joinpath( - f"{str(jobid)}-{sha256_digest_over_file(bundle)}-{bundle.name}") - return extractiondir, rqtl2.extract(zfile, extractiondir) - - -def undo_transpose(filetype: str, cdata: dict, extractiondir): - """Undo transposition of all files of type `filetype` in thebundle.""" - if len(cdata.get(filetype, [])) > 0 and cdata.get(f"{filetype}_transposed", False): - files = (extractiondir.joinpath(_file) for _file in cdata[filetype]) - for _file in files: - rqtl2.transpose_csv_with_rename( - _file, - build_line_splitter(cdata), - build_line_joiner(cdata)) - - -@contextlib.contextmanager -def redis_logger( - redisuri: str, loggername: str, filename: str, fqkey: str -) -> Iterator[logging.Logger]: - """Build a Redis message-list logger.""" - rconn = Redis.from_url(redisuri, decode_responses=True) - _logger = logging.getLogger(loggername) - _logger.propagate = False - handler = RedisMessageListHandler( - rconn, - fullyqualifiedkey(fqkey, filename))#type: ignore[arg-type] - handler.setFormatter(logging.getLogger().handlers[0].formatter) - _logger.addHandler(handler) - try: - yield _logger - finally: - rconn.close() - - -def push_error(rconn: Redis, fqkey: str, error: InvalidValue) -> InvalidValue: - """Persist the error in redis.""" - rconn.rpush(fqkey, json.dumps(error._asdict())) - return error - - -def file_fqkey(prefix: str, section: str, filepath: Path) -> str: - """Build a files fully-qualified key in a consistent manner""" - return f"{prefix}:{section}:{filepath.name}" - - -def qc_phenocovar_file( - filepath: Path, - redisuri, - fqkey: str, - separator: str, - comment_char: str): - """Check that `phenocovar` files are structured correctly.""" - with (redis_logger( - redisuri, - f"{__MODULE__}.qc_phenocovar_file", - filepath.name, - f"{fqkey}:logs") as _logger, - Redis.from_url(redisuri, decode_responses=True) as rconn): - print("Running QC on file: ", filepath.name) - _csvfile = read_csv_file( - filepath, separator, lambda line: line.startswith(comment_char)) - _headings = tuple(heading.lower() for heading in next(_csvfile)) - _errors: tuple[InvalidValue, ...] = tuple() - save_error = partial( - push_error, rconn, file_fqkey(fqkey, "errors", filepath)) - for heading in ("description", "units"): - if heading not in _headings: - _errors = (save_error(InvalidValue( - filepath.name, - "header row", - "-", - "-", - (f"File {filepath.name} is missing the {heading} heading " - "in the header row/line."))),) - - def collect_errors(errors_and_linecount, line): - _errs, _lc = errors_and_linecount - _logger.info("Testing record '%s'", line[0]) - if len(line) != len(_headings): - _errs = _errs + (save_error(InvalidValue( - filepath.name, - line[0], - "-", - "-", - (f"Record {_lc} in file {filepath.name} has a different " - "number of columns than the number of headings"))),) - _line = dict(zip(_headings, line)) - if not bool(_line.get("description")): - _errs = _errs + ( - save_error(InvalidValue(filepath.name, - _line[_headings[0]], - "description", - _line.get("description"), - "The description is not provided!")),) - - rconn.hset(file_fqkey(fqkey, "metadata", filepath), - mapping={ - "status": "checking", - "linecount": _lc+1, - "total-errors": len(_errs) - }) - return _errs, _lc+1 - - _errors, _linecount = reduce(collect_errors, _csvfile, (_errors, 1)) - rconn.hset(file_fqkey(fqkey, "metadata", filepath), - mapping={ - "status": "completed", - "linecount": _linecount, - "total-errors": len(_errors) - }) - return {filepath.name: {"errors": _errors, "linecount": _linecount}} - - -def merge_dicts(*dicts): - """Merge multiple dicts into a single one.""" - return reduce(lambda merged, dct: {**merged, **dct}, dicts, {}) - - -def decimal_points_error(# pylint: disable=[too-many-arguments,too-many-positional-arguments] - filename: str, - rowtitle: str, - coltitle: str, - cellvalue: str, - message: str, - decimal_places: int = 1 -) -> Optional[InvalidValue]: - """Returns an error if the value does not meet the checks.""" - if not bool(decimal_places_pattern( - decimal_places, allow_negative_values=True).match(cellvalue)): - return InvalidValue(filename, rowtitle, coltitle, cellvalue, message) - return None - - -def integer_error( - filename: str, - rowtitle: str, - coltitle: str, - cellvalue: str, - message: str -) -> Optional[InvalidValue]: - """Returns an error if the value does not meet the checks.""" - try: - value = int(cellvalue) - if value <= 0: - raise ValueError("Must be a non-zero, positive number.") - return None - except ValueError as _verr: - return InvalidValue(filename, rowtitle, coltitle, cellvalue, message) - - -def qc_pheno_file(# pylint: disable=[too-many-locals, too-many-arguments, too-many-positional-arguments] - filepath: Path, - redisuri: str, - fqkey: str, - samples: tuple[str, ...], - phenonames: tuple[str, ...], - separator: str, - comment_char: str, - na_strings: Sequence[str], - error_fn: Callable = decimal_points_error -): - """Run QC/QA on a `pheno` file.""" - with (redis_logger( - redisuri, - f"{__MODULE__}.qc_pheno_file", - filepath.name, - f"{fqkey}:logs") as _logger, - Redis.from_url(redisuri, decode_responses=True) as rconn): - print("Running QC on file: ", filepath.name) - save_error = partial( - push_error, rconn, file_fqkey(fqkey, "errors", filepath)) - _csvfile = read_csv_file( - filepath, separator, lambda line: line.startswith(comment_char)) - _headings: tuple[str, ...] = tuple( - # select lowercase for comparison purposes - heading.lower() for heading in next(_csvfile)) - _errors: tuple[InvalidValue, ...] = tuple() - - _absent = tuple(pheno for pheno in _headings[1:] if pheno - not in tuple( - # lower to have consistent case with headings for - # comparison - phe.lower() for phe in phenonames)) - if len(_absent) > 0: - _errors = _errors + (save_error(InvalidValue( - filepath.name, - "header row", - "-", - ", ".join(_absent), - ("The following trait names/identifiers do not exist in any of " - "the provided descriptions/covariates files: " - f"({', '.join(_absent)})"))),) - - def collect_errors(errors_and_linecount, line): - _errs, _lc = errors_and_linecount - _logger.debug("Checking row %s", line[0]) - if line[0] not in samples: - _errs = _errs + (save_error(InvalidValue( - filepath.name, - line[0], - _headings[0], - line[0], - (f"The sample named '{line[0]}' does not exist in the database. " - "You will need to upload that first."))),) - - for field, value in zip(_headings[1:], line[1:]): - if value in na_strings: - continue - _err = error_fn( - filepath.name, - line[0], - field, - value) - _errs = _errs + ((save_error(_err),) if bool(_err) else tuple()) - - rconn.hset(file_fqkey(fqkey, "metadata", filepath), - mapping={ - "status": "checking", - "linecount": _lc+1, - "total-errors": len(_errs) - }) - return _errs, _lc+1 - - _errors, _linecount = reduce(collect_errors, _csvfile, (_errors, 1)) - rconn.hset(file_fqkey(fqkey, "metadata", filepath), - mapping={ - "status": "completed", - "linecount": _linecount, - "total-errors": len(_errors) - }) - return {filepath.name: {"errors": _errors, "linecount": _linecount}} - - -def phenotype_names(filepath: Path, - separator: str, - comment_char: str) -> tuple[str, ...]: - """Read phenotype names from `phenocovar` file.""" - return reduce(lambda tpl, line: tpl + (line[0],),#type: ignore[arg-type, return-value] - read_csv_file( - filepath, - separator, - lambda line: line.startswith(comment_char)), - tuple())[1:] - -def fullyqualifiedkey( - prefix: str, - rest: Optional[str] = None -) -> Union[Callable[[str], str], str]: - """Compute fully qualified Redis key.""" - if not bool(rest): - return lambda _rest: f"{prefix}:{_rest}" - return f"{prefix}:{rest}" - -def run_qc(# pylint: disable=[too-many-locals] - rconn: Redis, - dbconn: mdb.Connection, - fullyqualifiedjobid: str, - args: Namespace, - logger: Logger = logger # pylint: disable=[redefined-outer-name] -) -> int: - """Run quality control checks on the bundle.""" - print("Beginning the quality assurance checks.") - results = check_for_averages_files( - **check_for_mandatory_pheno_keys( - **validate(args.rqtl2bundle, logger))) - errors = results.get("errors", tuple()) - if len(errors) > 0: - logger.error("We found the following errors:\n%s", - "\n".join(f" - {error}" for error in errors)) - return 1 - # Run QC on actual values - # Steps: - # - Extract file to specific directory - extractiondir, *_bundlefiles = extract_bundle( - args.rqtl2bundle, args.workingdir, args.jobid) - - # - For every pheno, phenocovar, phenose, phenonum file, undo - # transposition where relevant - cdata = rqtl2.control_data(extractiondir) - with mproc.Pool(mproc.cpu_count() - 1) as pool: - pool.starmap( - undo_transpose, - ((ftype, cdata, extractiondir) - for ftype in ("pheno", "phenocovar", "phenose", "phenonum"))) - - # - Fetch samples/individuals from database. - print("Fetching samples/individuals from the database.") - samples = tuple(#type: ignore[var-annotated] - item for item in set(reduce( - lambda acc, item: acc + ( - item["Name"], item["Name2"], item["Symbol"], item["Alias"]), - samples_by_species_and_population( - dbconn, args.speciesid, args.populationid), - tuple())) - if bool(item)) - - # - Check that `description` and `units` is present in phenocovar for - # all phenotypes - rconn.hset(fullyqualifiedjobid, - "fully-qualified-keys:phenocovar", - json.dumps(tuple(f"{fullyqualifiedjobid}:phenocovar:{_file}" - for _file in cdata.get("phenocovar", [])))) - with mproc.Pool(mproc.cpu_count() - 1) as pool: - print("Check for errors in 'phenocovar' file(s).") - _phenocovar_qc_res = merge_dicts(*pool.starmap(qc_phenocovar_file, tuple( - (extractiondir.joinpath(_file), - args.redisuri, - f"{fullyqualifiedjobid}:phenocovar", - cdata["sep"], - cdata["comment.char"]) - for _file in cdata.get("phenocovar", [])))) - - # - Check all samples in pheno files exist in database - # - Check all phenotypes in pheno files exist in phenocovar files - # - Check all numeric values in pheno files - phenonames = tuple(set( - name for names in pool.starmap(phenotype_names, tuple( - (extractiondir.joinpath(_file), cdata["sep"], cdata["comment.char"]) - for _file in cdata.get("phenocovar", []))) - for name in names)) - - dec_err_fn = partial(decimal_points_error, message=( - "Expected a non-negative number with at least one decimal " - "place.")) - - print("Check for errors in 'pheno' file(s).") - _pheno_qc_res = merge_dicts(*pool.starmap(qc_pheno_file, tuple(( - extractiondir.joinpath(_file), - args.redisuri, - chain( - "pheno", - fullyqualifiedkey(args.jobid), - fullyqualifiedkey(args.redisprefix)), - samples, - phenonames, - cdata["sep"], - cdata["comment.char"], - cdata["na.strings"], - dec_err_fn - ) for _file in cdata.get("pheno", [])))) - - # - Check the 3 checks above for phenose and phenonum values too - # qc_phenose_files(…) - # qc_phenonum_files(…) - print("Check for errors in 'phenose' file(s).") - _phenose_qc_res = merge_dicts(*pool.starmap(qc_pheno_file, tuple(( - extractiondir.joinpath(_file), - args.redisuri, - chain( - "phenose", - fullyqualifiedkey(args.jobid), - fullyqualifiedkey(args.redisprefix)), - samples, - phenonames, - cdata["sep"], - cdata["comment.char"], - cdata["na.strings"], - dec_err_fn - ) for _file in cdata.get("phenose", [])))) - - print("Check for errors in 'phenonum' file(s).") - _phenonum_qc_res = merge_dicts(*pool.starmap(qc_pheno_file, tuple(( - extractiondir.joinpath(_file), - args.redisuri, - chain( - "phenonum", - fullyqualifiedkey(args.jobid), - fullyqualifiedkey(args.redisprefix)), - samples, - phenonames, - cdata["sep"], - cdata["comment.char"], - cdata["na.strings"], - partial(integer_error, message=( - "Expected a non-negative, non-zero integer value.")) - ) for _file in cdata.get("phenonum", [])))) - - # - Delete all extracted files - shutil.rmtree(extractiondir) - return 0 - - -if __name__ == "__main__": - def cli_args(): - """Process command-line arguments for `install_phenos`""" - parser = add_bundle_argument(add_global_data_arguments(init_cli_parser( - program="PhenotypesQC", - description=( - "Perform Quality Control checks on a phenotypes bundle file")))) - parser.add_argument( - "--workingdir", - default=f"{tempfile.gettempdir()}/phenotypes_qc", - help=("The directory where this script will put its intermediate " - "files."), - type=Path) - return parser.parse_args() - - main = build_main(cli_args(), run_qc, logger) - sys.exit(main()) diff --git a/scripts/run_qtlreaper.py b/scripts/run_qtlreaper.py deleted file mode 100644 index a461d9a..0000000 --- a/scripts/run_qtlreaper.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Script to run rust-qtlreaper and update database with results.""" -import os -import sys -import csv -import time -import secrets -import logging -import subprocess -import multiprocessing -from pathlib import Path -from functools import reduce -from typing import Union, Iterator -from argparse import Namespace, ArgumentParser - -from gn_libs import mysqldb - -from uploader.phenotypes.models import phenotypes_vector_data -from uploader.population.models import population_by_species_and_id -from uploader.samples.models import samples_by_species_and_population - -from scripts.cli.logging import setup_logging -from scripts.cli.validators import directory_exists -from scripts.cli.options import add_logging, add_mariadb_uri, add_population_id - -logger = logging.getLogger(__name__) - - -def retrieve_genotype_file(genotypes_dir: Path, population_code: str) -> Path: - """Retrieves the genotype file""" - _genofile = genotypes_dir.joinpath(f"{population_code}.geno") - if _genofile.exists(): - return _genofile - raise FileNotFoundError(f"Could not find the genotype file '{population_code}.geno'") - - -def samples_from_genofile(genofile: Path) -> tuple[str, ...]: - """Read samples from the genotype file.""" - with genofile.open(mode="r", encoding="utf-8") as inptr: - while True: - line = inptr.readline() - if (line.startswith("#") # comment line - or line.startswith("@") # allele? heterozygosity? - or line.strip() == "" # empty line - ): - continue - return tuple(line.strip().split("\t")[4:]) - - -def reconcile_samples( - genosamples: tuple[str, ...], - dbsamples: tuple[str, ...] -) -> tuple[tuple[str, ...], tuple[str, ...]]: - """merge samples in genosamples and dbsamples and retain order in genosamples.""" - in_db_not_geno = set(dbsamples).difference(genosamples) - return genosamples, tuple(in_db_not_geno) - - -def generate_qtlreaper_traits_file( - outdir: Path, - samples: tuple[str, ...], - traits_data: tuple[dict[str, Union[int, float]], ...], - filename_prefix: str = "" -) -> Path: - """Generate a file for use with qtlreaper that contains the traits' data.""" - _dialect = csv.unix_dialect() - _dialect.delimiter="\t" - _dialect.quoting=0 - - _traitsfile = outdir.joinpath( - f"{filename_prefix}_{secrets.token_urlsafe(15)}.tsv")#type: ignore[attr-defined] - with _traitsfile.open(mode="w", encoding="utf-8") as outptr: - writer = csv.DictWriter( - outptr, fieldnames=("Trait",) + samples, dialect=_dialect) - writer.writeheader() - for row in traits_data: - writer.writerow({ - "Trait": row["xref_id"], - **{sample: row.get(sample, "") for sample in samples} - }) - - return _traitsfile - - -def parse_tsv_file(results_file: Path) -> Iterator[dict]: - """Parse the rust-qtlreaper output into usable python objects.""" - with results_file.open("r", encoding="utf-8") as readptr: - _dialect = csv.unix_dialect() - _dialect.delimiter = "\t" - reader = csv.DictReader(readptr, dialect=_dialect) - yield from reader - - -def __qtls_by_trait__(qtls, current): - """Organise QTL results by trait""" - return { - **qtls, - current["ID"]: qtls.get(current["ID"], tuple()) + (current,) - } - - -def save_qtl_values_to_db(conn, qtls: tuple[dict, ...]): - """Save computed QTLs to the database.""" - with conn.cursor() as cursor: - cursor.executemany( - "UPDATE PublishXRef SET " - "Locus=%(Locus)s, LRS=%(LRS)s, additive=%(Additive)s " - "WHERE Id=%(ID)s", - qtls) - - -def dispatch(args: Namespace) -> int:# pylint: disable=[too-many-locals] - """Dispatch the actual logic.""" - exitcode = 1 - with mysqldb.database_connection(args.db_uri) as conn: - try: - population = population_by_species_and_id(conn, args.species_id, args.population_id) - assert population, (f"No population with ID '{args.population_id} for " - f"species with ID '{args.species_id}'.") - _genofile = retrieve_genotype_file(args.genotypes_dir, population["Name"]) - logger.debug("Genotype file: %s", _genofile) - samples, _samples_not_in_genofile = reconcile_samples( - samples_from_genofile(_genofile), - tuple( - sample["Name"] for sample in - samples_by_species_and_population( - conn, args.species_id, args.population_id))) - if len(_samples_not_in_genofile) > 0: - logger.warning( - "Ignoring %d samples that are in the database but not in " - "the provided genotype file.", - len(_samples_not_in_genofile)) - logger.debug("Ignored the following samples: %s", - ", ".join(_samples_not_in_genofile)) - - # Fetch traits data: provided list, or all traits in db - _traitsdata = tuple(phenotypes_vector_data( - conn, - args.species_id, - args.population_id, - xref_ids=tuple(args.xref_ids)).values()) - logger.debug("Successfully got traits data. Generating the QTLReaper's traits file…") - _traitsfile = generate_qtlreaper_traits_file( - args.working_dir, - samples, - _traitsdata, - filename_prefix="qtlreaper_input_traits_file") - logger.debug("QTLReaper's Traits file: %s", _traitsfile) - - _qtlreaper_main_output = args.working_dir.joinpath( - f"main-output-{secrets.token_urlsafe(15)}.tsv")#type: ignore[attr-defined] - _qtlreaper_permu_output = args.working_dir.joinpath( - f"permu-output-{secrets.token_urlsafe(15)}.tsv") - logger.debug("Main output filename: %s", _qtlreaper_main_output) - with subprocess.Popen( - ("qtlreaper", - "--n_permutations", "1000", - "--geno", _genofile, - "--traits", _traitsfile, - "--main_output", _qtlreaper_main_output, - "--permu_output", _qtlreaper_permu_output, - "--threads", str(int(1+(multiprocessing.cpu_count()/2)))), - env=({**os.environ, "RUST_BACKTRACE": "full"} - if logger.getEffectiveLevel() == logging.DEBUG - else dict(os.environ))) as _qtlreaper: - while _qtlreaper.poll() is None: - logger.debug("QTLReaper process running…") - time.sleep(1) - results = ( - tuple(#type: ignore[var-annotated] - max(qtls, key=lambda qtl: qtl["LRS"]) - for qtls in - reduce(__qtls_by_trait__, - parse_tsv_file(_qtlreaper_main_output), - {}).values()) - if _qtlreaper_main_output.exists() - else tuple()) - logger.debug("Cleaning up temporary files.") - - # short-circuits to delete file if exists - if _traitsfile.exists(): - _traitsfile.unlink() - logger.info("Deleted generated traits' file for QTLReaper.") - - if _qtlreaper_main_output.exists(): - _qtlreaper_main_output.unlink() - logger.info("Deleted QTLReaper's main output file.") - - if _qtlreaper_permu_output.exists(): - _qtlreaper_permu_output.unlink() - logger.info("Deleted QTLReaper's permutations file.") - - if _qtlreaper.returncode != 0: - return _qtlreaper.returncode - - save_qtl_values_to_db(conn, results) - logger.info("Successfully computed p values for %s traits.", len(_traitsdata)) - return 0 - except FileNotFoundError as fnf: - logger.error(", ".join(str(arg) for arg in fnf.args), exc_info=False) - except AssertionError as aserr: - logger.error(", ".join(aserr.args), exc_info=False) - except Exception as _exc:# pylint: disable=[broad-exception-caught] - logger.debug("Type of exception: %s", type(_exc)) - logger.error("General exception!", exc_info=True) - - return exitcode - - -if __name__ == "__main__": - def main(): - """run_qtlreaper.py: entry point.""" - parser = add_logging(add_population_id(add_mariadb_uri( - ArgumentParser("run_qtlreaper")))) - parser.add_argument( - "genotypes_dir", - metavar="GENOTYPES-DIRECTORY", - type=directory_exists, - help="Path to directory with the genotypes.") - parser.add_argument( - "working_dir", - metavar="WORKING-DIRECTORY", - type=directory_exists, - help="Directory where the script will write temporary files.") - parser.add_argument( - "xref_ids", - metavar="CROSS-REFERENCE-IDS", - type=int, - nargs="*", - help=("Optional list of specific cross-reference IDs to narrow down" - " to. If provided, QTLReaper will only run against them. " - "If NOT provided, QTLReaper will run against all the traits " - "in the population.")) - args = parser.parse_args() - setup_logging(logger, args.log_level) - - return dispatch(args) - - sys.exit(main()) diff --git a/scripts/validate_file.py b/scripts/validate_file.py deleted file mode 100644 index 52e55ec..0000000 --- a/scripts/validate_file.py +++ /dev/null @@ -1,123 +0,0 @@ -"""External worker script that checks file for correctness""" -import os -import sys -import traceback -from typing import Callable -from zipfile import ZipFile, is_zipfile - -import jsonpickle -from redis import Redis -from redis.exceptions import ConnectionError # pylint: disable=[redefined-builtin] -from gn_libs.mysqldb import database_connection - -from quality_control.utils import make_progress_calculator -from quality_control.parsing import FileType, strain_names, collect_errors - -from uploader import jobs - -from .cli_parser import init_cli_parser -from .qc import add_file_validation_arguments - - -def make_progress_indicator(redis_connection: Redis, - fqjobid: str, - progress_calc_fn: Callable) -> Callable: - """Make function that will compute the progress and update redis""" - def __indicator__(linenumber, linetext): - progress = progress_calc_fn(linenumber, linetext) - redis_connection.hset(name=fqjobid, mapping=progress._asdict()) - - return progress - - return __indicator__ - -def cli_args_valid(args): - "Check that the command-line arguments are provided and correct" - if not os.path.exists(args.filepath): - print(f"The file '{args.filepath}' does not exist.", file=sys.stderr) - return None - - try: - _conn = Redis.from_url(args.redisuri) - except ConnectionError as _conn_err: - print(traceback.format_exc(), file=sys.stderr) - return None - - return args - -def process_cli_arguments(): - """Setup command-line parser""" - parser = init_cli_parser( - "validate-file", - ("Verify that the file with the expression data conforms to " - "expectations.")) - parser.add_argument("speciesid", - type=int, - help="Species for which the data is to be processed.") - parser = add_file_validation_arguments(parser) - - return cli_args_valid(parser.parse_args()) - -def stream_error(rconn: Redis, fqjobid: str, error): - """Update redis with the most current error(s) found""" - errors = jsonpickle.decode( - rconn.hget(fqjobid, key="errors") or jsonpickle.encode(tuple())) - rconn.hset( - fqjobid, key="errors", value=jsonpickle.encode(errors + (error,))) - -def make_user_aborted(rconn: Redis, fqjobid: str): - """Mkae function that checks whether the user aborted the process""" - def __aborted__(): - user_aborted = bool(int( - rconn.hget(name=fqjobid, key="user_aborted") or "0")) - if user_aborted: - rconn.hset(name=fqjobid, key="status", value="aborted") - - return user_aborted - return __aborted__ - -def get_zipfile_size(filepath): - "Compute size of given zipfile" - with ZipFile(filepath, "r") as zfile: - return zfile.infolist()[0].file_size - -def main(): - "entry point to the script" - args = process_cli_arguments() - if args is None: - print("Quiting due to errors!", file=sys.stderr) - return 1 - - with (Redis.from_url(args.redisuri) as rconn, - database_connection(args.databaseuri) as dbconn): - fqjobid = jobs.job_key(args.redisprefix, args.jobid) - progress_calculator = make_progress_calculator( - get_zipfile_size(args.filepath) if is_zipfile(args.filepath) - else os.stat(args.filepath).st_size) - progress_indicator = make_progress_indicator( - rconn, fqjobid, progress_calculator) - - rconn.hset(fqjobid, key="status", value="Processing") - rconn.hset(fqjobid, key="message", value="Collecting errors") - - error_count = 0 - for error in collect_errors(args.filepath, - (FileType.AVERAGE - if args.filetype == "average" - else FileType.STANDARD_ERROR), - strain_names(dbconn, args.speciesid), - progress_indicator, - make_user_aborted(rconn, fqjobid)): - stream_error(rconn, fqjobid, error) - - if args.count > 0: - error_count = error_count + 1 - if error_count >= args.count: - break - - rconn.hset(name=fqjobid, key="status", value="success") - - return 0 - -if __name__ == "__main__": - main() diff --git a/scripts/worker.py b/scripts/worker.py deleted file mode 100644 index 3165fe7..0000000 --- a/scripts/worker.py +++ /dev/null @@ -1,88 +0,0 @@ -"Generic worker script that runs actual worker script" -import os -import sys -import shlex -import argparse -import traceback -import subprocess -from time import sleep -from datetime import timedelta -from tempfile import TemporaryDirectory - -from redis import Redis - -from uploader import jobs -from uploader.check_connections import check_redis - -def parse_args(): - "Parse the command-line arguments" - parser = argparse.ArgumentParser( - prog="worker", description = ( - "Generic worker to launch and manage specific worker scripts")) - parser.add_argument( - "redisurl", default="redis:///", help="URL to the redis server") - parser.add_argument( - "redisprefix", type=str, help="The prefix before the job ID.") - parser.add_argument("job_id", help="The id of the job being processed") - - args = parser.parse_args() - check_redis(args.redisurl) - - return args - -def run_job(rconn: Redis, job: dict, redisprefix: str): - "Run the actual job." - try: - jobid = job["jobid"] - with TemporaryDirectory() as tmpdir: - stderrpath = f"{tmpdir}/{jobid}.stderr" - with open(stderrpath, "w+b") as tmpfl: - with subprocess.Popen( - shlex.split(job["command"]), stdout=subprocess.PIPE, - stderr=tmpfl) as process: - while process.poll() is None: - jobs.update_status(rconn, redisprefix, jobid, "running") - jobs.update_stdout_stderr(rconn, - redisprefix, - jobid, - process.stdout.read1(),#type: ignore[union-attr] - "stdout") - sleep(1) - - jobs.update_status( - rconn, - redisprefix, - jobid, - ("error" if process.returncode != 0 else "success")) - - with open(stderrpath, "rb") as stderr: - stderr_content = stderr.read() - jobs.update_stdout_stderr(rconn, redisprefix, jobid, stderr_content, "stderr") - - os.remove(stderrpath) - return process.poll() - except Exception as exc:# pylint: disable=[broad-except,unused-variable] - jobs.update_status(rconn, redisprefix, jobid, "error") - jobs.update_stdout_stderr( - rconn, redisprefix, jobid, traceback.format_exc().encode("utf-8"), "stderr") - print(traceback.format_exc(), file=sys.stderr) - return 4 - -def main(): - "Entry point function" - args = parse_args() - with Redis.from_url(args.redisurl, decode_responses=True) as rconn: - job = jobs.job(rconn, args.redisprefix, args.job_id) - if job: - return run_job(rconn, job, args.redisprefix) - jobs.update_status(rconn, args.redisprefix, args.job_id, "error") - fqjobid = jobs.job_key(args.redisprefix, args.jobid) - rconn.hset(fqjobid, "stderr", f"No such job. '{args.job_id}'.") - rconn.expire(name=jobs.job_key(args.redisprefix, args.job_id), - time=timedelta(seconds=2 * 60 * 60)) - print(f"No such job. '{args.job_id}'.", file=sys.stderr) - return 2 - return 3 - -if __name__ == "__main__": - sys.exit(main()) |
