diff options
| -rw-r--r-- | uploader/genotypes/normalise.py | 48 | ||||
| -rw-r--r-- | uploader/genotypes/qc.py | 55 | ||||
| -rw-r--r-- | uploader/genotypes/views.py | 34 | ||||
| -rw-r--r-- | uploader/scripts/genotypes/qc.py | 29 |
4 files changed, 97 insertions, 69 deletions
diff --git a/uploader/genotypes/normalise.py b/uploader/genotypes/normalise.py new file mode 100644 index 0000000..795eb63 --- /dev/null +++ b/uploader/genotypes/normalise.py @@ -0,0 +1,48 @@ +"""Normalise genotype files to a single (untransposed) form.""" +from pathlib import Path +from typing import Any, Mapping + +from uploader.csvprocessing import (is_blank_line, make_comment_line_checker, + read_text_file) + + +class NormalisationError(Exception): + """Raised when a genotype file cannot be normalised.""" + + +def work_filepath(original: Path) -> Path: + """Return the canonical working-file path for ``original``.""" + return original.with_name(f"{original.stem}__work{original.suffix}") + + +def normalise_file(original: Path, config: Mapping[str, Any]) -> Path: + """Write a canonical untransposed data matrix for ``original``. + + Comments and ``@`` directives are dropped (Stage 01 already captured them). + The original file is left untouched; the canonical file is written next to + it as ``<stem>__work<suffix>``. + """ + _is_comment = make_comment_line_checker(config["comment.char"]) + + def __is_skippable__(line: str) -> bool: + return (is_blank_line(line) or _is_comment(line) + or line.startswith("@")) + + rows = tuple( + tuple(field.strip() for field in line.split(config["separator"])) + for line in read_text_file(original) + if not __is_skippable__(line.strip())) + if not rows: + raise NormalisationError( + "The genotype file has no data matrix rows to normalise.") + if len({len(row) for row in rows}) != 1: + raise NormalisationError( + "The genotype data matrix has ragged rows: all rows must have the " + "same number of fields.") + + output_rows = tuple(zip(*rows)) if config["transposed"] else rows + outpath = work_filepath(original) + with outpath.open("w", encoding="utf8") as outfile: + for row in output_rows: + outfile.write(config["separator"].join(row) + "\n") + return outpath diff --git a/uploader/genotypes/qc.py b/uploader/genotypes/qc.py index 9c94d50..bf16281 100644 --- a/uploader/genotypes/qc.py +++ b/uploader/genotypes/qc.py @@ -8,7 +8,7 @@ from uploader.csvprocessing import (is_blank_line, make_comment_line_checker, SPECIAL_FIELD_KEYS = ("markers", "chromosome", "cm", "mb") SORT_WARNING = ( "Marker ordering is a placeholder: records are sorted by chromosome, then " - "cM, then Mb. The appropriate ordering has not been established yet." + "Mb, then cM. The appropriate ordering has not been established yet." ) @@ -33,23 +33,19 @@ def sort_markers( markers, key=lambda marker: ( str(marker.get("chromosome", "")), - _position(marker.get("cm")), - _position(marker.get("mb")))) + _position(marker.get("mb")), + _position(marker.get("cm")))) def check_file(filepath, config: Mapping[str, Any], stage02: Mapping[str, Any]) -> dict: - """Check the genotype values in ``filepath`` against the Stage 02 encoding.""" + """Check the genotype values in a normalised file against the encoding.""" _allowed = allowed_symbols(stage02) _is_skippable = _make_is_skippable(config) _splitter = _make_splitter(config) - if config["transposed"]: - errors, checked = _check_transposed( - filepath, config, _allowed, _is_skippable, _splitter) - else: - headers = read_csv_file_headers( - filepath, config["transposed"], config["separator"], _is_skippable) - errors, checked = _check_untransposed( - filepath, config, headers, _allowed, _is_skippable, _splitter) + headers = read_csv_file_headers( + filepath, False, config["separator"], _is_skippable) + errors, checked = _check_untransposed( + filepath, config, headers, _allowed, _is_skippable, _splitter) return {"values_checked": checked, "errors": errors} @@ -133,38 +129,3 @@ def _check_untransposed(# pylint: disable=[too-many-locals, too-many-arguments, checked += 1 _check_value(lineno, marker, header, fields[idx], allowed, errors) return errors, checked - - -def _check_transposed(# pylint: disable=[too-many-locals] - filepath, config, allowed, is_skippable, splitter -) -> tuple[list[dict], int]: - _special = {config[key] for key in SPECIAL_FIELD_KEYS} - markers = _transposed_markers(filepath, config, is_skippable, splitter) - errors: list[dict] = [] - checked = 0 - lineno = 0 - for line in read_text_file(filepath): - lineno += 1 - line = line.strip() - if is_skippable(line): - continue - fields = splitter(line) - if len(fields) == 0 or fields[0] in _special: - continue - sample = fields[0] - for idx, value in enumerate(fields[1:]): - marker = markers[idx] if idx < len(markers) else "" - checked += 1 - _check_value(lineno, marker, sample, value, allowed, errors) - return errors, checked - - -def _transposed_markers(filepath, config, is_skippable, splitter) -> tuple[str, ...]: - for line in read_text_file(filepath): - line = line.strip() - if is_skippable(line): - continue - fields = splitter(line) - if fields and fields[0] == config["markers"]: - return fields[1:] - return tuple() diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py index a18f37e..d7b7de0 100644 --- a/uploader/genotypes/views.py +++ b/uploader/genotypes/views.py @@ -107,8 +107,14 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals] flash(str(_exc), "alert-danger") return redirect(__dataset_uri_()) + _existing_qc_jobs = gnlibs_jobs.jobs_by_metadata( + conn, {"genotype_collector_job_id": str(job_id), + "job-type": "genotype-file-qc"}) + _stage02_json = ( + _existing_qc_jobs[0]["metadata"].get("stage02") + if _existing_qc_jobs else None) try: - _saved = json.loads(job["metadata"].get("stage02") or "{}") + _saved = json.loads(_stage02_json or "{}") except ValueError: _saved = {} @@ -122,27 +128,35 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals] "missing_symbols": parse_missing_values_symbols( request.form.get("missing_symbols", "")) } - gnlibs_jobs.update_metadata( - conn, job_id, "stage02", json.dumps(_saved)) - _stage03_job_id = job["metadata"].get("stage03_job_id") - if _stage03_job_id: + if _existing_qc_jobs: return redirect(url_for( - "background-jobs.job_status", job_id=_stage03_job_id)) + "background-jobs.job_status", + job_id=_existing_qc_jobs[0]["job_id"])) + _file = job["metadata"].get("file") + _config = job["metadata"].get("config") + if not (_file and _config): + flash("The statistics collection job is missing its file " + "details; please upload the file again.", "alert-danger") + return redirect(__dataset_uri_()) + _qc_job_id = uuid.uuid4() _qc_job = gnlibs_jobs.launch_job( gnlibs_jobs.initialise_job( conn, - str(uuid.uuid4()), + _qc_job_id, [sys.executable, "-m", "uploader.scripts.genotypes.qc", - _jobs_db, str(job_id)], + _jobs_db, str(_qc_job_id)], "genotype-file-qc", extra_meta={ + "stage02": json.dumps(_saved), + "genotype_collector_job_id": str(job_id), + "file": _file, + "config": _config, "job_name": ( f"Genotype file QC: " f"{Path(job['metadata'].get('file', '')).name}"), "species_id": species["SpeciesId"], "population_id": population["Id"], "dataset_id": dataset["Id"], - "collector_job_id": str(job_id), "success_handler": ( "uploader.genotypes.views." "genotype_qc_success_handler") @@ -152,8 +166,6 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals] Path(f"{uploads_dir(app)}/job_errors").absolute(), loglevel=logging.getLevelName( app.logger.getEffectiveLevel()).lower()) - gnlibs_jobs.update_metadata( - conn, job_id, "stage03_job_id", str(_qc_job["job_id"])) flash("Study design and genotype encoding saved. Running QC...", "alert-success") return redirect(url_for( diff --git a/uploader/scripts/genotypes/qc.py b/uploader/scripts/genotypes/qc.py index 408c883..0255901 100644 --- a/uploader/scripts/genotypes/qc.py +++ b/uploader/scripts/genotypes/qc.py @@ -1,12 +1,13 @@ -"""Run quality-control checks on a genotype file for a collector job.""" +"""Normalise a genotype file and run QC for a genotype-file-qc job.""" import sys import json import logging +from pathlib import Path from argparse import ArgumentParser from gn_libs import jobs, sqlite3 -from uploader.genotypes import qc +from uploader.genotypes import normalise, qc logger = logging.getLogger(__name__) @@ -15,10 +16,10 @@ def parse_args(): """Parse the command-line arguments.""" parser = ArgumentParser( prog="genotype-file-qc", - description="Run QC on a genotype file for a collector job.") + description="Normalise a genotype file and run QC on it.") parser.add_argument("jobs_db", help="Path to the jobs SQLite database.") parser.add_argument( - "collector_job_id", help="The id of the genotype-file-stats job.") + "qc_job_id", help="The id of the genotype-file-qc job.") parser.add_argument( "--log-level", default="info", choices=("debug", "info", "warning", "error", "critical")) @@ -26,16 +27,22 @@ def parse_args(): def main(): - """Entry point to the genotype file QC.""" + """Normalise the genotype file, then run QC on it.""" args = parse_args() logging.basicConfig(level=getattr(logging, args.log_level.upper())) with sqlite3.connection(args.jobs_db) as conn: - job = jobs.job(conn, args.collector_job_id, fulldetails=True) - metadata = job["metadata"] - result = qc.check_file( - metadata["file"], - json.loads(metadata["config"]), - json.loads(metadata.get("stage02") or "{}")) + job = jobs.job(conn, args.qc_job_id, fulldetails=True) + metadata = job["metadata"] + config = json.loads(metadata["config"]) + stage02 = json.loads(metadata.get("stage02") or "{}") + try: + work_file = normalise.normalise_file( + Path(metadata["file"]), config) + except normalise.NormalisationError as _exc: + print(json.dumps({"errors": [{"message": str(_exc)}]})) + return 1 + result = qc.check_file( + work_file, {**config, "transposed": False}, stage02) print(json.dumps(result)) return 0 if not result["errors"] else 1 |
