diff options
| author | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-11-11 17:48:53 +0000 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-11-11 13:43:09 -0600 |
| commit | f2196e817c7a665c4fb899680ddb2e028d5edfb2 (patch) | |
| tree | 9a4e6972aa7f0c0756af45a7239d2e51c900025e | |
| parent | 75d30d520c3332e08f1200cce56b7da7e4796efc (diff) | |
| download | gn-uploader-f2196e817c7a665c4fb899680ddb2e028d5edfb2.tar.gz | |
Normalise genotype files and move Stage 02 state into the QC job
Add `uploader.genotypes.normalise`, which writes a canonical untransposed matrix (`<stem>__work<suffix>`) next to the original, dropping comments and `@` directives and leaving the original untouched. Ragged or empty matrices raise `NormalisationError`, and `qc.check_file` now handles only the normal orientation. Stage 02 now puts `stage02`, `genotype_collector_job_id`, `file` and `config` in the QC job's metadata at launch, and uses `jobs_by_metadata` to reuse an existing QC job. The completed collector job's metadata is no longer modified. The Stage 03 worker normalises and QCs the working file from its own metadata. Change the placeholder marker ordering to chromosome, then Mb, then cM. Co-authored-by: pi coding agent <pi@pi.dev>
| -rw-r--r-- | uploader/genotypes/normalise.py | 48 | ||||
| -rw-r--r-- | uploader/genotypes/qc.py | 55 | ||||
| -rw-r--r-- | uploader/genotypes/views.py | 34 | ||||
| -rw-r--r-- | uploader/scripts/genotypes/qc.py | 29 |
4 files changed, 97 insertions, 69 deletions
diff --git a/uploader/genotypes/normalise.py b/uploader/genotypes/normalise.py new file mode 100644 index 0000000..795eb63 --- /dev/null +++ b/uploader/genotypes/normalise.py @@ -0,0 +1,48 @@ +"""Normalise genotype files to a single (untransposed) form.""" +from pathlib import Path +from typing import Any, Mapping + +from uploader.csvprocessing import (is_blank_line, make_comment_line_checker, + read_text_file) + + +class NormalisationError(Exception): + """Raised when a genotype file cannot be normalised.""" + + +def work_filepath(original: Path) -> Path: + """Return the canonical working-file path for ``original``.""" + return original.with_name(f"{original.stem}__work{original.suffix}") + + +def normalise_file(original: Path, config: Mapping[str, Any]) -> Path: + """Write a canonical untransposed data matrix for ``original``. + + Comments and ``@`` directives are dropped (Stage 01 already captured them). + The original file is left untouched; the canonical file is written next to + it as ``<stem>__work<suffix>``. + """ + _is_comment = make_comment_line_checker(config["comment.char"]) + + def __is_skippable__(line: str) -> bool: + return (is_blank_line(line) or _is_comment(line) + or line.startswith("@")) + + rows = tuple( + tuple(field.strip() for field in line.split(config["separator"])) + for line in read_text_file(original) + if not __is_skippable__(line.strip())) + if not rows: + raise NormalisationError( + "The genotype file has no data matrix rows to normalise.") + if len({len(row) for row in rows}) != 1: + raise NormalisationError( + "The genotype data matrix has ragged rows: all rows must have the " + "same number of fields.") + + output_rows = tuple(zip(*rows)) if config["transposed"] else rows + outpath = work_filepath(original) + with outpath.open("w", encoding="utf8") as outfile: + for row in output_rows: + outfile.write(config["separator"].join(row) + "\n") + return outpath diff --git a/uploader/genotypes/qc.py b/uploader/genotypes/qc.py index 9c94d50..bf16281 100644 --- a/uploader/genotypes/qc.py +++ b/uploader/genotypes/qc.py @@ -8,7 +8,7 @@ from uploader.csvprocessing import (is_blank_line, make_comment_line_checker, SPECIAL_FIELD_KEYS = ("markers", "chromosome", "cm", "mb") SORT_WARNING = ( "Marker ordering is a placeholder: records are sorted by chromosome, then " - "cM, then Mb. The appropriate ordering has not been established yet." + "Mb, then cM. The appropriate ordering has not been established yet." ) @@ -33,23 +33,19 @@ def sort_markers( markers, key=lambda marker: ( str(marker.get("chromosome", "")), - _position(marker.get("cm")), - _position(marker.get("mb")))) + _position(marker.get("mb")), + _position(marker.get("cm")))) def check_file(filepath, config: Mapping[str, Any], stage02: Mapping[str, Any]) -> dict: - """Check the genotype values in ``filepath`` against the Stage 02 encoding.""" + """Check the genotype values in a normalised file against the encoding.""" _allowed = allowed_symbols(stage02) _is_skippable = _make_is_skippable(config) _splitter = _make_splitter(config) - if config["transposed"]: - errors, checked = _check_transposed( - filepath, config, _allowed, _is_skippable, _splitter) - else: - headers = read_csv_file_headers( - filepath, config["transposed"], config["separator"], _is_skippable) - errors, checked = _check_untransposed( - filepath, config, headers, _allowed, _is_skippable, _splitter) + headers = read_csv_file_headers( + filepath, False, config["separator"], _is_skippable) + errors, checked = _check_untransposed( + filepath, config, headers, _allowed, _is_skippable, _splitter) return {"values_checked": checked, "errors": errors} @@ -133,38 +129,3 @@ def _check_untransposed(# pylint: disable=[too-many-locals, too-many-arguments, checked += 1 _check_value(lineno, marker, header, fields[idx], allowed, errors) return errors, checked - - -def _check_transposed(# pylint: disable=[too-many-locals] - filepath, config, allowed, is_skippable, splitter -) -> tuple[list[dict], int]: - _special = {config[key] for key in SPECIAL_FIELD_KEYS} - markers = _transposed_markers(filepath, config, is_skippable, splitter) - errors: list[dict] = [] - checked = 0 - lineno = 0 - for line in read_text_file(filepath): - lineno += 1 - line = line.strip() - if is_skippable(line): - continue - fields = splitter(line) - if len(fields) == 0 or fields[0] in _special: - continue - sample = fields[0] - for idx, value in enumerate(fields[1:]): - marker = markers[idx] if idx < len(markers) else "" - checked += 1 - _check_value(lineno, marker, sample, value, allowed, errors) - return errors, checked - - -def _transposed_markers(filepath, config, is_skippable, splitter) -> tuple[str, ...]: - for line in read_text_file(filepath): - line = line.strip() - if is_skippable(line): - continue - fields = splitter(line) - if fields and fields[0] == config["markers"]: - return fields[1:] - return tuple() diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py index a18f37e..d7b7de0 100644 --- a/uploader/genotypes/views.py +++ b/uploader/genotypes/views.py @@ -107,8 +107,14 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals] flash(str(_exc), "alert-danger") return redirect(__dataset_uri_()) + _existing_qc_jobs = gnlibs_jobs.jobs_by_metadata( + conn, {"genotype_collector_job_id": str(job_id), + "job-type": "genotype-file-qc"}) + _stage02_json = ( + _existing_qc_jobs[0]["metadata"].get("stage02") + if _existing_qc_jobs else None) try: - _saved = json.loads(job["metadata"].get("stage02") or "{}") + _saved = json.loads(_stage02_json or "{}") except ValueError: _saved = {} @@ -122,27 +128,35 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals] "missing_symbols": parse_missing_values_symbols( request.form.get("missing_symbols", "")) } - gnlibs_jobs.update_metadata( - conn, job_id, "stage02", json.dumps(_saved)) - _stage03_job_id = job["metadata"].get("stage03_job_id") - if _stage03_job_id: + if _existing_qc_jobs: return redirect(url_for( - "background-jobs.job_status", job_id=_stage03_job_id)) + "background-jobs.job_status", + job_id=_existing_qc_jobs[0]["job_id"])) + _file = job["metadata"].get("file") + _config = job["metadata"].get("config") + if not (_file and _config): + flash("The statistics collection job is missing its file " + "details; please upload the file again.", "alert-danger") + return redirect(__dataset_uri_()) + _qc_job_id = uuid.uuid4() _qc_job = gnlibs_jobs.launch_job( gnlibs_jobs.initialise_job( conn, - str(uuid.uuid4()), + _qc_job_id, [sys.executable, "-m", "uploader.scripts.genotypes.qc", - _jobs_db, str(job_id)], + _jobs_db, str(_qc_job_id)], "genotype-file-qc", extra_meta={ + "stage02": json.dumps(_saved), + "genotype_collector_job_id": str(job_id), + "file": _file, + "config": _config, "job_name": ( f"Genotype file QC: " f"{Path(job['metadata'].get('file', '')).name}"), "species_id": species["SpeciesId"], "population_id": population["Id"], "dataset_id": dataset["Id"], - "collector_job_id": str(job_id), "success_handler": ( "uploader.genotypes.views." "genotype_qc_success_handler") @@ -152,8 +166,6 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals] Path(f"{uploads_dir(app)}/job_errors").absolute(), loglevel=logging.getLevelName( app.logger.getEffectiveLevel()).lower()) - gnlibs_jobs.update_metadata( - conn, job_id, "stage03_job_id", str(_qc_job["job_id"])) flash("Study design and genotype encoding saved. Running QC...", "alert-success") return redirect(url_for( diff --git a/uploader/scripts/genotypes/qc.py b/uploader/scripts/genotypes/qc.py index 408c883..0255901 100644 --- a/uploader/scripts/genotypes/qc.py +++ b/uploader/scripts/genotypes/qc.py @@ -1,12 +1,13 @@ -"""Run quality-control checks on a genotype file for a collector job.""" +"""Normalise a genotype file and run QC for a genotype-file-qc job.""" import sys import json import logging +from pathlib import Path from argparse import ArgumentParser from gn_libs import jobs, sqlite3 -from uploader.genotypes import qc +from uploader.genotypes import normalise, qc logger = logging.getLogger(__name__) @@ -15,10 +16,10 @@ def parse_args(): """Parse the command-line arguments.""" parser = ArgumentParser( prog="genotype-file-qc", - description="Run QC on a genotype file for a collector job.") + description="Normalise a genotype file and run QC on it.") parser.add_argument("jobs_db", help="Path to the jobs SQLite database.") parser.add_argument( - "collector_job_id", help="The id of the genotype-file-stats job.") + "qc_job_id", help="The id of the genotype-file-qc job.") parser.add_argument( "--log-level", default="info", choices=("debug", "info", "warning", "error", "critical")) @@ -26,16 +27,22 @@ def parse_args(): def main(): - """Entry point to the genotype file QC.""" + """Normalise the genotype file, then run QC on it.""" args = parse_args() logging.basicConfig(level=getattr(logging, args.log_level.upper())) with sqlite3.connection(args.jobs_db) as conn: - job = jobs.job(conn, args.collector_job_id, fulldetails=True) - metadata = job["metadata"] - result = qc.check_file( - metadata["file"], - json.loads(metadata["config"]), - json.loads(metadata.get("stage02") or "{}")) + job = jobs.job(conn, args.qc_job_id, fulldetails=True) + metadata = job["metadata"] + config = json.loads(metadata["config"]) + stage02 = json.loads(metadata.get("stage02") or "{}") + try: + work_file = normalise.normalise_file( + Path(metadata["file"]), config) + except normalise.NormalisationError as _exc: + print(json.dumps({"errors": [{"message": str(_exc)}]})) + return 1 + result = qc.check_file( + work_file, {**config, "transposed": False}, stage02) print(json.dumps(result)) return 0 if not result["errors"] else 1 |
