From f2196e817c7a665c4fb899680ddb2e028d5edfb2 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Wed, 11 Nov 2026 17:48:53 +0000 Subject: Normalise genotype files and move Stage 02 state into the QC job Add `uploader.genotypes.normalise`, which writes a canonical untransposed matrix (`__work`) next to the original, dropping comments and `@` directives and leaving the original untouched. Ragged or empty matrices raise `NormalisationError`, and `qc.check_file` now handles only the normal orientation. Stage 02 now puts `stage02`, `genotype_collector_job_id`, `file` and `config` in the QC job's metadata at launch, and uses `jobs_by_metadata` to reuse an existing QC job. The completed collector job's metadata is no longer modified. The Stage 03 worker normalises and QCs the working file from its own metadata. Change the placeholder marker ordering to chromosome, then Mb, then cM. Co-authored-by: pi coding agent --- uploader/scripts/genotypes/qc.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) (limited to 'uploader/scripts') diff --git a/uploader/scripts/genotypes/qc.py b/uploader/scripts/genotypes/qc.py index 408c883..0255901 100644 --- a/uploader/scripts/genotypes/qc.py +++ b/uploader/scripts/genotypes/qc.py @@ -1,12 +1,13 @@ -"""Run quality-control checks on a genotype file for a collector job.""" +"""Normalise a genotype file and run QC for a genotype-file-qc job.""" import sys import json import logging +from pathlib import Path from argparse import ArgumentParser from gn_libs import jobs, sqlite3 -from uploader.genotypes import qc +from uploader.genotypes import normalise, qc logger = logging.getLogger(__name__) @@ -15,10 +16,10 @@ def parse_args(): """Parse the command-line arguments.""" parser = ArgumentParser( prog="genotype-file-qc", - description="Run QC on a genotype file for a collector job.") + description="Normalise a genotype file and run QC on it.") parser.add_argument("jobs_db", help="Path to the jobs SQLite database.") parser.add_argument( - "collector_job_id", help="The id of the genotype-file-stats job.") + "qc_job_id", help="The id of the genotype-file-qc job.") parser.add_argument( "--log-level", default="info", choices=("debug", "info", "warning", "error", "critical")) @@ -26,16 +27,22 @@ def parse_args(): def main(): - """Entry point to the genotype file QC.""" + """Normalise the genotype file, then run QC on it.""" args = parse_args() logging.basicConfig(level=getattr(logging, args.log_level.upper())) with sqlite3.connection(args.jobs_db) as conn: - job = jobs.job(conn, args.collector_job_id, fulldetails=True) - metadata = job["metadata"] - result = qc.check_file( - metadata["file"], - json.loads(metadata["config"]), - json.loads(metadata.get("stage02") or "{}")) + job = jobs.job(conn, args.qc_job_id, fulldetails=True) + metadata = job["metadata"] + config = json.loads(metadata["config"]) + stage02 = json.loads(metadata.get("stage02") or "{}") + try: + work_file = normalise.normalise_file( + Path(metadata["file"]), config) + except normalise.NormalisationError as _exc: + print(json.dumps({"errors": [{"message": str(_exc)}]})) + return 1 + result = qc.check_file( + work_file, {**config, "transposed": False}, stage02) print(json.dumps(result)) return 0 if not result["errors"] else 1 -- cgit 1.4.1