about summary refs log tree commit diff
diff options
context:
space:
mode:
authorFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-11-11 17:48:53 +0000
committerFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-11-11 13:43:09 -0600
commitf2196e817c7a665c4fb899680ddb2e028d5edfb2 (patch)
tree9a4e6972aa7f0c0756af45a7239d2e51c900025e
parent75d30d520c3332e08f1200cce56b7da7e4796efc (diff)
downloadgn-uploader-f2196e817c7a665c4fb899680ddb2e028d5edfb2.tar.gz
Normalise genotype files and move Stage 02 state into the QC job
Add `uploader.genotypes.normalise`, which writes a canonical untransposed
matrix (`<stem>__work<suffix>`) next to the original, dropping comments and
`@` directives and leaving the original untouched. Ragged or empty matrices
raise `NormalisationError`, and `qc.check_file` now handles only the normal
orientation.

Stage 02 now puts `stage02`, `genotype_collector_job_id`, `file` and `config`
in the QC job's metadata at launch, and uses `jobs_by_metadata` to reuse an
existing QC job. The completed collector job's metadata is no longer modified.
The Stage 03 worker normalises and QCs the working file from its own metadata.

Change the placeholder marker ordering to chromosome, then Mb, then cM.

Co-authored-by: pi coding agent <pi@pi.dev>
-rw-r--r--uploader/genotypes/normalise.py48
-rw-r--r--uploader/genotypes/qc.py55
-rw-r--r--uploader/genotypes/views.py34
-rw-r--r--uploader/scripts/genotypes/qc.py29
4 files changed, 97 insertions, 69 deletions
diff --git a/uploader/genotypes/normalise.py b/uploader/genotypes/normalise.py
new file mode 100644
index 0000000..795eb63
--- /dev/null
+++ b/uploader/genotypes/normalise.py
@@ -0,0 +1,48 @@
+"""Normalise genotype files to a single (untransposed) form."""
+from pathlib import Path
+from typing import Any, Mapping
+
+from uploader.csvprocessing import (is_blank_line, make_comment_line_checker,
+                                    read_text_file)
+
+
+class NormalisationError(Exception):
+    """Raised when a genotype file cannot be normalised."""
+
+
+def work_filepath(original: Path) -> Path:
+    """Return the canonical working-file path for ``original``."""
+    return original.with_name(f"{original.stem}__work{original.suffix}")
+
+
+def normalise_file(original: Path, config: Mapping[str, Any]) -> Path:
+    """Write a canonical untransposed data matrix for ``original``.
+
+    Comments and ``@`` directives are dropped (Stage 01 already captured them).
+    The original file is left untouched; the canonical file is written next to
+    it as ``<stem>__work<suffix>``.
+    """
+    _is_comment = make_comment_line_checker(config["comment.char"])
+
+    def __is_skippable__(line: str) -> bool:
+        return (is_blank_line(line) or _is_comment(line)
+                or line.startswith("@"))
+
+    rows = tuple(
+        tuple(field.strip() for field in line.split(config["separator"]))
+        for line in read_text_file(original)
+        if not __is_skippable__(line.strip()))
+    if not rows:
+        raise NormalisationError(
+            "The genotype file has no data matrix rows to normalise.")
+    if len({len(row) for row in rows}) != 1:
+        raise NormalisationError(
+            "The genotype data matrix has ragged rows: all rows must have the "
+            "same number of fields.")
+
+    output_rows = tuple(zip(*rows)) if config["transposed"] else rows
+    outpath = work_filepath(original)
+    with outpath.open("w", encoding="utf8") as outfile:
+        for row in output_rows:
+            outfile.write(config["separator"].join(row) + "\n")
+    return outpath
diff --git a/uploader/genotypes/qc.py b/uploader/genotypes/qc.py
index 9c94d50..bf16281 100644
--- a/uploader/genotypes/qc.py
+++ b/uploader/genotypes/qc.py
@@ -8,7 +8,7 @@ from uploader.csvprocessing import (is_blank_line, make_comment_line_checker,
 SPECIAL_FIELD_KEYS = ("markers", "chromosome", "cm", "mb")
 SORT_WARNING = (
     "Marker ordering is a placeholder: records are sorted by chromosome, then "
-    "cM, then Mb. The appropriate ordering has not been established yet."
+    "Mb, then cM. The appropriate ordering has not been established yet."
 )
 
 
@@ -33,23 +33,19 @@ def sort_markers(
         markers,
         key=lambda marker: (
             str(marker.get("chromosome", "")),
-            _position(marker.get("cm")),
-            _position(marker.get("mb"))))
+            _position(marker.get("mb")),
+            _position(marker.get("cm"))))
 
 
 def check_file(filepath, config: Mapping[str, Any], stage02: Mapping[str, Any]) -> dict:
-    """Check the genotype values in ``filepath`` against the Stage 02 encoding."""
+    """Check the genotype values in a normalised file against the encoding."""
     _allowed = allowed_symbols(stage02)
     _is_skippable = _make_is_skippable(config)
     _splitter = _make_splitter(config)
-    if config["transposed"]:
-        errors, checked = _check_transposed(
-            filepath, config, _allowed, _is_skippable, _splitter)
-    else:
-        headers = read_csv_file_headers(
-            filepath, config["transposed"], config["separator"], _is_skippable)
-        errors, checked = _check_untransposed(
-            filepath, config, headers, _allowed, _is_skippable, _splitter)
+    headers = read_csv_file_headers(
+        filepath, False, config["separator"], _is_skippable)
+    errors, checked = _check_untransposed(
+        filepath, config, headers, _allowed, _is_skippable, _splitter)
     return {"values_checked": checked, "errors": errors}
 
 
@@ -133,38 +129,3 @@ def _check_untransposed(# pylint: disable=[too-many-locals, too-many-arguments,
             checked += 1
             _check_value(lineno, marker, header, fields[idx], allowed, errors)
     return errors, checked
-
-
-def _check_transposed(# pylint: disable=[too-many-locals]
-        filepath, config, allowed, is_skippable, splitter
-) -> tuple[list[dict], int]:
-    _special = {config[key] for key in SPECIAL_FIELD_KEYS}
-    markers = _transposed_markers(filepath, config, is_skippable, splitter)
-    errors: list[dict] = []
-    checked = 0
-    lineno = 0
-    for line in read_text_file(filepath):
-        lineno += 1
-        line = line.strip()
-        if is_skippable(line):
-            continue
-        fields = splitter(line)
-        if len(fields) == 0 or fields[0] in _special:
-            continue
-        sample = fields[0]
-        for idx, value in enumerate(fields[1:]):
-            marker = markers[idx] if idx < len(markers) else ""
-            checked += 1
-            _check_value(lineno, marker, sample, value, allowed, errors)
-    return errors, checked
-
-
-def _transposed_markers(filepath, config, is_skippable, splitter) -> tuple[str, ...]:
-    for line in read_text_file(filepath):
-        line = line.strip()
-        if is_skippable(line):
-            continue
-        fields = splitter(line)
-        if fields and fields[0] == config["markers"]:
-            return fields[1:]
-    return tuple()
diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py
index a18f37e..d7b7de0 100644
--- a/uploader/genotypes/views.py
+++ b/uploader/genotypes/views.py
@@ -107,8 +107,14 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals]
             flash(str(_exc), "alert-danger")
             return redirect(__dataset_uri_())
 
+        _existing_qc_jobs = gnlibs_jobs.jobs_by_metadata(
+            conn, {"genotype_collector_job_id": str(job_id),
+                   "job-type": "genotype-file-qc"})
+        _stage02_json = (
+            _existing_qc_jobs[0]["metadata"].get("stage02")
+            if _existing_qc_jobs else None)
         try:
-            _saved = json.loads(job["metadata"].get("stage02") or "{}")
+            _saved = json.loads(_stage02_json or "{}")
         except ValueError:
             _saved = {}
 
@@ -122,27 +128,35 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals]
                 "missing_symbols": parse_missing_values_symbols(
                     request.form.get("missing_symbols", ""))
             }
-            gnlibs_jobs.update_metadata(
-                conn, job_id, "stage02", json.dumps(_saved))
-            _stage03_job_id = job["metadata"].get("stage03_job_id")
-            if _stage03_job_id:
+            if _existing_qc_jobs:
                 return redirect(url_for(
-                    "background-jobs.job_status", job_id=_stage03_job_id))
+                    "background-jobs.job_status",
+                    job_id=_existing_qc_jobs[0]["job_id"]))
+            _file = job["metadata"].get("file")
+            _config = job["metadata"].get("config")
+            if not (_file and _config):
+                flash("The statistics collection job is missing its file "
+                      "details; please upload the file again.", "alert-danger")
+                return redirect(__dataset_uri_())
+            _qc_job_id = uuid.uuid4()
             _qc_job = gnlibs_jobs.launch_job(
                 gnlibs_jobs.initialise_job(
                     conn,
-                    str(uuid.uuid4()),
+                    _qc_job_id,
                     [sys.executable, "-m", "uploader.scripts.genotypes.qc",
-                     _jobs_db, str(job_id)],
+                     _jobs_db, str(_qc_job_id)],
                     "genotype-file-qc",
                     extra_meta={
+                        "stage02": json.dumps(_saved),
+                        "genotype_collector_job_id": str(job_id),
+                        "file": _file,
+                        "config": _config,
                         "job_name": (
                             f"Genotype file QC: "
                             f"{Path(job['metadata'].get('file', '')).name}"),
                         "species_id": species["SpeciesId"],
                         "population_id": population["Id"],
                         "dataset_id": dataset["Id"],
-                        "collector_job_id": str(job_id),
                         "success_handler": (
                             "uploader.genotypes.views."
                             "genotype_qc_success_handler")
@@ -152,8 +166,6 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals]
                 Path(f"{uploads_dir(app)}/job_errors").absolute(),
                 loglevel=logging.getLevelName(
                     app.logger.getEffectiveLevel()).lower())
-            gnlibs_jobs.update_metadata(
-                conn, job_id, "stage03_job_id", str(_qc_job["job_id"]))
             flash("Study design and genotype encoding saved. Running QC...",
                   "alert-success")
             return redirect(url_for(
diff --git a/uploader/scripts/genotypes/qc.py b/uploader/scripts/genotypes/qc.py
index 408c883..0255901 100644
--- a/uploader/scripts/genotypes/qc.py
+++ b/uploader/scripts/genotypes/qc.py
@@ -1,12 +1,13 @@
-"""Run quality-control checks on a genotype file for a collector job."""
+"""Normalise a genotype file and run QC for a genotype-file-qc job."""
 import sys
 import json
 import logging
+from pathlib import Path
 from argparse import ArgumentParser
 
 from gn_libs import jobs, sqlite3
 
-from uploader.genotypes import qc
+from uploader.genotypes import normalise, qc
 
 logger = logging.getLogger(__name__)
 
@@ -15,10 +16,10 @@ def parse_args():
     """Parse the command-line arguments."""
     parser = ArgumentParser(
         prog="genotype-file-qc",
-        description="Run QC on a genotype file for a collector job.")
+        description="Normalise a genotype file and run QC on it.")
     parser.add_argument("jobs_db", help="Path to the jobs SQLite database.")
     parser.add_argument(
-        "collector_job_id", help="The id of the genotype-file-stats job.")
+        "qc_job_id", help="The id of the genotype-file-qc job.")
     parser.add_argument(
         "--log-level", default="info",
         choices=("debug", "info", "warning", "error", "critical"))
@@ -26,16 +27,22 @@ def parse_args():
 
 
 def main():
-    """Entry point to the genotype file QC."""
+    """Normalise the genotype file, then run QC on it."""
     args = parse_args()
     logging.basicConfig(level=getattr(logging, args.log_level.upper()))
     with sqlite3.connection(args.jobs_db) as conn:
-        job = jobs.job(conn, args.collector_job_id, fulldetails=True)
-    metadata = job["metadata"]
-    result = qc.check_file(
-        metadata["file"],
-        json.loads(metadata["config"]),
-        json.loads(metadata.get("stage02") or "{}"))
+        job = jobs.job(conn, args.qc_job_id, fulldetails=True)
+        metadata = job["metadata"]
+        config = json.loads(metadata["config"])
+        stage02 = json.loads(metadata.get("stage02") or "{}")
+        try:
+            work_file = normalise.normalise_file(
+                Path(metadata["file"]), config)
+        except normalise.NormalisationError as _exc:
+            print(json.dumps({"errors": [{"message": str(_exc)}]}))
+            return 1
+        result = qc.check_file(
+            work_file, {**config, "transposed": False}, stage02)
     print(json.dumps(result))
     return 0 if not result["errors"] else 1