about summary refs log tree commit diff
diff options
context:
space:
mode:
-rw-r--r--uploader/genotypes/normalise.py48
-rw-r--r--uploader/genotypes/qc.py55
-rw-r--r--uploader/genotypes/views.py34
-rw-r--r--uploader/scripts/genotypes/qc.py29
4 files changed, 97 insertions, 69 deletions
diff --git a/uploader/genotypes/normalise.py b/uploader/genotypes/normalise.py
new file mode 100644
index 0000000..795eb63
--- /dev/null
+++ b/uploader/genotypes/normalise.py
@@ -0,0 +1,48 @@
+"""Normalise genotype files to a single (untransposed) form."""
+from pathlib import Path
+from typing import Any, Mapping
+
+from uploader.csvprocessing import (is_blank_line, make_comment_line_checker,
+                                    read_text_file)
+
+
+class NormalisationError(Exception):
+    """Raised when a genotype file cannot be normalised."""
+
+
+def work_filepath(original: Path) -> Path:
+    """Return the canonical working-file path for ``original``."""
+    return original.with_name(f"{original.stem}__work{original.suffix}")
+
+
+def normalise_file(original: Path, config: Mapping[str, Any]) -> Path:
+    """Write a canonical untransposed data matrix for ``original``.
+
+    Comments and ``@`` directives are dropped (Stage 01 already captured them).
+    The original file is left untouched; the canonical file is written next to
+    it as ``<stem>__work<suffix>``.
+    """
+    _is_comment = make_comment_line_checker(config["comment.char"])
+
+    def __is_skippable__(line: str) -> bool:
+        return (is_blank_line(line) or _is_comment(line)
+                or line.startswith("@"))
+
+    rows = tuple(
+        tuple(field.strip() for field in line.split(config["separator"]))
+        for line in read_text_file(original)
+        if not __is_skippable__(line.strip()))
+    if not rows:
+        raise NormalisationError(
+            "The genotype file has no data matrix rows to normalise.")
+    if len({len(row) for row in rows}) != 1:
+        raise NormalisationError(
+            "The genotype data matrix has ragged rows: all rows must have the "
+            "same number of fields.")
+
+    output_rows = tuple(zip(*rows)) if config["transposed"] else rows
+    outpath = work_filepath(original)
+    with outpath.open("w", encoding="utf8") as outfile:
+        for row in output_rows:
+            outfile.write(config["separator"].join(row) + "\n")
+    return outpath
diff --git a/uploader/genotypes/qc.py b/uploader/genotypes/qc.py
index 9c94d50..bf16281 100644
--- a/uploader/genotypes/qc.py
+++ b/uploader/genotypes/qc.py
@@ -8,7 +8,7 @@ from uploader.csvprocessing import (is_blank_line, make_comment_line_checker,
 SPECIAL_FIELD_KEYS = ("markers", "chromosome", "cm", "mb")
 SORT_WARNING = (
     "Marker ordering is a placeholder: records are sorted by chromosome, then "
-    "cM, then Mb. The appropriate ordering has not been established yet."
+    "Mb, then cM. The appropriate ordering has not been established yet."
 )
 
 
@@ -33,23 +33,19 @@ def sort_markers(
         markers,
         key=lambda marker: (
             str(marker.get("chromosome", "")),
-            _position(marker.get("cm")),
-            _position(marker.get("mb"))))
+            _position(marker.get("mb")),
+            _position(marker.get("cm"))))
 
 
 def check_file(filepath, config: Mapping[str, Any], stage02: Mapping[str, Any]) -> dict:
-    """Check the genotype values in ``filepath`` against the Stage 02 encoding."""
+    """Check the genotype values in a normalised file against the encoding."""
     _allowed = allowed_symbols(stage02)
     _is_skippable = _make_is_skippable(config)
     _splitter = _make_splitter(config)
-    if config["transposed"]:
-        errors, checked = _check_transposed(
-            filepath, config, _allowed, _is_skippable, _splitter)
-    else:
-        headers = read_csv_file_headers(
-            filepath, config["transposed"], config["separator"], _is_skippable)
-        errors, checked = _check_untransposed(
-            filepath, config, headers, _allowed, _is_skippable, _splitter)
+    headers = read_csv_file_headers(
+        filepath, False, config["separator"], _is_skippable)
+    errors, checked = _check_untransposed(
+        filepath, config, headers, _allowed, _is_skippable, _splitter)
     return {"values_checked": checked, "errors": errors}
 
 
@@ -133,38 +129,3 @@ def _check_untransposed(# pylint: disable=[too-many-locals, too-many-arguments,
             checked += 1
             _check_value(lineno, marker, header, fields[idx], allowed, errors)
     return errors, checked
-
-
-def _check_transposed(# pylint: disable=[too-many-locals]
-        filepath, config, allowed, is_skippable, splitter
-) -> tuple[list[dict], int]:
-    _special = {config[key] for key in SPECIAL_FIELD_KEYS}
-    markers = _transposed_markers(filepath, config, is_skippable, splitter)
-    errors: list[dict] = []
-    checked = 0
-    lineno = 0
-    for line in read_text_file(filepath):
-        lineno += 1
-        line = line.strip()
-        if is_skippable(line):
-            continue
-        fields = splitter(line)
-        if len(fields) == 0 or fields[0] in _special:
-            continue
-        sample = fields[0]
-        for idx, value in enumerate(fields[1:]):
-            marker = markers[idx] if idx < len(markers) else ""
-            checked += 1
-            _check_value(lineno, marker, sample, value, allowed, errors)
-    return errors, checked
-
-
-def _transposed_markers(filepath, config, is_skippable, splitter) -> tuple[str, ...]:
-    for line in read_text_file(filepath):
-        line = line.strip()
-        if is_skippable(line):
-            continue
-        fields = splitter(line)
-        if fields and fields[0] == config["markers"]:
-            return fields[1:]
-    return tuple()
diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py
index a18f37e..d7b7de0 100644
--- a/uploader/genotypes/views.py
+++ b/uploader/genotypes/views.py
@@ -107,8 +107,14 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals]
             flash(str(_exc), "alert-danger")
             return redirect(__dataset_uri_())
 
+        _existing_qc_jobs = gnlibs_jobs.jobs_by_metadata(
+            conn, {"genotype_collector_job_id": str(job_id),
+                   "job-type": "genotype-file-qc"})
+        _stage02_json = (
+            _existing_qc_jobs[0]["metadata"].get("stage02")
+            if _existing_qc_jobs else None)
         try:
-            _saved = json.loads(job["metadata"].get("stage02") or "{}")
+            _saved = json.loads(_stage02_json or "{}")
         except ValueError:
             _saved = {}
 
@@ -122,27 +128,35 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals]
                 "missing_symbols": parse_missing_values_symbols(
                     request.form.get("missing_symbols", ""))
             }
-            gnlibs_jobs.update_metadata(
-                conn, job_id, "stage02", json.dumps(_saved))
-            _stage03_job_id = job["metadata"].get("stage03_job_id")
-            if _stage03_job_id:
+            if _existing_qc_jobs:
                 return redirect(url_for(
-                    "background-jobs.job_status", job_id=_stage03_job_id))
+                    "background-jobs.job_status",
+                    job_id=_existing_qc_jobs[0]["job_id"]))
+            _file = job["metadata"].get("file")
+            _config = job["metadata"].get("config")
+            if not (_file and _config):
+                flash("The statistics collection job is missing its file "
+                      "details; please upload the file again.", "alert-danger")
+                return redirect(__dataset_uri_())
+            _qc_job_id = uuid.uuid4()
             _qc_job = gnlibs_jobs.launch_job(
                 gnlibs_jobs.initialise_job(
                     conn,
-                    str(uuid.uuid4()),
+                    _qc_job_id,
                     [sys.executable, "-m", "uploader.scripts.genotypes.qc",
-                     _jobs_db, str(job_id)],
+                     _jobs_db, str(_qc_job_id)],
                     "genotype-file-qc",
                     extra_meta={
+                        "stage02": json.dumps(_saved),
+                        "genotype_collector_job_id": str(job_id),
+                        "file": _file,
+                        "config": _config,
                         "job_name": (
                             f"Genotype file QC: "
                             f"{Path(job['metadata'].get('file', '')).name}"),
                         "species_id": species["SpeciesId"],
                         "population_id": population["Id"],
                         "dataset_id": dataset["Id"],
-                        "collector_job_id": str(job_id),
                         "success_handler": (
                             "uploader.genotypes.views."
                             "genotype_qc_success_handler")
@@ -152,8 +166,6 @@ def study_design(# pylint: disable=[unused-argument,too-many-locals]
                 Path(f"{uploads_dir(app)}/job_errors").absolute(),
                 loglevel=logging.getLevelName(
                     app.logger.getEffectiveLevel()).lower())
-            gnlibs_jobs.update_metadata(
-                conn, job_id, "stage03_job_id", str(_qc_job["job_id"]))
             flash("Study design and genotype encoding saved. Running QC...",
                   "alert-success")
             return redirect(url_for(
diff --git a/uploader/scripts/genotypes/qc.py b/uploader/scripts/genotypes/qc.py
index 408c883..0255901 100644
--- a/uploader/scripts/genotypes/qc.py
+++ b/uploader/scripts/genotypes/qc.py
@@ -1,12 +1,13 @@
-"""Run quality-control checks on a genotype file for a collector job."""
+"""Normalise a genotype file and run QC for a genotype-file-qc job."""
 import sys
 import json
 import logging
+from pathlib import Path
 from argparse import ArgumentParser
 
 from gn_libs import jobs, sqlite3
 
-from uploader.genotypes import qc
+from uploader.genotypes import normalise, qc
 
 logger = logging.getLogger(__name__)
 
@@ -15,10 +16,10 @@ def parse_args():
     """Parse the command-line arguments."""
     parser = ArgumentParser(
         prog="genotype-file-qc",
-        description="Run QC on a genotype file for a collector job.")
+        description="Normalise a genotype file and run QC on it.")
     parser.add_argument("jobs_db", help="Path to the jobs SQLite database.")
     parser.add_argument(
-        "collector_job_id", help="The id of the genotype-file-stats job.")
+        "qc_job_id", help="The id of the genotype-file-qc job.")
     parser.add_argument(
         "--log-level", default="info",
         choices=("debug", "info", "warning", "error", "critical"))
@@ -26,16 +27,22 @@ def parse_args():
 
 
 def main():
-    """Entry point to the genotype file QC."""
+    """Normalise the genotype file, then run QC on it."""
     args = parse_args()
     logging.basicConfig(level=getattr(logging, args.log_level.upper()))
     with sqlite3.connection(args.jobs_db) as conn:
-        job = jobs.job(conn, args.collector_job_id, fulldetails=True)
-    metadata = job["metadata"]
-    result = qc.check_file(
-        metadata["file"],
-        json.loads(metadata["config"]),
-        json.loads(metadata.get("stage02") or "{}"))
+        job = jobs.job(conn, args.qc_job_id, fulldetails=True)
+        metadata = job["metadata"]
+        config = json.loads(metadata["config"])
+        stage02 = json.loads(metadata.get("stage02") or "{}")
+        try:
+            work_file = normalise.normalise_file(
+                Path(metadata["file"]), config)
+        except normalise.NormalisationError as _exc:
+            print(json.dumps({"errors": [{"message": str(_exc)}]}))
+            return 1
+        result = qc.check_file(
+            work_file, {**config, "transposed": False}, stage02)
     print(json.dumps(result))
     return 0 if not result["errors"] else 1