Collect and display errors on 'geno' files in the bundle.

4 files changed, 110 insertions, 39 deletions

diff --git a/qc_app/static/css/styles.css b/qc_app/static/css/styles.css
index 474c7f7..79dab0e 100644
--- a/qc_app/static/css/styles.css
+++ b/qc_app/static/css/styles.css
@@ -134,11 +134,7 @@ form fieldset:nth-child(odd) {
 }
 
 @media(min-width: 1250px) {
-    form {
-	width: 65ch;
-    }
-
-    .explainer {
+    form, .explainer, .error-table {
 	width: 65ch;
     }
 }
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
index f9a912c..49df061 100644
--- a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
+++ b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
@@ -3,6 +3,31 @@
 
 {%block title%}R/qtl2 bundle: QC Job Error{%endblock%}
 
+{%macro errors_table(tableid, errors)%}
+<table id="{{tableid}}"
+       class="error-table"
+       style="display: block; overflow: scroll; max-height: 250px;">
+  <thead style="position: sticky; top: 0;">
+    <tr>
+      <th>Line</th>
+      <th>Column</th>
+      <th>Value</th>
+      <th>Message</th>
+    </tr>
+  </thead>
+  <tbody>
+    {%for error in errors%}
+    <tr>
+      <td>{{error.line}}</td>
+      <td>{{error.column}}</td>
+      <td>{{error.value}}</td>
+      <td>{{error.message}}</td>
+    </tr>
+    {%endfor%}
+  </tbody>
+</table>
+{%endmacro%}
+
 {%block contents%}
 <h1 class="heading">R/qtl2 bundle: QC job Error</h1>
 
@@ -24,7 +49,29 @@
 </ul>
 {%endif%}
 
-<p><emph>list other errors here by file type, I think &hellip;</emph></p>
+{%if errorsgeno | length > 0%}
+<h2 class="heading">Geno Errors ({{errorsgeno | length}})</h3>
+<div class="explainer">
+  We found the following errors in the 'geno' file in your R/qtl2 bundle:
+</div>
+{{errors_table("tbl-errors-geno", errorsgeno[0:50])}}
+{%endif%}
+
+{%if errorspheno | length > 0%}
+<h2 class="heading">Pheno Errors ({{errorspheno | length}})</h3>
+<div class="explainer">
+  We found the following errors in the 'pheno' file in your R/qtl2 bundle:
+</div>
+{{errorspheno}}
+{%endif%}
+
+{%if errorsphenocovar | length > 0%}
+<h2 class="heading">Phenocovar Errors ({{errorsphenocovar | length}})</h3>
+<div class="explainer">
+  We found the following errors in the 'phenocovar' file in your R/qtl2 bundle:
+</div>
+{{errorsphenocovar}}
+{%endif%}
 
 <h4>stdout</h4>
 {{cli_output(job, "stdout")}}
diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py
index 4b00891..c45952c 100644
--- a/qc_app/upload/rqtl2.py
+++ b/qc_app/upload/rqtl2.py
@@ -207,6 +207,12 @@ def rqtl2_bundle_qc_status(jobid: UUID):
                                        job=thejob,
                                        errorsgeneric=json.loads(
                                            thejob.get("errors-generic", "[]")),
+                                       errorsgeno=json.loads(
+                                           thejob.get("errors-geno", "[]")),
+                                       errorspheno=json.loads(
+                                           thejob.get("errors-pheno", "[]")),
+                                       errorsphenocovar=json.loads(
+                                           thejob.get("errors-phenocovar", "[]")),
                                        messages=logmessages)
             if jobstatus == "success":
                 jobmeta = json.loads(thejob["job-metadata"])
diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py
index 43f766a..02c8c3a 100644
--- a/scripts/qc_on_rqtl2_bundle.py
+++ b/scripts/qc_on_rqtl2_bundle.py
@@ -1,7 +1,6 @@
 """Run Quality Control checks on R/qtl2 bundle."""
 import sys
 import json
-from pathlib import Path
 from zipfile import ZipFile
 from argparse import Namespace
 from typing import Union, Sequence
@@ -9,6 +8,8 @@ from logging import Logger, getLogger, StreamHandler
 
 from redis import Redis
 
+from quality_control.errors import InvalidValue
+
 from qc_app import jobs
 from qc_app.db_utils import database_connection
 from qc_app.check_connections import check_db, check_redis
@@ -20,38 +21,59 @@ from scripts.cli_parser import init_cli_parser
 from scripts.process_rqtl2_bundle import parse_job
 from scripts.redis_logger import setup_redis_logger
 
-def add_to_errors(rconn: Redis, fqjobid: str, key: str, errors: Sequence[rqfe.MissingFile]):
+def dict2tuple(dct: dict) -> tuple:
+    """Utility to convert items in dicts to pairs of tuples."""
+    return tuple((key, val) for key,val in dct.items())
+
+def add_to_errors(rconn: Redis,
+                  fqjobid: str,
+                  key: str,
+                  errors: Sequence[Union[InvalidValue, rqfe.MissingFile]]):
     """Add `errors` to a given list of errors"""
-    errs = tuple(set(
-        json.loads(rconn.hget(fqjobid, key) or "[]") +
-        [error.message for error in errors]))
+    errs = tuple(dict(item) for item in set(
+        [dict2tuple(old) for old in
+         json.loads(rconn.hget(fqjobid, key) or "[]")] +
+        [dict2tuple({"type": type(error).__name__, **error._asdict()})
+         for error in errors]))
     rconn.hset(fqjobid, key, json.dumps(errs))
 
-def qc_missing_files(rconn: Redis, fqjobid: str,
-                     bundlefilepath: Union[str, Path]) -> tuple[
-                         tuple[str, str], ...]:
+def qc_missing_files(rconn: Redis,
+                     fqjobid: str,
+                     zfile: ZipFile,
+                     logger: Logger) -> bool:
     """Run QC for files listed in control file that don't exist in bundle."""
-    with ZipFile(str(bundlefilepath), "r") as zfile:
-        missing = rqc.missing_files(zfile)
-        add_to_errors(rconn, fqjobid, "errors-generic", tuple(
-            rqfe.MissingFile(
-                mfile[0], mfile[1], (
-                    f"File '{mfile[1]}' is listed in the control file under "
-                    f"the '{mfile[0]}' key, but it does not actually exist in "
-                    "the bundle."))
-            for mfile in missing))
-
-    return missing
-
-def qc_geno_errors(_rconn, _fqjobid, _job) -> bool:
+    logger.info("Checking for missing files…")
+    missing = rqc.missing_files(zfile)
+    add_to_errors(rconn, fqjobid, "errors-generic", tuple(
+        rqfe.MissingFile(
+            mfile[0], mfile[1], (
+                f"File '{mfile[1]}' is listed in the control file under "
+                f"the '{mfile[0]}' key, but it does not actually exist in "
+                "the bundle."))
+        for mfile in missing))
+    if len(missing) > 0:
+        logger.error("Missing files in the bundle!")
+        return True
+    return False
+
+def qc_geno_errors(rconn, fqjobid, zfile, logger) -> bool:
     """Check for errors in `geno` file(s)."""
+    logger.info("Checking for geno errors…")
+    gerrs = tuple(rqc.geno_errors(zfile))
+    add_to_errors(rconn, fqjobid, "errors-generic", tuple(
+        err for err in gerrs if isinstance(err, rqfe.MissingFile)))
+    add_to_errors(rconn, fqjobid, "errors-geno", tuple(
+        err for err in gerrs if not isinstance(err, rqfe.MissingFile)))
+    if len(gerrs) > 0:
+        logger.error("The 'geno' file has errors.")
+        return True
     return False
 
-def qc_pheno_errors(_rconn, _fqjobid, _job) -> bool:
+def qc_pheno_errors(_rconn, _fqjobid, _zfile, _logger) -> bool:
     """Check for errors in `pheno` file(s)."""
     return False
 
-def qc_phenocovar_errors(_rconn, _fqjobid, _job) -> bool:
+def qc_phenocovar_errors(_rconn, _fqjobid, _zfile, _logger) -> bool:
     """Check for errors in `phenocovar` file(s)."""
     return False
 
@@ -63,16 +85,16 @@ def run_qc(rconn: Redis,
     thejob = parse_job(rconn, args.redisprefix, args.jobid)
     jobmeta = thejob["job-metadata"]
 
-    if len(qc_missing_files(rconn, fqjobid, jobmeta["rqtl2-bundle-file"])) > 0:
-        logger.error("Missing files in the bundle!")
-        return 1
-
-    return (
-        1 if any((
-            qc_geno_errors(rconn, fqjobid, thejob),
-            qc_pheno_errors(rconn, fqjobid, thejob),
-            qc_phenocovar_errors(rconn, fqjobid, thejob)))
-        else 0)
+    with ZipFile(jobmeta["rqtl2-bundle-file"], "r") as zfile:
+        if qc_missing_files(rconn, fqjobid, zfile, logger):
+            return 1
+
+        return (
+            1 if any((
+                qc_geno_errors(rconn, fqjobid, zfile, logger),
+                qc_pheno_errors(rconn, fqjobid, zfile, logger),
+                qc_phenocovar_errors(rconn, fqjobid, zfile, logger)))
+            else 0)
 
 if __name__ == "__main__":
     def main():