Retrive and display generic bundle errors

Implement the first QC check for generic errors e.g. missing files in
bundle, etc. Display retrieved errors on UI.

4 files changed, 65 insertions, 10 deletions

diff --git a/qc_app/static/css/styles.css b/qc_app/static/css/styles.css
index f6e829f..474c7f7 100644
--- a/qc_app/static/css/styles.css
+++ b/qc_app/static/css/styles.css
@@ -198,3 +198,11 @@ form fieldset legend {
     width: 90%;
     overflow: scroll;
 }
+
+.qc-error-display {
+    border-radius: 0.8em;
+    padding-top: 0.5em;
+    padding-bottom: 0.5em;
+    max-height: 250px;
+    overflow: scroll;
+}
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
index 5d2ebee..f9a912c 100644
--- a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
+++ b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
@@ -12,9 +12,19 @@
     be uploaded onto GeneNetwork.</p>
 </div>
 
-<h2 class="heading">Errors</h2>
+{%if errorsgeneric | length > 0%}
+<h2 class="heading">Generic Errors ({{errorsgeneric | length}})</h3>
+<div class="explainer">
+  We found the following generic errors in your R/qtl2 bundle:
+</div>
+<ul class="alert-error qc-error-display">
+  {%for msg in errorsgeneric%}
+  <li>{{msg}}</li>
+  {%endfor%}
+</ul>
+{%endif%}
 
-<p><emph>list errors here by file type, I think &hellip;</emph></p>
+<p><emph>list other errors here by file type, I think &hellip;</emph></p>
 
 <h4>stdout</h4>
 {{cli_output(job, "stdout")}}
diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py
index b96d9f0..c20bd8f 100644
--- a/qc_app/upload/rqtl2.py
+++ b/qc_app/upload/rqtl2.py
@@ -175,12 +175,15 @@ def upload_rqtl2_bundle(species_id: int, population_id: int):
                     jobs.jobsnamespace(),
                     jobid,
                     [sys.executable, "-m", "scripts.qc_on_rqtl2_bundle",
-                    app.config["SQL_URI"], app.config["REDIS_URL"],
-                    jobs.jobsnamespace(), jobid, "--redisexpiry",
-                    str(redis_ttl_seconds)],
+                     app.config["SQL_URI"], app.config["REDIS_URL"],
+                     jobs.jobsnamespace(), jobid, "--redisexpiry",
+                     str(redis_ttl_seconds)],
                     "rqtl2-bundle-qc-job",
                     redis_ttl_seconds,
-                    {}),
+                    {"job-metadata": json.dumps({
+                        "speciesid": species_id,
+                        "populationid": population_id,
+                        "rqtl2-bundle-file": str(the_file.absolute())})}),
                 redisuri,
                 f"{app.config['UPLOAD_FOLDER']}/job_errors")
             return redirect(url_for(
@@ -200,6 +203,8 @@ def rqtl2_bundle_qc_status(jobid: UUID):
             if jobstatus == "error":
                 return render_template("rqtl2/rqtl2-qc-job-error.html",
                                        job=thejob,
+                                       errorsgeneric=json.loads(
+                                           thejob.get("errors-generic", "[]")),
                                        messages=logmessages)
             if jobstatus == "success":
                 return render_template("rqtl2/rqtl2-qc-job-results.html",
diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py
index 5802f3a..8dd6b7e 100644
--- a/scripts/qc_on_rqtl2_bundle.py
+++ b/scripts/qc_on_rqtl2_bundle.py
@@ -1,6 +1,10 @@
 """Run Quality Control checks on R/qtl2 bundle."""
 import sys
-from logging import getLogger, StreamHandler
+import json
+from typing import Sequence
+from zipfile import ZipFile
+from argparse import Namespace
+from logging import Logger, getLogger, StreamHandler
 
 from redis import Redis
 
@@ -8,12 +12,40 @@ from qc_app import jobs
 from qc_app.db_utils import database_connection
 from qc_app.check_connections import check_db, check_redis
 
+from r_qtl import r_qtl2_qc as rqc
+from r_qtl import fileerrors as rqfe
+
 from scripts.cli_parser import init_cli_parser
+from scripts.process_rqtl2_bundle import parse_job
 from scripts.redis_logger import setup_redis_logger
 
-def run_qc(_args):
+def add_to_errors(rconn: Redis, fqjobid: str, key: str, errors: Sequence[rqfe.MissingFile]):
+    """Add `errors` to a given list of errors"""
+    errs = tuple(set(
+        json.loads(rconn.hget(fqjobid, key) or "[]") +
+        [error.message for error in errors]))
+    rconn.hset(fqjobid, key, json.dumps(errs))
+
+def run_qc(rconn: Redis, args: Namespace, logger: Logger) -> int:
     """Run the QC programs."""
-    return 1
+    fqjobid = jobs.job_key(args.redisprefix, args.jobid)
+    thejob = parse_job(rconn, args.redisprefix, args.jobid)
+    meta = thejob["job-metadata"]
+    with ZipFile(meta["rqtl2-bundle-file"], "r") as zfile:
+        missing = rqc.missing_files(zfile)
+        add_to_errors(rconn, fqjobid, "errors-generic", tuple(
+            rqfe.MissingFile(
+                mfile[0], mfile[1], (
+                    f"File '{mfile[1]}' is listed in the control file under "
+                    f"the '{mfile[0]}' key, but it does not actually exist in "
+                    "the bundle."))
+            for mfile in missing))
+
+    if len(missing) > 0:
+        logger.error("Missing files in the bundle!")
+        return 1
+
+    return 0
 
 if __name__ == "__main__":
     def main():
@@ -34,7 +66,7 @@ if __name__ == "__main__":
                 rconn, fqjobid, f"{fqjobid}:log-messages",
                 args.redisexpiry))
 
-            exitcode = run_qc(args)
+            exitcode = run_qc(rconn, args, logger)
             rconn.hset(
                 jobs.job_key(args.redisprefix, args.jobid), "exitcode", exitcode)
             return exitcode