R/qtl2 QC: Set up scaffolding for QC UI

Set up the scaffolding for the flows and UI that will be used when
running QC against the uploaded R/qtl2 bundle.

This will be fleshed out later, and the UI is likely to change
somewhat, down the line.

6 files changed, 186 insertions, 19 deletions

diff --git a/qc_app/templates/base.html b/qc_app/templates/base.html
index 3b701bc..2228356 100644
--- a/qc_app/templates/base.html
+++ b/qc_app/templates/base.html
@@ -6,7 +6,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     {%block extrameta%}{%endblock%}
 
-    <title>QC: {%block title%}{%endblock%}</title>
+    <title>GN Uploader: {%block title%}{%endblock%}</title>
 
     <link rel="stylesheet" type="text/css" href="/static/css/styles.css" />
     {%block css%}{%endblock%}
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
new file mode 100644
index 0000000..5d2ebee
--- /dev/null
+++ b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
@@ -0,0 +1,32 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}R/qtl2 bundle: QC Job Error{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 bundle: QC job Error</h1>
+
+<div class="explainer">
+  <p>The R/qtl2 bundle has failed some <emph>Quality Control</emph> checks.</p>
+  <p>We list below some of the errors that need to be fixed before the data can
+    be uploaded onto GeneNetwork.</p>
+</div>
+
+<h2 class="heading">Errors</h2>
+
+<p><emph>list errors here by file type, I think &hellip;</emph></p>
+
+<h4>stdout</h4>
+{{cli_output(job, "stdout")}}
+
+<h4>stderr</h4>
+{{cli_output(job, "stderr")}}
+
+<h4>Log</h4>
+<div class="cli-output">
+  {%for msg in messages%}
+  {{msg}}<br />
+  {%endfor%}
+</div>
+
+{%endblock%}
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-status.html b/qc_app/templates/rqtl2/rqtl2-qc-job-status.html
new file mode 100644
index 0000000..85b8864
--- /dev/null
+++ b/qc_app/templates/rqtl2/rqtl2-qc-job-status.html
@@ -0,0 +1,29 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="3">
+{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 bundle: QC job status</h1>
+
+<h2 class="heading">R/qtl2 bundle: QC Job Status</h2>
+
+<hr />
+<p>The job:</p>
+<hr />
+{{job}}
+<hr />
+<hr />
+
+<h4>Log</h4>
+<div class="cli-output">
+  {%for msg in messages%}
+  {{msg}}<br />
+  {%endfor%}
+</div>
+
+{%endblock%}
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-success.html b/qc_app/templates/rqtl2/rqtl2-qc-job-success.html
new file mode 100644
index 0000000..396f241
--- /dev/null
+++ b/qc_app/templates/rqtl2/rqtl2-qc-job-success.html
@@ -0,0 +1,37 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}R/qtl2 Bundle: Quality Control Successful{%endblock%}
+
+{%block contents%}
+<h2 class="heading">R/qtl2 Bundle: Quality Control Successful</h2>
+
+<div class="explainer">
+  <p>The R/qtl2 bundle you uploaded has passed <emph>all</emph> quality control
+    checks successfully, and is now ready for uploading into the database.</p>
+  <p>Click "Continue" below to proceed.</p>
+</div>
+
+<!--
+    The "action" on this form takes us to the next step, where we can
+    select all the other data necessary to enter the data into the database.
+  -->
+<form id="frm-upload-rqtl2-bundle"
+      action="{{url_for('upload.rqtl2.select_dataset_info',
+	      species_id=species.SpeciesId,
+	      population_id=population.InbredSetId)}}"
+      method="POST"
+      enctype="multipart/form-data">
+  {{flash_all_messages()}}
+  <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+  <input type="hidden" name="population_id"
+	 value="{{population.InbredSetId}}" />
+  <input type="hidden" name="rqtl2_bundle_file"
+	 value="{{rqtl2_bundle_file}}" />
+
+  <fieldset>
+    <input type="submit" value="continue" class="btn btn-main form-col-2" />
+  </fieldset>
+</form>
+
+{%endblock%}
diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py
index 9d787e1..afe0c2e 100644
--- a/qc_app/upload/rqtl2.py
+++ b/qc_app/upload/rqtl2.py
@@ -18,8 +18,6 @@ from flask import (
     current_app as app)
 
 from r_qtl import r_qtl2
-from r_qtl import r_qtl2_qc as rqc
-from r_qtl.errors import InvalidFormat
 
 from qc_app import jobs
 from qc_app.files import save_file, fullpath
@@ -135,12 +133,6 @@ class __RequestError__(Exception): #pylint: disable=[invalid-name]
     methods=["GET", "POST"])
 def upload_rqtl2_bundle(species_id: int, population_id: int):
     """Allow upload of R/qtl2 bundle."""
-    this_page_with_errors = redirect(url_for("upload.rqtl2.upload_rqtl2_bundle",
-                                             species_id=species_id,
-                                             population_id=population_id,
-                                             pgsrc="error"),
-                                     code=307)
-
     with database_connection(app.config["SQL_URI"]) as conn:
         species = species_by_id(conn, species_id)
         population = population_by_species_and_id(
@@ -171,17 +163,52 @@ def upload_rqtl2_bundle(species_id: int, population_id: int):
         if not is_zipfile(str(the_file)):
             raise __RequestError__("Invalid file! Expected a zip file.")
 
+        redisuri = app.config["REDIS_URL"]
+        with Redis.from_url(redisuri, decode_responses=True) as rconn:
+            jobid = str(uuid4())
+            redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"]
+            jobs.launch_job(
+                jobs.initialise_job(
+                    rconn,
+                    jobs.jobsnamespace(),
+                    jobid,
+                    [sys.executable, "-m", "scripts.qc_on_rqtl2_bundle",
+                    app.config["SQL_URI"], app.config["REDIS_URL"],
+                    jobs.jobsnamespace(), jobid, "--redisexpiry",
+                    str(redis_ttl_seconds)],
+                    "rqtl2-bundle-qc-job",
+                    redis_ttl_seconds,
+                    {}),
+                redisuri,
+                f"{app.config['UPLOAD_FOLDER']}/job_errors")
+            return redirect(url_for(
+                "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid))
+
+@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>",
+             methods=["GET", "POST"])
+def rqtl2_bundle_qc_status(jobid: UUID):
+    """Check the status of the QC jobs."""
+    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
         try:
-            with ZipFile(str(the_file), "r") as zfile:
-                rqc.validate_bundle(zfile)
-                return render_template(
-                    "rqtl2/upload-rqtl2-bundle-step-02.html",
-                    species=species,
-                    population=population,
-                    rqtl2_bundle_file=the_file.name)#type: ignore[union-attr]
-        except (InvalidFormat, __RequestError__) as exc:
-            flash("".join(exc.args), "alert-error error-rqtl2")
-            return this_page_with_errors
+            thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid)
+            messagelistname = thejob.get("log-messagelist")
+            logmessages = (rconn.lrange(messagelistname, 0, -1)
+                           if bool(messagelistname) else [])
+            jobstatus = thejob["status"]
+            if jobstatus == "error":
+                return render_template("rqtl2/rqtl2-qc-job-error.html",
+                                       job=thejob,
+                                       messages=logmessages)
+            if jobstatus == "success":
+                return render_template("rqtl2/rqtl2-qc-job-results.html",
+                                       job=thejob,
+                                       messages=logmessages)
+
+            return render_template("rqtl2/rqtl2-qc-job-status.html",
+                                   job=thejob,
+                                   messages=tuple())
+        except jobs.JobNotFound:
+            return render_template("rqtl2/no-such-job.html", jobid=jobid)
 
 def check_errors(conn, *args, **kwargs):#pylint: disable=[too-many-return-statements]
     """Check for select errors in the forms and return a page to redirect to."""
diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py
new file mode 100644
index 0000000..5802f3a
--- /dev/null
+++ b/scripts/qc_on_rqtl2_bundle.py
@@ -0,0 +1,42 @@
+"""Run Quality Control checks on R/qtl2 bundle."""
+import sys
+from logging import getLogger, StreamHandler
+
+from redis import Redis
+
+from qc_app import jobs
+from qc_app.db_utils import database_connection
+from qc_app.check_connections import check_db, check_redis
+
+from scripts.cli_parser import init_cli_parser
+from scripts.redis_logger import setup_redis_logger
+
+def run_qc(_args):
+    """Run the QC programs."""
+    return 1
+
+if __name__ == "__main__":
+    def main():
+        """Enter R/qtl2 bundle QC runner."""
+        args = init_cli_parser(
+            "qc-on-rqtl2-bundle", "Run QC on R/qtl2 bundle.").parse_args()
+        check_redis(args.redisuri)
+        check_db(args.databaseuri)
+
+        logger = getLogger("qc-on-rqtl2-bundle")
+        logger.addHandler(StreamHandler(stream=sys.stderr))
+        logger.setLevel("DEBUG")
+
+        fqjobid = jobs.job_key(args.redisprefix, args.jobid)
+        with (database_connection(args.databaseuri) as _dbconn,
+              Redis.from_url(args.redisuri, decode_responses=True) as rconn):
+            logger.addHandler(setup_redis_logger(
+                rconn, fqjobid, f"{fqjobid}:log-messages",
+                args.redisexpiry))
+
+            exitcode = run_qc(args)
+            rconn.hset(
+                jobs.job_key(args.redisprefix, args.jobid), "exitcode", exitcode)
+            return exitcode
+
+    sys.exit(main())