Implement data insertion

- Hook up external data insertion script to webserver code
- Provide rudimentary status indication
- Generalise some job creation details

5 files changed, 99 insertions, 11 deletions

diff --git a/qc_app/dbinsert.py b/qc_app/dbinsert.py
index 0b87c20..70aeb6d 100644
--- a/qc_app/dbinsert.py
+++ b/qc_app/dbinsert.py
@@ -5,7 +5,6 @@ from typing import Union
 from functools import reduce
 from datetime import datetime
 
-import requests
 from redis import Redis
 from MySQLdb.cursors import DictCursor
 from flask import (
@@ -359,3 +358,45 @@ def final_confirmation():
                  "DataScale")))
     except AssertionError as aserr:
         return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/insert-data", methods=["POST"])
+def insert_data():
+    "Trigger data insertion"
+    form = request.form
+    try:
+        assert form.get("filename"), "filename"
+        assert form.get("filetype"), "filetype"
+        assert form.get("species"), "species"
+        assert form.get("genechipid"), "platform"
+        assert form.get("studyid"), "study"
+        assert form.get("datasetid"), "dataset"
+
+        filename = form["filename"]
+        filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}"
+        redisurl = app.config["REDIS_URL"]
+        if os.path.exists(filepath):
+            with Redis.from_url(redisurl, decode_responses=True) as rconn:
+                job = jobs.launch_job(
+                    jobs.data_insertion_job(
+                        rconn, filepath, form.get("filetype"),
+                        form.get("species"), form.get("genechipid"),
+                        form.get("datasetid"), app.config["SQL_URI"], redisurl,
+                        app.config["JOBS_TTL_SECONDS"]),
+                    redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+            return redirect(url_for("dbinsert.insert_status", job_id=job["job_id"]))
+        return render_error(f"File '{filename}' no longer exists.")
+    except AssertionError as aserr:
+        return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/status/<job_id>", methods=["GET"])
+def insert_status(job_id: str):
+    "Retrieve status of data insertion."
+    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+        job = jobs.job(rconn, job_id)
+
+    if job:
+        if job["status"] == "success":
+            return render_template("insert_success.html", job=job)
+        return render_template("insert_progress.html", job=job)
+    return render_template("no_such_job.html", job_id=job_id), 400
diff --git a/qc_app/jobs.py b/qc_app/jobs.py
index 406874a..9b350b7 100644
--- a/qc_app/jobs.py
+++ b/qc_app/jobs.py
@@ -11,6 +11,18 @@ def error_filename(job_id, error_dir):
     "Compute the path of the file where errors will be dumped."
     return f"{error_dir}/job_{job_id}.error"
 
+def __init_job__(# pylint: disable=[too-many-arguments]
+        redis_conn: Redis, job_id: str, command: list, job_type: str,
+        ttl_seconds: int, extra_meta: dict) -> dict:
+    "Initialise a job 'object' and put in on redis"
+    the_job = {
+        "job_id": job_id, "command": shlex.join(command), "status": "pending",
+        "percent": 0, "job-type": job_type, **extra_meta
+    }
+    redis_conn.hset(name=the_job["job_id"], mapping=the_job)
+    redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds))
+    return the_job
+
 def build_file_verification_job(
         redis_conn: Redis, filepath: str, filetype: str, redisurl: str,
         ttl_seconds: int):
@@ -20,14 +32,25 @@ def build_file_verification_job(
         "python3", "-m", "scripts.validate_file", filetype, filepath, redisurl,
         job_id
     ]
-    the_job = {
-        "job_id": job_id, "command": shlex.join(command), "status": "pending",
-        "filename": os.path.basename(filepath), "percent": 0,
-        "filetype": filetype, "job-type": "file-verification"
-    }
-    redis_conn.hset(name=the_job["job_id"], mapping=the_job)
-    redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds))
-    return the_job
+    return __init_job__(
+        redis_conn, job_id, command, "file-verification", ttl_seconds, {
+            "filetype": filetype,
+            "filename": os.path.basename(filepath), "percent": 0
+        })
+
+def data_insertion_job(# pylint: disable=[too-many-arguments]
+        redis_conn: Redis, filepath: str, filetype: str, speciesid: int,
+        platformid: int, datasetid: int, databaseuri: str, redisuri: str,
+        ttl_seconds: int) -> dict:
+    "Build a data insertion job"
+    command = [
+        "python3", "-m", "scripts.insert_data", filetype, filepath, speciesid,
+        platformid, datasetid, databaseuri, redisuri
+    ]
+    return __init_job__(
+        redis_conn, str(uuid4()), command, "data-insertion", ttl_seconds, {
+            "filename": os.path.basename(filepath), "filetype": filetype,
+        })
 
 def launch_job(the_job: dict, redisurl: str, error_dir):
     """Launch a job in the background"""
diff --git a/qc_app/templates/final_confirmation.html b/qc_app/templates/final_confirmation.html
index f2de028..018f8d6 100644
--- a/qc_app/templates/final_confirmation.html
+++ b/qc_app/templates/final_confirmation.html
@@ -35,7 +35,7 @@
   </ul>
 </div>
 
-<form method="POST" action="#">
+<form method="POST" action="{{url_for('dbinsert.insert_data')}}">
   {{hidden_fields(
   filename, filetype, species=species, genechipid=genechipid,
   studyid=studyid,datasetid=datasetid)}}
diff --git a/qc_app/templates/insert_progress.html b/qc_app/templates/insert_progress.html
new file mode 100644
index 0000000..1ac6212
--- /dev/null
+++ b/qc_app/templates/insert_progress.html
@@ -0,0 +1,24 @@
+{%extends "base.html"%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="5">
+{%endblock%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job_name}}</h2>
+
+<label for="job_status">status:</label>
+<span>{{job_status}}: {{message}}</span><br />
+
+{{job}}
+<label for="job_{{job_id}}">inserting: </label>
+<progress id="job_{{job_id}}" value="{{(job['percent'] | float) / 100}}">{{job["percent"]}}</progress>
+<span>{{"%.2f" | format(job['percent'] | float)}}%</span>
+
+<div>
+  {{job.stderr}}
+</div>
+
+{%endblock%}
diff --git a/scripts/worker.py b/scripts/worker.py
index 879442a..03751d2 100644
--- a/scripts/worker.py
+++ b/scripts/worker.py
@@ -54,7 +54,7 @@ def run_job(job, rconn):
 
             os.remove(stderrpath)
         return process.poll()
-    except Exception as exc:
+    except Exception as exc:# pylint: disable=[broad-except,unused-variable]
         update_stdout_stderr(
             traceback.format_exc().encode("utf-8"), "stderr", rconn, job_id)
         print(traceback.format_exc(), file=sys.stderr)