aboutsummaryrefslogtreecommitdiff
path: root/qc_app
diff options
context:
space:
mode:
authorFrederick Muriuki Muriithi2022-08-04 08:51:15 +0300
committerFrederick Muriuki Muriithi2022-08-04 08:51:15 +0300
commit3da6848381b6103fbb58eeab8d7051cba0bded58 (patch)
treefd20aa2d8c73638b72a7667504582f0146aa182f /qc_app
parent7bd9116019afe4aed369c5dfe69496abc6867381 (diff)
downloadgn-uploader-3da6848381b6103fbb58eeab8d7051cba0bded58.tar.gz
Implement data insertion
- Hook up external data insertion script to webserver code - Provide rudimentary status indication - Generalise some job creation details
Diffstat (limited to 'qc_app')
-rw-r--r--qc_app/dbinsert.py43
-rw-r--r--qc_app/jobs.py39
-rw-r--r--qc_app/templates/final_confirmation.html2
-rw-r--r--qc_app/templates/insert_progress.html24
4 files changed, 98 insertions, 10 deletions
diff --git a/qc_app/dbinsert.py b/qc_app/dbinsert.py
index 0b87c20..70aeb6d 100644
--- a/qc_app/dbinsert.py
+++ b/qc_app/dbinsert.py
@@ -5,7 +5,6 @@ from typing import Union
from functools import reduce
from datetime import datetime
-import requests
from redis import Redis
from MySQLdb.cursors import DictCursor
from flask import (
@@ -359,3 +358,45 @@ def final_confirmation():
"DataScale")))
except AssertionError as aserr:
return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/insert-data", methods=["POST"])
+def insert_data():
+ "Trigger data insertion"
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+ assert form.get("studyid"), "study"
+ assert form.get("datasetid"), "dataset"
+
+ filename = form["filename"]
+ filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}"
+ redisurl = app.config["REDIS_URL"]
+ if os.path.exists(filepath):
+ with Redis.from_url(redisurl, decode_responses=True) as rconn:
+ job = jobs.launch_job(
+ jobs.data_insertion_job(
+ rconn, filepath, form.get("filetype"),
+ form.get("species"), form.get("genechipid"),
+ form.get("datasetid"), app.config["SQL_URI"], redisurl,
+ app.config["JOBS_TTL_SECONDS"]),
+ redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+ return redirect(url_for("dbinsert.insert_status", job_id=job["job_id"]))
+ return render_error(f"File '{filename}' no longer exists.")
+ except AssertionError as aserr:
+ return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/status/<job_id>", methods=["GET"])
+def insert_status(job_id: str):
+ "Retrieve status of data insertion."
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ job = jobs.job(rconn, job_id)
+
+ if job:
+ if job["status"] == "success":
+ return render_template("insert_success.html", job=job)
+ return render_template("insert_progress.html", job=job)
+ return render_template("no_such_job.html", job_id=job_id), 400
diff --git a/qc_app/jobs.py b/qc_app/jobs.py
index 406874a..9b350b7 100644
--- a/qc_app/jobs.py
+++ b/qc_app/jobs.py
@@ -11,6 +11,18 @@ def error_filename(job_id, error_dir):
"Compute the path of the file where errors will be dumped."
return f"{error_dir}/job_{job_id}.error"
+def __init_job__(# pylint: disable=[too-many-arguments]
+ redis_conn: Redis, job_id: str, command: list, job_type: str,
+ ttl_seconds: int, extra_meta: dict) -> dict:
+ "Initialise a job 'object' and put in on redis"
+ the_job = {
+ "job_id": job_id, "command": shlex.join(command), "status": "pending",
+ "percent": 0, "job-type": job_type, **extra_meta
+ }
+ redis_conn.hset(name=the_job["job_id"], mapping=the_job)
+ redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds))
+ return the_job
+
def build_file_verification_job(
redis_conn: Redis, filepath: str, filetype: str, redisurl: str,
ttl_seconds: int):
@@ -20,14 +32,25 @@ def build_file_verification_job(
"python3", "-m", "scripts.validate_file", filetype, filepath, redisurl,
job_id
]
- the_job = {
- "job_id": job_id, "command": shlex.join(command), "status": "pending",
- "filename": os.path.basename(filepath), "percent": 0,
- "filetype": filetype, "job-type": "file-verification"
- }
- redis_conn.hset(name=the_job["job_id"], mapping=the_job)
- redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds))
- return the_job
+ return __init_job__(
+ redis_conn, job_id, command, "file-verification", ttl_seconds, {
+ "filetype": filetype,
+ "filename": os.path.basename(filepath), "percent": 0
+ })
+
+def data_insertion_job(# pylint: disable=[too-many-arguments]
+ redis_conn: Redis, filepath: str, filetype: str, speciesid: int,
+ platformid: int, datasetid: int, databaseuri: str, redisuri: str,
+ ttl_seconds: int) -> dict:
+ "Build a data insertion job"
+ command = [
+ "python3", "-m", "scripts.insert_data", filetype, filepath, speciesid,
+ platformid, datasetid, databaseuri, redisuri
+ ]
+ return __init_job__(
+ redis_conn, str(uuid4()), command, "data-insertion", ttl_seconds, {
+ "filename": os.path.basename(filepath), "filetype": filetype,
+ })
def launch_job(the_job: dict, redisurl: str, error_dir):
"""Launch a job in the background"""
diff --git a/qc_app/templates/final_confirmation.html b/qc_app/templates/final_confirmation.html
index f2de028..018f8d6 100644
--- a/qc_app/templates/final_confirmation.html
+++ b/qc_app/templates/final_confirmation.html
@@ -35,7 +35,7 @@
</ul>
</div>
-<form method="POST" action="#">
+<form method="POST" action="{{url_for('dbinsert.insert_data')}}">
{{hidden_fields(
filename, filetype, species=species, genechipid=genechipid,
studyid=studyid,datasetid=datasetid)}}
diff --git a/qc_app/templates/insert_progress.html b/qc_app/templates/insert_progress.html
new file mode 100644
index 0000000..1ac6212
--- /dev/null
+++ b/qc_app/templates/insert_progress.html
@@ -0,0 +1,24 @@
+{%extends "base.html"%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="5">
+{%endblock%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job_name}}</h2>
+
+<label for="job_status">status:</label>
+<span>{{job_status}}: {{message}}</span><br />
+
+{{job}}
+<label for="job_{{job_id}}">inserting: </label>
+<progress id="job_{{job_id}}" value="{{(job['percent'] | float) / 100}}">{{job["percent"]}}</progress>
+<span>{{"%.2f" | format(job['percent'] | float)}}%</span>
+
+<div>
+ {{job.stderr}}
+</div>
+
+{%endblock%}