Move code handling expression data upload into new module.

1 files changed, 178 insertions, 0 deletions

diff --git a/uploader/expression_data/parse.py b/uploader/expression_data/parse.py
new file mode 100644
index 0000000..fc1c3f0
--- /dev/null
+++ b/uploader/expression_data/parse.py
@@ -0,0 +1,178 @@
+"""File parsing module"""
+import os
+
+import jsonpickle
+from redis import Redis
+from flask import flash, request, url_for, redirect, Blueprint, render_template
+from flask import current_app as app
+
+from quality_control.errors import InvalidValue, DuplicateHeading
+
+from uploader import jobs
+from uploader.dbinsert import species_by_id
+from uploader.db_utils import with_db_connection
+from uploader.authorisation import require_login
+
+parsebp = Blueprint("parse", __name__)
+
+def isinvalidvalue(item):
+    """Check whether item is of type InvalidValue"""
+    return isinstance(item, InvalidValue)
+
+def isduplicateheading(item):
+    """Check whether item is of type DuplicateHeading"""
+    return isinstance(item, DuplicateHeading)
+
+@parsebp.route("/parse", methods=["GET"])
+@require_login
+def parse():
+    """Trigger file parsing"""
+    errors = False
+    speciesid = request.args.get("speciesid")
+    filename = request.args.get("filename")
+    filetype = request.args.get("filetype")
+    if speciesid is None:
+        flash("No species selected", "alert-error error-expr-data")
+        errors = True
+    else:
+        try:
+            speciesid = int(speciesid)
+            species = with_db_connection(
+                lambda con: species_by_id(con, speciesid))
+            if not bool(species):
+                flash("No such species.", "alert-error error-expr-data")
+                errors = True
+        except ValueError:
+            flash("Invalid speciesid provided. Expected an integer.",
+                  "alert-error error-expr-data")
+            errors = True
+
+    if filename is None:
+        flash("No file provided", "alert-error error-expr-data")
+        errors = True
+
+    if filetype is None:
+        flash("No filetype provided", "alert-error error-expr-data")
+        errors = True
+
+    if filetype not in ("average", "standard-error"):
+        flash("Invalid filetype provided", "alert-error error-expr-data")
+        errors = True
+
+    if filename:
+        filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename)
+        if not os.path.exists(filepath):
+            flash("Selected file does not exist (any longer)",
+                  "alert-error error-expr-data")
+            errors = True
+
+    if errors:
+        return redirect(url_for("expression-data.index.upload_file"))
+
+    redisurl = app.config["REDIS_URL"]
+    with Redis.from_url(redisurl, decode_responses=True) as rconn:
+        job = jobs.launch_job(
+            jobs.build_file_verification_job(
+                rconn, app.config["SQL_URI"], redisurl,
+                speciesid, filepath, filetype,
+                app.config["JOBS_TTL_SECONDS"]),
+            redisurl,
+            f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+    return redirect(url_for("expression-data.parse.parse_status", job_id=job["jobid"]))
+
+@parsebp.route("/status/<job_id>", methods=["GET"])
+def parse_status(job_id: str):
+    "Retrieve the status of the job"
+    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+        try:
+            job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+        except jobs.JobNotFound as _exc:
+            return render_template("no_such_job.html", job_id=job_id), 400
+
+    error_filename = jobs.error_filename(
+        job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+    if os.path.exists(error_filename):
+        stat = os.stat(error_filename)
+        if stat.st_size > 0:
+            return redirect(url_for("parse.fail", job_id=job_id))
+
+    job_id = job["jobid"]
+    progress = float(job["percent"])
+    status = job["status"]
+    filename = job.get("filename", "uploaded file")
+    errors = jsonpickle.decode(
+        job.get("errors", jsonpickle.encode(tuple())))
+    if status in ("success", "aborted"):
+        return redirect(url_for("expression-data.parse.results", job_id=job_id))
+
+    if status == "parse-error":
+        return redirect(url_for("parse.fail", job_id=job_id))
+
+    app.jinja_env.globals.update(
+        isinvalidvalue=isinvalidvalue,
+        isduplicateheading=isduplicateheading)
+    return render_template(
+        "job_progress.html",
+        job_id = job_id,
+        job_status = status,
+        progress = progress,
+        message = job.get("message", ""),
+        job_name = f"Parsing '{filename}'",
+        errors=errors)
+
+@parsebp.route("/results/<job_id>", methods=["GET"])
+def results(job_id: str):
+    """Show results of parsing..."""
+    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+        job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+    if job:
+        filename = job["filename"]
+        errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple())))
+        app.jinja_env.globals.update(
+            isinvalidvalue=isinvalidvalue,
+            isduplicateheading=isduplicateheading)
+        return render_template(
+            "parse_results.html",
+            errors=errors,
+            job_name = f"Parsing '{filename}'",
+            user_aborted = job.get("user_aborted"),
+            job_id=job["jobid"])
+
+    return render_template("no_such_job.html", job_id=job_id)
+
+@parsebp.route("/fail/<job_id>", methods=["GET"])
+def fail(job_id: str):
+    """Handle parsing failure"""
+    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+        job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+    if job:
+        error_filename = jobs.error_filename(
+            job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+        if os.path.exists(error_filename):
+            stat = os.stat(error_filename)
+            if stat.st_size > 0:
+                return render_template(
+                    "worker_failure.html", job_id=job_id)
+
+        return render_template("parse_failure.html", job=job)
+
+    return render_template("no_such_job.html", job_id=job_id)
+
+@parsebp.route("/abort", methods=["POST"])
+@require_login
+def abort():
+    """Handle user request to abort file processing"""
+    job_id = request.form["job_id"]
+
+    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+        job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+        if job:
+            rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id),
+                       key="user_aborted",
+                       value=int(True))
+
+    return redirect(url_for("expression-data.parse.parse_status", job_id=job_id))