diff options
Diffstat (limited to 'uploader/expression_data/parse.py')
| -rw-r--r-- | uploader/expression_data/parse.py | 178 |
1 files changed, 178 insertions, 0 deletions
diff --git a/uploader/expression_data/parse.py b/uploader/expression_data/parse.py new file mode 100644 index 0000000..fc1c3f0 --- /dev/null +++ b/uploader/expression_data/parse.py @@ -0,0 +1,178 @@ +"""File parsing module""" +import os + +import jsonpickle +from redis import Redis +from flask import flash, request, url_for, redirect, Blueprint, render_template +from flask import current_app as app + +from quality_control.errors import InvalidValue, DuplicateHeading + +from uploader import jobs +from uploader.dbinsert import species_by_id +from uploader.db_utils import with_db_connection +from uploader.authorisation import require_login + +parsebp = Blueprint("parse", __name__) + +def isinvalidvalue(item): + """Check whether item is of type InvalidValue""" + return isinstance(item, InvalidValue) + +def isduplicateheading(item): + """Check whether item is of type DuplicateHeading""" + return isinstance(item, DuplicateHeading) + +@parsebp.route("/parse", methods=["GET"]) +@require_login +def parse(): + """Trigger file parsing""" + errors = False + speciesid = request.args.get("speciesid") + filename = request.args.get("filename") + filetype = request.args.get("filetype") + if speciesid is None: + flash("No species selected", "alert-error error-expr-data") + errors = True + else: + try: + speciesid = int(speciesid) + species = with_db_connection( + lambda con: species_by_id(con, speciesid)) + if not bool(species): + flash("No such species.", "alert-error error-expr-data") + errors = True + except ValueError: + flash("Invalid speciesid provided. Expected an integer.", + "alert-error error-expr-data") + errors = True + + if filename is None: + flash("No file provided", "alert-error error-expr-data") + errors = True + + if filetype is None: + flash("No filetype provided", "alert-error error-expr-data") + errors = True + + if filetype not in ("average", "standard-error"): + flash("Invalid filetype provided", "alert-error error-expr-data") + errors = True + + if filename: + filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename) + if not os.path.exists(filepath): + flash("Selected file does not exist (any longer)", + "alert-error error-expr-data") + errors = True + + if errors: + return redirect(url_for("expression-data.index.upload_file")) + + redisurl = app.config["REDIS_URL"] + with Redis.from_url(redisurl, decode_responses=True) as rconn: + job = jobs.launch_job( + jobs.build_file_verification_job( + rconn, app.config["SQL_URI"], redisurl, + speciesid, filepath, filetype, + app.config["JOBS_TTL_SECONDS"]), + redisurl, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + + return redirect(url_for("expression-data.parse.parse_status", job_id=job["jobid"])) + +@parsebp.route("/status/<job_id>", methods=["GET"]) +def parse_status(job_id: str): + "Retrieve the status of the job" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + except jobs.JobNotFound as _exc: + return render_template("no_such_job.html", job_id=job_id), 400 + + error_filename = jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + if os.path.exists(error_filename): + stat = os.stat(error_filename) + if stat.st_size > 0: + return redirect(url_for("parse.fail", job_id=job_id)) + + job_id = job["jobid"] + progress = float(job["percent"]) + status = job["status"] + filename = job.get("filename", "uploaded file") + errors = jsonpickle.decode( + job.get("errors", jsonpickle.encode(tuple()))) + if status in ("success", "aborted"): + return redirect(url_for("expression-data.parse.results", job_id=job_id)) + + if status == "parse-error": + return redirect(url_for("parse.fail", job_id=job_id)) + + app.jinja_env.globals.update( + isinvalidvalue=isinvalidvalue, + isduplicateheading=isduplicateheading) + return render_template( + "job_progress.html", + job_id = job_id, + job_status = status, + progress = progress, + message = job.get("message", ""), + job_name = f"Parsing '{filename}'", + errors=errors) + +@parsebp.route("/results/<job_id>", methods=["GET"]) +def results(job_id: str): + """Show results of parsing...""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + filename = job["filename"] + errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple()))) + app.jinja_env.globals.update( + isinvalidvalue=isinvalidvalue, + isduplicateheading=isduplicateheading) + return render_template( + "parse_results.html", + errors=errors, + job_name = f"Parsing '{filename}'", + user_aborted = job.get("user_aborted"), + job_id=job["jobid"]) + + return render_template("no_such_job.html", job_id=job_id) + +@parsebp.route("/fail/<job_id>", methods=["GET"]) +def fail(job_id: str): + """Handle parsing failure""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + error_filename = jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + if os.path.exists(error_filename): + stat = os.stat(error_filename) + if stat.st_size > 0: + return render_template( + "worker_failure.html", job_id=job_id) + + return render_template("parse_failure.html", job=job) + + return render_template("no_such_job.html", job_id=job_id) + +@parsebp.route("/abort", methods=["POST"]) +@require_login +def abort(): + """Handle user request to abort file processing""" + job_id = request.form["job_id"] + + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id), + key="user_aborted", + value=int(True)) + + return redirect(url_for("expression-data.parse.parse_status", job_id=job_id)) |