"""File parsing module""" import os import jsonpickle from redis import Redis from flask import flash, request, url_for, redirect, Blueprint, render_template from flask import current_app as app from quality_control.errors import InvalidValue, DuplicateHeading from qc_app import jobs from qc_app.dbinsert import species_by_id from qc_app.db_utils import with_db_connection parsebp = Blueprint("parse", __name__) def isinvalidvalue(item): """Check whether item is of type InvalidValue""" return isinstance(item, InvalidValue) def isduplicateheading(item): """Check whether item is of type DuplicateHeading""" return isinstance(item, DuplicateHeading) @parsebp.route("/parse", methods=["GET"]) def parse(): """Trigger file parsing""" errors = False speciesid = request.args.get("speciesid") filename = request.args.get("filename") filetype = request.args.get("filetype") if speciesid is None: flash("No species selected", "alert-error error-expr-data") errors = True else: try: speciesid = int(speciesid) species = with_db_connection( lambda con: species_by_id(con, speciesid)) if not bool(species): flash("No such species.", "alert-error error-expr-data") errors = True except ValueError: flash("Invalid speciesid provided. Expected an integer.", "alert-error error-expr-data") errors = True if filename is None: flash("No file provided", "alert-error error-expr-data") errors = True if filetype is None: flash("No filetype provided", "alert-error error-expr-data") errors = True if filetype not in ("average", "standard-error"): flash("Invalid filetype provided", "alert-error error-expr-data") errors = True if filename: filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename) if not os.path.exists(filepath): flash("Selected file does not exist (any longer)", "alert-error error-expr-data") errors = True if errors: return redirect(url_for("entry.upload_file")) redisurl = app.config["REDIS_URL"] with Redis.from_url(redisurl, decode_responses=True) as rconn: job = jobs.launch_job( jobs.build_file_verification_job( rconn, app.config["SQL_URI"], redisurl, speciesid, filepath, filetype, app.config["JOBS_TTL_SECONDS"]), redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors") return redirect(url_for("parse.parse_status", job_id=job["jobid"])) @parsebp.route("/status/<job_id>", methods=["GET"]) def parse_status(job_id: str): "Retrieve the status of the job" with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: try: job = jobs.job(rconn, jobs.jobsnamespace(), job_id) except jobs.JobNotFound as _exc: return render_template("no_such_job.html", job_id=job_id), 400 error_filename = jobs.error_filename( job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") if os.path.exists(error_filename): stat = os.stat(error_filename) if stat.st_size > 0: return redirect(url_for("parse.fail", job_id=job_id)) job_id = job["jobid"] progress = float(job["percent"]) status = job["status"] filename = job.get("filename", "uploaded file") errors = jsonpickle.decode( job.get("errors", jsonpickle.encode(tuple()))) if status in ("success", "aborted"): return redirect(url_for("parse.results", job_id=job_id)) if status == "parse-error": return redirect(url_for("parse.fail", job_id=job_id)) app.jinja_env.globals.update( isinvalidvalue=isinvalidvalue, isduplicateheading=isduplicateheading) return render_template( "job_progress.html", job_id = job_id, job_status = status, progress = progress, message = job.get("message", ""), job_name = f"Parsing '{filename}'", errors=errors) @parsebp.route("/results/<job_id>", methods=["GET"]) def results(job_id: str): """Show results of parsing...""" with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: job = jobs.job(rconn, jobs.jobsnamespace(), job_id) if job: filename = job["filename"] errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple()))) app.jinja_env.globals.update( isinvalidvalue=isinvalidvalue, isduplicateheading=isduplicateheading) return render_template( "parse_results.html", errors=errors, job_name = f"Parsing '{filename}'", user_aborted = job.get("user_aborted"), job_id=job["jobid"]) return render_template("no_such_job.html", job_id=job_id) @parsebp.route("/fail/<job_id>", methods=["GET"]) def fail(job_id: str): """Handle parsing failure""" with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: job = jobs.job(rconn, jobs.jobsnamespace(), job_id) if job: error_filename = jobs.error_filename( job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") if os.path.exists(error_filename): stat = os.stat(error_filename) if stat.st_size > 0: return render_template( "worker_failure.html", job_id=job_id) return render_template("parse_failure.html", job=job) return render_template("no_such_job.html", job_id=job_id) @parsebp.route("/abort", methods=["POST"]) def abort(): """Handle user request to abort file processing""" job_id = request.form["job_id"] with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: job = jobs.job(rconn, jobs.jobsnamespace(), job_id) if job: rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id), key="user_aborted", value=int(True)) return redirect(url_for("parse.parse_status", job_id=job_id))