From d2605cb72d7cdbc7d3cc633b94a451c0acd2edbb Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Wed, 15 Jun 2022 09:36:18 +0300 Subject: Fix linting and type errors --- qc_app/__init__.py | 4 ++-- qc_app/entry.py | 50 ++++++++++++++++++++++++-------------------------- qc_app/jobs.py | 20 ++++++++++++-------- qc_app/parse.py | 21 ++++++--------------- 4 files changed, 44 insertions(+), 51 deletions(-) (limited to 'qc_app') diff --git a/qc_app/__init__.py b/qc_app/__init__.py index 7f423c2..08b56c9 100644 --- a/qc_app/__init__.py +++ b/qc_app/__init__.py @@ -17,10 +17,10 @@ def instance_path(): return path -def create_app(instance_path): +def create_app(instance_dir): """The application factory""" app = Flask( - __name__, instance_path=instance_path, instance_relative_config=True) + __name__, instance_path=instance_dir, instance_relative_config=True) app.config.from_pyfile(os.path.join(os.getcwd(), "etc/default_config.py")) app.config.from_pyfile("config.py") # Override defaults with instance path diff --git a/qc_app/entry.py b/qc_app/entry.py index 7f67a33..c53648a 100644 --- a/qc_app/entry.py +++ b/qc_app/entry.py @@ -1,7 +1,5 @@ """Entry-point module""" import os -import random -import string import mimetypes from typing import Tuple from zipfile import ZipFile, is_zipfile @@ -18,24 +16,24 @@ from flask import ( entrybp = Blueprint("entry", __name__) -def errors(request) -> Tuple[str, ...]: - """Return a tuple of the errors found in the `request`. If no error is +def errors(rqst) -> Tuple[str, ...]: + """Return a tuple of the errors found in the request `rqst`. If no error is found, then an empty tuple is returned.""" def __filetype_error__(): return ( ("Invalid file type provided.",) - if request.form.get("filetype") not in ("average", "standard-error") + if rqst.form.get("filetype") not in ("average", "standard-error") else tuple()) def __file_missing_error__(): return ( ("No file was uploaded.",) - if ("qc_text_file" not in request.files or - request.files["qc_text_file"].filename == "") + if ("qc_text_file" not in rqst.files or + rqst.files["qc_text_file"].filename == "") else tuple()) def __file_mimetype_error__(): - text_file = request.files["qc_text_file"] + text_file = rqst.files["qc_text_file"] return ( ( ("Invalid file! Expected a tab-separated-values file, or a zip " @@ -50,26 +48,26 @@ def errors(request) -> Tuple[str, ...]: def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]: """Check the uploaded zip file for errors.""" - zfile_errors = tuple() + zfile_errors: Tuple[str, ...] = tuple() if is_zipfile(filepath): - zfile = ZipFile(filepath, "r") - infolist = zfile.infolist() - if len(infolist) != 1: - zfile_errors = zfile_errors + ( - ("Expected exactly one (1) member file within the uploaded zip " - "file. Got {len(infolist)} member files.")) - if len(infolist) == 1 and infolist[0].is_dir(): - zfile_errors = zfile_errors + ( - ("Expected a member text file in the uploaded zip file. Got a " - "directory/folder.")) - - if len(infolist) == 1 and not infolist[0].is_dir(): - zfile.extract(infolist[0], path=upload_dir) - mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}") - if mime[0] != "text/tab-separated-values": + with ZipFile(filepath, "r") as zfile: + infolist = zfile.infolist() + if len(infolist) != 1: + zfile_errors = zfile_errors + ( + ("Expected exactly one (1) member file within the uploaded zip " + "file. Got {len(infolist)} member files."),) + if len(infolist) == 1 and infolist[0].is_dir(): zfile_errors = zfile_errors + ( - ("Expected the member text file in the uploaded zip file to" - " be a tab-separated file.")) + ("Expected a member text file in the uploaded zip file. Got a " + "directory/folder."),) + + if len(infolist) == 1 and not infolist[0].is_dir(): + zfile.extract(infolist[0], path=upload_dir) + mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}") + if mime[0] != "text/tab-separated-values": + zfile_errors = zfile_errors + ( + ("Expected the member text file in the uploaded zip file to" + " be a tab-separated file."),) return zfile_errors diff --git a/qc_app/jobs.py b/qc_app/jobs.py index e97d175..4e6a11e 100644 --- a/qc_app/jobs.py +++ b/qc_app/jobs.py @@ -1,3 +1,4 @@ +"""Handle jobs""" import os import shlex import subprocess @@ -7,29 +8,32 @@ from datetime import timedelta from redis import Redis def error_filename(job_id, error_dir): + "Compute the path of the file where errors will be dumped." return f"{error_dir}/job_{job_id}.error" -def launch_job( +def launch_job(# pylint: disable=[too-many-arguments] redis_conn: Redis, filepath: str, filetype, redisurl, error_dir, ttl_seconds: int): """Launch a job in the background""" job_id = str(uuid4()) command = [ - "python3", "-m" "scripts.worker", filetype, filepath, redisurl, job_id] - job = { + "python3", "-m", "scripts.worker", filetype, filepath, redisurl, job_id] + the_job = { "job_id": job_id, "command": shlex.join(command), "status": "pending", "filename": os.path.basename(filepath), "percent": 0 } - redis_conn.hset(name=job["job_id"], mapping=job) - redis_conn.expire(name=job["job_id"], time=timedelta(seconds=ttl_seconds)) + redis_conn.hset(name=the_job["job_id"], mapping=the_job) + redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds)) if not os.path.exists(error_dir): os.mkdir(error_dir) - with open(error_filename(job_id, error_dir), "w") as errorfile: - subprocess.Popen(command, stderr=errorfile) + with open(error_filename(job_id, error_dir), + "w", + encoding="utf-8") as errorfile: + subprocess.Popen(command, stderr=errorfile) # pylint: disable=[consider-using-with] - return job + return the_job def job(redis_conn, job_id: str): "Retrieve the job" diff --git a/qc_app/parse.py b/qc_app/parse.py index b2a0156..e72a163 100644 --- a/qc_app/parse.py +++ b/qc_app/parse.py @@ -1,23 +1,13 @@ """File parsing module""" import os -from functools import reduce import jsonpickle from redis import Redis -from flask import ( - flash, - request, - url_for, - redirect, - Blueprint, - render_template, - current_app as app) +from flask import flash, request, url_for, redirect, Blueprint, render_template +from flask import current_app as app -from . import jobs from quality_control.errors import InvalidValue -from quality_control.parsing import ( - FileType, - strain_names) +from . import jobs parsebp = Blueprint("parse", __name__) isinvalidvalue = lambda item: isinstance(item, InvalidValue) @@ -25,7 +15,6 @@ isinvalidvalue = lambda item: isinstance(item, InvalidValue) @parsebp.route("/parse", methods=["GET"]) def parse(): """Trigger file parsing""" - # TODO: Maybe implement external process to parse the files errors = False filename = request.args.get("filename") filetype = request.args.get("filetype") @@ -59,6 +48,7 @@ def parse(): @parsebp.route("/status/", methods=["GET"]) def parse_status(job_id: str): + "Retrieve the status of the job" with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: job = jobs.job(rconn, job_id) @@ -76,7 +66,7 @@ def parse_status(job_id: str): filename = job.get("filename", "uploaded file") errors = jsonpickle.decode( job.get("errors", jsonpickle.encode(tuple()))) - if status == "success" or status == "aborted": + if status in ("success", "aborted"): return redirect(url_for("parse.results", job_id=job_id)) if status == "parse-error": @@ -133,6 +123,7 @@ def fail(job_id: str): @parsebp.route("/abort", methods=["POST"]) def abort(): + """Handle user request to abort file processing""" job_id = request.form["job_id"] with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: -- cgit v1.2.3