aboutsummaryrefslogtreecommitdiff
path: root/qc_app
diff options
context:
space:
mode:
Diffstat (limited to 'qc_app')
-rw-r--r--qc_app/__init__.py4
-rw-r--r--qc_app/entry.py50
-rw-r--r--qc_app/jobs.py20
-rw-r--r--qc_app/parse.py21
4 files changed, 44 insertions, 51 deletions
diff --git a/qc_app/__init__.py b/qc_app/__init__.py
index 7f423c2..08b56c9 100644
--- a/qc_app/__init__.py
+++ b/qc_app/__init__.py
@@ -17,10 +17,10 @@ def instance_path():
return path
-def create_app(instance_path):
+def create_app(instance_dir):
"""The application factory"""
app = Flask(
- __name__, instance_path=instance_path, instance_relative_config=True)
+ __name__, instance_path=instance_dir, instance_relative_config=True)
app.config.from_pyfile(os.path.join(os.getcwd(), "etc/default_config.py"))
app.config.from_pyfile("config.py") # Override defaults with instance path
diff --git a/qc_app/entry.py b/qc_app/entry.py
index 7f67a33..c53648a 100644
--- a/qc_app/entry.py
+++ b/qc_app/entry.py
@@ -1,7 +1,5 @@
"""Entry-point module"""
import os
-import random
-import string
import mimetypes
from typing import Tuple
from zipfile import ZipFile, is_zipfile
@@ -18,24 +16,24 @@ from flask import (
entrybp = Blueprint("entry", __name__)
-def errors(request) -> Tuple[str, ...]:
- """Return a tuple of the errors found in the `request`. If no error is
+def errors(rqst) -> Tuple[str, ...]:
+ """Return a tuple of the errors found in the request `rqst`. If no error is
found, then an empty tuple is returned."""
def __filetype_error__():
return (
("Invalid file type provided.",)
- if request.form.get("filetype") not in ("average", "standard-error")
+ if rqst.form.get("filetype") not in ("average", "standard-error")
else tuple())
def __file_missing_error__():
return (
("No file was uploaded.",)
- if ("qc_text_file" not in request.files or
- request.files["qc_text_file"].filename == "")
+ if ("qc_text_file" not in rqst.files or
+ rqst.files["qc_text_file"].filename == "")
else tuple())
def __file_mimetype_error__():
- text_file = request.files["qc_text_file"]
+ text_file = rqst.files["qc_text_file"]
return (
(
("Invalid file! Expected a tab-separated-values file, or a zip "
@@ -50,26 +48,26 @@ def errors(request) -> Tuple[str, ...]:
def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]:
"""Check the uploaded zip file for errors."""
- zfile_errors = tuple()
+ zfile_errors: Tuple[str, ...] = tuple()
if is_zipfile(filepath):
- zfile = ZipFile(filepath, "r")
- infolist = zfile.infolist()
- if len(infolist) != 1:
- zfile_errors = zfile_errors + (
- ("Expected exactly one (1) member file within the uploaded zip "
- "file. Got {len(infolist)} member files."))
- if len(infolist) == 1 and infolist[0].is_dir():
- zfile_errors = zfile_errors + (
- ("Expected a member text file in the uploaded zip file. Got a "
- "directory/folder."))
-
- if len(infolist) == 1 and not infolist[0].is_dir():
- zfile.extract(infolist[0], path=upload_dir)
- mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}")
- if mime[0] != "text/tab-separated-values":
+ with ZipFile(filepath, "r") as zfile:
+ infolist = zfile.infolist()
+ if len(infolist) != 1:
+ zfile_errors = zfile_errors + (
+ ("Expected exactly one (1) member file within the uploaded zip "
+ "file. Got {len(infolist)} member files."),)
+ if len(infolist) == 1 and infolist[0].is_dir():
zfile_errors = zfile_errors + (
- ("Expected the member text file in the uploaded zip file to"
- " be a tab-separated file."))
+ ("Expected a member text file in the uploaded zip file. Got a "
+ "directory/folder."),)
+
+ if len(infolist) == 1 and not infolist[0].is_dir():
+ zfile.extract(infolist[0], path=upload_dir)
+ mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}")
+ if mime[0] != "text/tab-separated-values":
+ zfile_errors = zfile_errors + (
+ ("Expected the member text file in the uploaded zip file to"
+ " be a tab-separated file."),)
return zfile_errors
diff --git a/qc_app/jobs.py b/qc_app/jobs.py
index e97d175..4e6a11e 100644
--- a/qc_app/jobs.py
+++ b/qc_app/jobs.py
@@ -1,3 +1,4 @@
+"""Handle jobs"""
import os
import shlex
import subprocess
@@ -7,29 +8,32 @@ from datetime import timedelta
from redis import Redis
def error_filename(job_id, error_dir):
+ "Compute the path of the file where errors will be dumped."
return f"{error_dir}/job_{job_id}.error"
-def launch_job(
+def launch_job(# pylint: disable=[too-many-arguments]
redis_conn: Redis, filepath: str, filetype, redisurl, error_dir,
ttl_seconds: int):
"""Launch a job in the background"""
job_id = str(uuid4())
command = [
- "python3", "-m" "scripts.worker", filetype, filepath, redisurl, job_id]
- job = {
+ "python3", "-m", "scripts.worker", filetype, filepath, redisurl, job_id]
+ the_job = {
"job_id": job_id, "command": shlex.join(command), "status": "pending",
"filename": os.path.basename(filepath), "percent": 0
}
- redis_conn.hset(name=job["job_id"], mapping=job)
- redis_conn.expire(name=job["job_id"], time=timedelta(seconds=ttl_seconds))
+ redis_conn.hset(name=the_job["job_id"], mapping=the_job)
+ redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds))
if not os.path.exists(error_dir):
os.mkdir(error_dir)
- with open(error_filename(job_id, error_dir), "w") as errorfile:
- subprocess.Popen(command, stderr=errorfile)
+ with open(error_filename(job_id, error_dir),
+ "w",
+ encoding="utf-8") as errorfile:
+ subprocess.Popen(command, stderr=errorfile) # pylint: disable=[consider-using-with]
- return job
+ return the_job
def job(redis_conn, job_id: str):
"Retrieve the job"
diff --git a/qc_app/parse.py b/qc_app/parse.py
index b2a0156..e72a163 100644
--- a/qc_app/parse.py
+++ b/qc_app/parse.py
@@ -1,23 +1,13 @@
"""File parsing module"""
import os
-from functools import reduce
import jsonpickle
from redis import Redis
-from flask import (
- flash,
- request,
- url_for,
- redirect,
- Blueprint,
- render_template,
- current_app as app)
+from flask import flash, request, url_for, redirect, Blueprint, render_template
+from flask import current_app as app
-from . import jobs
from quality_control.errors import InvalidValue
-from quality_control.parsing import (
- FileType,
- strain_names)
+from . import jobs
parsebp = Blueprint("parse", __name__)
isinvalidvalue = lambda item: isinstance(item, InvalidValue)
@@ -25,7 +15,6 @@ isinvalidvalue = lambda item: isinstance(item, InvalidValue)
@parsebp.route("/parse", methods=["GET"])
def parse():
"""Trigger file parsing"""
- # TODO: Maybe implement external process to parse the files
errors = False
filename = request.args.get("filename")
filetype = request.args.get("filetype")
@@ -59,6 +48,7 @@ def parse():
@parsebp.route("/status/<job_id>", methods=["GET"])
def parse_status(job_id: str):
+ "Retrieve the status of the job"
with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
job = jobs.job(rconn, job_id)
@@ -76,7 +66,7 @@ def parse_status(job_id: str):
filename = job.get("filename", "uploaded file")
errors = jsonpickle.decode(
job.get("errors", jsonpickle.encode(tuple())))
- if status == "success" or status == "aborted":
+ if status in ("success", "aborted"):
return redirect(url_for("parse.results", job_id=job_id))
if status == "parse-error":
@@ -133,6 +123,7 @@ def fail(job_id: str):
@parsebp.route("/abort", methods=["POST"])
def abort():
+ """Handle user request to abort file processing"""
job_id = request.form["job_id"]
with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
generated by cgit v1.2.3 (git 2.44.0) at 2024-11-05 13:01:34 +0000