aboutsummaryrefslogtreecommitdiff
path: root/qc_app
diff options
context:
space:
mode:
authorFrederick Muriuki Muriithi2022-07-15 07:36:55 +0300
committerFrederick Muriuki Muriithi2022-07-19 05:08:17 +0300
commit52024b1f6cde74b51181ce4108cf01611a7ea636 (patch)
treea02306cadf4f7a1b1e5c8f89d295eb41a4a6fd8b /qc_app
parente59108d3cea2e61f1a23c22c20edf2d3974e7a10 (diff)
downloadgn-uploader-52024b1f6cde74b51181ce4108cf01611a7ea636.tar.gz
Rework: Use generic worker script to launch process
Use the generic worker script as the interface for launching external processes.
Diffstat (limited to 'qc_app')
-rw-r--r--qc_app/jobs.py16
-rw-r--r--qc_app/parse.py11
2 files changed, 18 insertions, 9 deletions
diff --git a/qc_app/jobs.py b/qc_app/jobs.py
index 8c93e9f..406874a 100644
--- a/qc_app/jobs.py
+++ b/qc_app/jobs.py
@@ -11,10 +11,10 @@ def error_filename(job_id, error_dir):
"Compute the path of the file where errors will be dumped."
return f"{error_dir}/job_{job_id}.error"
-def launch_job(# pylint: disable=[too-many-arguments]
- redis_conn: Redis, filepath: str, filetype, redisurl, error_dir,
+def build_file_verification_job(
+ redis_conn: Redis, filepath: str, filetype: str, redisurl: str,
ttl_seconds: int):
- """Launch a job in the background"""
+ "Build a file verification job"
job_id = str(uuid4())
command = [
"python3", "-m", "scripts.validate_file", filetype, filepath, redisurl,
@@ -23,18 +23,24 @@ def launch_job(# pylint: disable=[too-many-arguments]
the_job = {
"job_id": job_id, "command": shlex.join(command), "status": "pending",
"filename": os.path.basename(filepath), "percent": 0,
- "filetype": filetype
+ "filetype": filetype, "job-type": "file-verification"
}
redis_conn.hset(name=the_job["job_id"], mapping=the_job)
redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds))
+ return the_job
+def launch_job(the_job: dict, redisurl: str, error_dir):
+ """Launch a job in the background"""
if not os.path.exists(error_dir):
os.mkdir(error_dir)
+ job_id = the_job["job_id"]
with open(error_filename(job_id, error_dir),
"w",
encoding="utf-8") as errorfile:
- subprocess.Popen(command, stderr=errorfile) # pylint: disable=[consider-using-with]
+ subprocess.Popen( # pylint: disable=[consider-using-with]
+ ["python3", "-m", "scripts.worker", redisurl, job_id],
+ stderr=errorfile)
return the_job
diff --git a/qc_app/parse.py b/qc_app/parse.py
index 2a33fd0..70ef551 100644
--- a/qc_app/parse.py
+++ b/qc_app/parse.py
@@ -40,11 +40,14 @@ def parse():
if errors:
return redirect(url_for("entry.upload_file"))
- with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ redisurl = app.config["REDIS_URL"]
+ with Redis.from_url(redisurl, decode_responses=True) as rconn:
job = jobs.launch_job(
- rconn, filepath, filetype, app.config["REDIS_URL"],
- f"{app.config['UPLOAD_FOLDER']}/job_errors",
- app.config["JOBS_TTL_SECONDS"])
+ jobs.build_file_verification_job(
+ rconn, filepath, filetype, redisurl,
+ app.config["JOBS_TTL_SECONDS"]),
+ redisurl,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors")
return redirect(url_for("parse.parse_status", job_id=job["job_id"]))