aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--etc/default_config.py1
-rw-r--r--qc_app/jobs.py36
2 files changed, 28 insertions, 9 deletions
diff --git a/etc/default_config.py b/etc/default_config.py
index 5bafd1a..45272de 100644
--- a/etc/default_config.py
+++ b/etc/default_config.py
@@ -10,6 +10,7 @@ SECRET_KEY = b"<Please! Please! Please! Change This!>"
UPLOAD_FOLDER = "/tmp/qc_app_files"
REDIS_URL = "redis://"
JOBS_TTL_SECONDS = 1209600 # 14 days
+GNQC_REDIS_PREFIX="GNQC"
GN3_URL="http://localhost:8080"
SQL_URI = os.getenv(
"SQL_URI", "mysql://gn2:mysql_password@localhost/db_webqtl_s")
diff --git a/qc_app/jobs.py b/qc_app/jobs.py
index f5e5173..dc1f967 100644
--- a/qc_app/jobs.py
+++ b/qc_app/jobs.py
@@ -3,15 +3,32 @@ import os
import sys
import shlex
import subprocess
-from typing import Union
from uuid import UUID, uuid4
from datetime import timedelta
+from typing import Union, Optional
from redis import Redis
+from flask import current_app as app
+
+JOBS_PREFIX = "JOBS"
class JobNotFound(Exception):
"""Raised if we try to retrieve a non-existent job."""
+def jobsnamespace():
+ """
+ Return the jobs namespace prefix. It depends on app configuration.
+
+ Calling this function outside of an application context will cause an
+ exception to be raised. It is mostly a convenience utility to use within the
+ application.
+ """
+ return f"{app.config['GNQC_REDIS_PREFIX']}:{JOBS_PREFIX}"
+
+def job_key(namespaceprefix: str, jobid: Union[str, UUID]) -> str:
+ """Build the key by appending it to the namespace prefix."""
+ return f"{namespaceprefix}:{jobid}"
+
def raise_jobnotfound(jobid: Union[str,UUID]):
"""Utility to raise a `NoSuchJobError`"""
raise JobNotFound(f"Could not retrieve job '{jobid}'.")
@@ -21,15 +38,16 @@ def error_filename(job_id, error_dir):
return f"{error_dir}/job_{job_id}.error"
def initialise_job(# pylint: disable=[too-many-arguments]
- redis_conn: Redis, job_id: str, command: list, job_type: str,
- ttl_seconds: int, extra_meta: dict) -> dict:
+ rconn: Redis, rprefix: str, jobid: str, command: list, job_type: str,
+ ttl_seconds: int = 86400, extra_meta: Optional[dict] = None) -> dict:
"Initialise a job 'object' and put in on redis"
the_job = {
- "job_id": job_id, "command": shlex.join(command), "status": "pending",
- "percent": 0, "job-type": job_type, **extra_meta
+ "job_id": jobid, "command": shlex.join(command), "status": "pending",
+ "percent": 0, "job-type": job_type, **(extra_meta or {})
}
- redis_conn.hset(name=the_job["job_id"], mapping=the_job)
- redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds))
+ rconn.hset(job_key(rprefix, jobid), mapping=the_job)
+ rconn.expire(
+ name=job_key(rprefix, jobid), time=timedelta(seconds=ttl_seconds))
return the_job
def build_file_verification_job(#pylint: disable=[too-many-arguments]
@@ -85,7 +103,7 @@ def launch_job(the_job: dict, redisurl: str, error_dir):
return the_job
-def job(redis_conn, job_id: Union[str,UUID]):
+def job(rconn: Redis, rprefix: str, jobid: Union[str,UUID]):
"Retrieve the job"
- thejob = redis_conn.hgetall(str(job_id)) or raise_jobnotfound(job_id)
+ thejob = rconn.hgetall(job_key(rprefix, jobid)) or raise_jobnotfound(jobid)
return thejob