diff options
Diffstat (limited to 'qc_app/jobs.py')
| -rw-r--r-- | qc_app/jobs.py | 39 |
1 files changed, 31 insertions, 8 deletions
diff --git a/qc_app/jobs.py b/qc_app/jobs.py index 406874a..9b350b7 100644 --- a/qc_app/jobs.py +++ b/qc_app/jobs.py @@ -11,6 +11,18 @@ def error_filename(job_id, error_dir): "Compute the path of the file where errors will be dumped." return f"{error_dir}/job_{job_id}.error" +def __init_job__(# pylint: disable=[too-many-arguments] + redis_conn: Redis, job_id: str, command: list, job_type: str, + ttl_seconds: int, extra_meta: dict) -> dict: + "Initialise a job 'object' and put in on redis" + the_job = { + "job_id": job_id, "command": shlex.join(command), "status": "pending", + "percent": 0, "job-type": job_type, **extra_meta + } + redis_conn.hset(name=the_job["job_id"], mapping=the_job) + redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds)) + return the_job + def build_file_verification_job( redis_conn: Redis, filepath: str, filetype: str, redisurl: str, ttl_seconds: int): @@ -20,14 +32,25 @@ def build_file_verification_job( "python3", "-m", "scripts.validate_file", filetype, filepath, redisurl, job_id ] - the_job = { - "job_id": job_id, "command": shlex.join(command), "status": "pending", - "filename": os.path.basename(filepath), "percent": 0, - "filetype": filetype, "job-type": "file-verification" - } - redis_conn.hset(name=the_job["job_id"], mapping=the_job) - redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds)) - return the_job + return __init_job__( + redis_conn, job_id, command, "file-verification", ttl_seconds, { + "filetype": filetype, + "filename": os.path.basename(filepath), "percent": 0 + }) + +def data_insertion_job(# pylint: disable=[too-many-arguments] + redis_conn: Redis, filepath: str, filetype: str, speciesid: int, + platformid: int, datasetid: int, databaseuri: str, redisuri: str, + ttl_seconds: int) -> dict: + "Build a data insertion job" + command = [ + "python3", "-m", "scripts.insert_data", filetype, filepath, speciesid, + platformid, datasetid, databaseuri, redisuri + ] + return __init_job__( + redis_conn, str(uuid4()), command, "data-insertion", ttl_seconds, { + "filename": os.path.basename(filepath), "filetype": filetype, + }) def launch_job(the_job: dict, redisurl: str, error_dir): """Launch a job in the background""" |