From 3da6848381b6103fbb58eeab8d7051cba0bded58 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Thu, 4 Aug 2022 08:51:15 +0300
Subject: Implement data insertion

- Hook up external data insertion script to webserver code
- Provide rudimentary status indication
- Generalise some job creation details
---
 qc_app/jobs.py | 39 +++++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 8 deletions(-)

(limited to 'qc_app/jobs.py')

diff --git a/qc_app/jobs.py b/qc_app/jobs.py
index 406874a..9b350b7 100644
--- a/qc_app/jobs.py
+++ b/qc_app/jobs.py
@@ -11,6 +11,18 @@ def error_filename(job_id, error_dir):
     "Compute the path of the file where errors will be dumped."
     return f"{error_dir}/job_{job_id}.error"
 
+def __init_job__(# pylint: disable=[too-many-arguments]
+        redis_conn: Redis, job_id: str, command: list, job_type: str,
+        ttl_seconds: int, extra_meta: dict) -> dict:
+    "Initialise a job 'object' and put in on redis"
+    the_job = {
+        "job_id": job_id, "command": shlex.join(command), "status": "pending",
+        "percent": 0, "job-type": job_type, **extra_meta
+    }
+    redis_conn.hset(name=the_job["job_id"], mapping=the_job)
+    redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds))
+    return the_job
+
 def build_file_verification_job(
         redis_conn: Redis, filepath: str, filetype: str, redisurl: str,
         ttl_seconds: int):
@@ -20,14 +32,25 @@ def build_file_verification_job(
         "python3", "-m", "scripts.validate_file", filetype, filepath, redisurl,
         job_id
     ]
-    the_job = {
-        "job_id": job_id, "command": shlex.join(command), "status": "pending",
-        "filename": os.path.basename(filepath), "percent": 0,
-        "filetype": filetype, "job-type": "file-verification"
-    }
-    redis_conn.hset(name=the_job["job_id"], mapping=the_job)
-    redis_conn.expire(name=the_job["job_id"], time=timedelta(seconds=ttl_seconds))
-    return the_job
+    return __init_job__(
+        redis_conn, job_id, command, "file-verification", ttl_seconds, {
+            "filetype": filetype,
+            "filename": os.path.basename(filepath), "percent": 0
+        })
+
+def data_insertion_job(# pylint: disable=[too-many-arguments]
+        redis_conn: Redis, filepath: str, filetype: str, speciesid: int,
+        platformid: int, datasetid: int, databaseuri: str, redisuri: str,
+        ttl_seconds: int) -> dict:
+    "Build a data insertion job"
+    command = [
+        "python3", "-m", "scripts.insert_data", filetype, filepath, speciesid,
+        platformid, datasetid, databaseuri, redisuri
+    ]
+    return __init_job__(
+        redis_conn, str(uuid4()), command, "data-insertion", ttl_seconds, {
+            "filename": os.path.basename(filepath), "filetype": filetype,
+        })
 
 def launch_job(the_job: dict, redisurl: str, error_dir):
     """Launch a job in the background"""
-- 
cgit v1.2.3