Migrate background job to use SQLite rather than Redis.

2 files changed, 31 insertions, 56 deletions

diff --git a/scripts/insert_samples.py b/scripts/insert_samples.py
index fc029f9..96ae8e2 100644
--- a/scripts/insert_samples.py
+++ b/scripts/insert_samples.py
@@ -6,10 +6,10 @@ import argparse
 import traceback
 
 import MySQLdb as mdb
-from redis import Redis
+
 from gn_libs.mysqldb import database_connection
 
-from uploader.check_connections import check_db, check_redis
+from uploader.check_connections import check_db
 from uploader.species.models import species_by_id
 from uploader.population.models import population_by_id
 from uploader.samples.models import (
@@ -35,7 +35,6 @@ class SeparatorAction(argparse.Action):
         setattr(namespace, self.dest, (chr(9) if values == "\\t" else values))
 
 def insert_samples(conn: mdb.Connection,# pylint: disable=[too-many-arguments, too-many-positional-arguments]
-                   rconn: Redis,# pylint: disable=[unused-argument]
                    speciesid: int,
                    populationid: int,
                    samplesfile: pathlib.Path,
@@ -119,11 +118,6 @@ if __name__ == "__main__":
             help=("The character used to delimit (surround?) the value in "
                   "each column."))
 
-        # == Script-specific extras ==
-        parser.add_argument("--redisuri",
-                            help="URL to initialise connection to redis",
-                            default="redis:///")
-
         args = parser.parse_args()
         return args
 
@@ -132,17 +126,13 @@ if __name__ == "__main__":
         status_code = 1 # Exit with an Exception
         args = cli_args()
         check_db(args.databaseuri)
-        check_redis(args.redisuri)
         if not args.samplesfile.exists():
             logging.error("File not found: '%s'.", args.samplesfile)
             return 2
 
-        with (Redis.from_url(args.redisuri, decode_responses=True) as rconn,
-              database_connection(args.databaseuri) as dbconn):
-
+        with database_connection(args.databaseuri) as dbconn:
             try:
                 status_code = insert_samples(dbconn,
-                                             rconn,
                                              args.speciesid,
                                              args.populationid,
                                              args.samplesfile,
diff --git a/uploader/samples/views.py b/uploader/samples/views.py
index f8baf7e..8555588 100644
--- a/uploader/samples/views.py
+++ b/uploader/samples/views.py
@@ -2,16 +2,20 @@
 import os
 import sys
 import uuid
+import logging
 from pathlib import Path
 
-from redis import Redis
 from flask import (flash,
                    request,
                    redirect,
                    Blueprint,
                    current_app as app)
 
+from gn_libs import sqlite3
+from gn_libs import jobs as gnlibsjobs
+
 from uploader import jobs
+from uploader import session
 from uploader.files import save_file
 from uploader.flask_extensions import url_for
 from uploader.ui import make_template_renderer
@@ -96,22 +100,6 @@ def list_samples(species: dict, population: dict, **kwargs):# pylint: disable=[u
                                activelink="list-samples")
 
 
-def build_sample_upload_job(# pylint: disable=[too-many-arguments, too-many-positional-arguments]
-        speciesid: int,
-        populationid: int,
-        samplesfile: Path,
-        separator: str,
-        firstlineheading: bool,
-        quotechar: str):
-    """Define the async command to run the actual samples data upload."""
-    return [
-        sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"],
-        str(speciesid), str(populationid), str(samplesfile.absolute()),
-        separator, f"--redisuri={app.config['REDIS_URL']}",
-        f"--quotechar={quotechar}"
-    ] + (["--firstlineheading"] if firstlineheading else [])
-
-
 @samplesbp.route("<int:species_id>/populations/<int:population_id>/upload-samples",
                methods=["GET", "POST"])
 @require_login
@@ -170,35 +158,32 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma
 
     quotechar = (request.form.get("field_delimiter", '"') or '"')
 
-    redisuri = app.config["REDIS_URL"]
-    with Redis.from_url(redisuri, decode_responses=True) as rconn:
-        #T0DO: Add a QC step here — what do we check?
-        # 1. Does any sample in the uploaded file exist within the database?
-        #    If yes, what is/are its/their species and population?
-        # 2. If yes 1. above, provide error with notes on which species and
-        #    populations already own the samples.
-        the_job = jobs.launch_job(
-            jobs.initialise_job(
-                rconn,
-                jobs.jobsnamespace(),
+    _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]
+    with sqlite3.connection(_jobs_db) as conn:
+        job = gnlibsjobs.launch_job(
+            gnlibsjobs.initialise_job(
+                conn,
                 str(uuid.uuid4()),
-                build_sample_upload_job(
-                    species["SpeciesId"],
-                    population["InbredSetId"],
-                    samples_file,
+                [
+                    sys.executable, "-m", "scripts.insert_samples",
+                    app.config["SQL_URI"],
+                    str(species["SpeciesId"]),
+                    str(population["InbredSetId"]),
+                    str(samples_file.absolute()),
                     separator,
-                    firstlineheading,
-                    quotechar),
+                    f"--quotechar={quotechar}"
+                ] + (["--firstlineheading"] if firstlineheading else []),
                 "samples_upload",
-                app.config["JOBS_TTL_SECONDS"],
-                {"job_name": f"Samples Upload: {samples_file.name}"}),
-            redisuri,
-            f"{app.config['UPLOAD_FOLDER']}/job_errors")
-        return redirect(url_for(
-            "species.populations.samples.upload_status",
-            species_id=species_id,
-            population_id=population_id,
-            job_id=the_job["jobid"]))
+                extra_meta={
+                    "job_name": f"Samples Upload: {samples_file.name}"
+                },
+                external_id=session.logged_in_user_id()),
+            _jobs_db,
+            Path(f"{app.config['UPLOAD_FOLDER']}/job_errors").absolute(),
+            loglevel=logging.getLevelName(
+                app.logger.getEffectiveLevel()).lower())
+        return redirect(
+            url_for("background-jobs.job_status", job_id=job["job_id"]))
 
 
 @samplesbp.route("<int:species_id>/populations/<int:population_id>/"