From 53b1e7cb181380a24aab4cbc7a9634b2d8dd2d29 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Fri, 19 Jan 2024 08:29:04 +0300
Subject: scripts: Process R/qtl2 bundle

Build script to start the processing of the R/qtl2 bundle.
---
 qc_app/jobs.py         | 15 ++++++++++++---
 qc_app/upload/rqtl2.py | 44 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 55 insertions(+), 4 deletions(-)

(limited to 'qc_app')

diff --git a/qc_app/jobs.py b/qc_app/jobs.py
index c5bf5e5..a8257a3 100644
--- a/qc_app/jobs.py
+++ b/qc_app/jobs.py
@@ -3,11 +3,19 @@ import os
 import sys
 import shlex
 import subprocess
-from uuid import uuid4
+from typing import Union
+from uuid import UUID, uuid4
 from datetime import timedelta
 
 from redis import Redis
 
+class JobNotFound(Exception):
+    """Raised if we try to retrieve a non-existent job."""
+
+def raise_jobnotfound(jobid: Union[str,UUID]):
+    """Utility to raise a `NoSuchJobError`"""
+    raise JobNotFound(f"Could not retrieve job '{jobid}'.")
+
 def error_filename(job_id, error_dir):
     "Compute the path of the file where errors will be dumped."
     return f"{error_dir}/job_{job_id}.error"
@@ -70,6 +78,7 @@ def launch_job(the_job: dict, redisurl: str, error_dir):
 
     return the_job
 
-def job(redis_conn, job_id: str):
+def job(redis_conn, job_id: Union[str,UUID]):
     "Retrieve the job"
-    return redis_conn.hgetall(job_id)
+    thejob = redis_conn.hgetall(str(job_id)) or raise_jobnotfound(job_id)
+    return thejob
diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py
index 7609fa9..7ba90c2 100644
--- a/qc_app/upload/rqtl2.py
+++ b/qc_app/upload/rqtl2.py
@@ -1,8 +1,12 @@
 """Module to handle uploading of R/qtl2 bundles."""
+import sys
+import json
+from uuid import uuid4
 from pathlib import Path
 from datetime import date
 from zipfile import ZipFile, is_zipfile
 
+from redis import Redis
 from MySQLdb.cursors import DictCursor
 from flask import (
     flash,
@@ -16,6 +20,7 @@ from flask import (
 from r_qtl import r_qtl2
 from r_qtl.errors import InvalidFormat
 
+from qc_app import jobs
 from qc_app.files import save_file, fullpath
 from qc_app.dbinsert import species as all_species
 from qc_app.db_utils import with_db_connection, database_connection
@@ -521,6 +526,7 @@ def select_dataset_info(species_id: int, population_id: int):
     return render_template("rqtl2/summary-info.html",
                            species=species,
                            population=population,
+                           rqtl2_bundle_file=thefile.name,
                            geno_dataset=geno_dataset,
                            probe_study=probeset_study,
                            probe_dataset=probeset_dataset)
@@ -532,11 +538,47 @@ def select_dataset_info(species_id: int, population_id: int):
              methods=["POST"])
 def confirm_bundle_details(species_id: int, population_id: int):
     """Confirm the details and trigger R/qtl2 bundle processing..."""
-    with database_connection(app.config["SQL_URI"]) as conn:
+    redisuri = app.config["REDIS_URL"]
+    with (database_connection(app.config["SQL_URI"]) as conn,
+          Redis.from_url(redisuri, decode_responses=True) as rconn):
         error = check_errors(
             conn, "species", "population", "rqtl2_bundle_file", "geno-dataset",
             "probe-study-id", "probe-dataset-id")
         if bool(error):
             return error
 
+        redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"]
+        jobid = str(uuid4())
+        _job = jobs.launch_job(
+            jobs.initialise_job(
+                rconn,
+                jobid,
+                [
+                    sys.executable, "-m", "scripts.process_rqtl2_bundle",
+                    app.config["SQL_URI"], app.config["REDIS_URL"], jobid,
+                    "--redisexpiry", str(redis_ttl_seconds)],
+                "R/qtl2 Bundle Upload",
+                redis_ttl_seconds,
+                {
+                    "bundle-metadata": json.dumps({
+                        "speciesid": species_id,
+                        "populationid": population_id,
+                        "rqtl2-bundle-file": str(fullpath(
+                            request.form["rqtl2_bundle_file"])),
+                        "geno-dataset-id": request.form.get(
+                            "geno-dataset-id", ""),
+                        "probe-study-id": request.form.get(
+                            "probe-study-id", ""),
+                        "probe-dataset-id": request.form.get(
+                            "probe-dataset-id", ""),
+                        **({
+                            "platformid": probeset_study_by_id(
+                                conn,
+                                int(request.form["probe-study-id"]))["ChipId"]
+                        } if bool(request.form.get("probe-study-id")) else {})
+                    })
+                }),
+            redisuri,
+            f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
         raise NotImplementedError
-- 
cgit v1.2.3