diff options
Diffstat (limited to 'qc_app')
| -rw-r--r-- | qc_app/jobs.py | 15 | ||||
| -rw-r--r-- | qc_app/upload/rqtl2.py | 44 |
2 files changed, 55 insertions, 4 deletions
diff --git a/qc_app/jobs.py b/qc_app/jobs.py index c5bf5e5..a8257a3 100644 --- a/qc_app/jobs.py +++ b/qc_app/jobs.py @@ -3,11 +3,19 @@ import os import sys import shlex import subprocess -from uuid import uuid4 +from typing import Union +from uuid import UUID, uuid4 from datetime import timedelta from redis import Redis +class JobNotFound(Exception): + """Raised if we try to retrieve a non-existent job.""" + +def raise_jobnotfound(jobid: Union[str,UUID]): + """Utility to raise a `NoSuchJobError`""" + raise JobNotFound(f"Could not retrieve job '{jobid}'.") + def error_filename(job_id, error_dir): "Compute the path of the file where errors will be dumped." return f"{error_dir}/job_{job_id}.error" @@ -70,6 +78,7 @@ def launch_job(the_job: dict, redisurl: str, error_dir): return the_job -def job(redis_conn, job_id: str): +def job(redis_conn, job_id: Union[str,UUID]): "Retrieve the job" - return redis_conn.hgetall(job_id) + thejob = redis_conn.hgetall(str(job_id)) or raise_jobnotfound(job_id) + return thejob diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py index 7609fa9..7ba90c2 100644 --- a/qc_app/upload/rqtl2.py +++ b/qc_app/upload/rqtl2.py @@ -1,8 +1,12 @@ """Module to handle uploading of R/qtl2 bundles.""" +import sys +import json +from uuid import uuid4 from pathlib import Path from datetime import date from zipfile import ZipFile, is_zipfile +from redis import Redis from MySQLdb.cursors import DictCursor from flask import ( flash, @@ -16,6 +20,7 @@ from flask import ( from r_qtl import r_qtl2 from r_qtl.errors import InvalidFormat +from qc_app import jobs from qc_app.files import save_file, fullpath from qc_app.dbinsert import species as all_species from qc_app.db_utils import with_db_connection, database_connection @@ -521,6 +526,7 @@ def select_dataset_info(species_id: int, population_id: int): return render_template("rqtl2/summary-info.html", species=species, population=population, + rqtl2_bundle_file=thefile.name, geno_dataset=geno_dataset, probe_study=probeset_study, probe_dataset=probeset_dataset) @@ -532,11 +538,47 @@ def select_dataset_info(species_id: int, population_id: int): methods=["POST"]) def confirm_bundle_details(species_id: int, population_id: int): """Confirm the details and trigger R/qtl2 bundle processing...""" - with database_connection(app.config["SQL_URI"]) as conn: + redisuri = app.config["REDIS_URL"] + with (database_connection(app.config["SQL_URI"]) as conn, + Redis.from_url(redisuri, decode_responses=True) as rconn): error = check_errors( conn, "species", "population", "rqtl2_bundle_file", "geno-dataset", "probe-study-id", "probe-dataset-id") if bool(error): return error + redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"] + jobid = str(uuid4()) + _job = jobs.launch_job( + jobs.initialise_job( + rconn, + jobid, + [ + sys.executable, "-m", "scripts.process_rqtl2_bundle", + app.config["SQL_URI"], app.config["REDIS_URL"], jobid, + "--redisexpiry", str(redis_ttl_seconds)], + "R/qtl2 Bundle Upload", + redis_ttl_seconds, + { + "bundle-metadata": json.dumps({ + "speciesid": species_id, + "populationid": population_id, + "rqtl2-bundle-file": str(fullpath( + request.form["rqtl2_bundle_file"])), + "geno-dataset-id": request.form.get( + "geno-dataset-id", ""), + "probe-study-id": request.form.get( + "probe-study-id", ""), + "probe-dataset-id": request.form.get( + "probe-dataset-id", ""), + **({ + "platformid": probeset_study_by_id( + conn, + int(request.form["probe-study-id"]))["ChipId"] + } if bool(request.form.get("probe-study-id")) else {}) + }) + }), + redisuri, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + raise NotImplementedError |