"""Module to handle uploading of R/qtl2 bundles."""#pylint: disable=[too-many-lines]
import sys
import json
import traceback
from pathlib import Path
from datetime import date
from uuid import UUID, uuid4
from functools import partial
from zipfile import ZipFile, is_zipfile
from typing import Union, Callable, Optional
import MySQLdb as mdb
from redis import Redis
from MySQLdb.cursors import DictCursor
from flask import (
flash,
escape,
request,
url_for,
redirect,
Response,
Blueprint,
render_template,
current_app as app)
from r_qtl import r_qtl2
from qc_app import jobs
from qc_app.files import save_file, fullpath
from qc_app.dbinsert import species as all_species
from qc_app.db_utils import with_db_connection, database_connection
from qc_app.db.platforms import platform_by_id, platforms_by_species
from qc_app.db.averaging import averaging_methods, averaging_method_by_id
from qc_app.db.tissues import all_tissues, tissue_by_id, create_new_tissue
from qc_app.db import (
species_by_id,
save_population,
populations_by_species,
population_by_species_and_id,)
from qc_app.db.datasets import (
geno_dataset_by_id,
geno_datasets_by_species_and_population,
probeset_study_by_id,
probeset_create_study,
probeset_dataset_by_id,
probeset_create_dataset,
probeset_datasets_by_study,
probeset_studies_by_species_and_population)
rqtl2 = Blueprint("rqtl2", __name__)
@rqtl2.route("/", methods=["GET", "POST"])
@rqtl2.route("/select-species", methods=["GET", "POST"])
def select_species():
"""Select the species."""
if request.method == "GET":
return render_template("rqtl2/index.html", species=with_db_connection(all_species))
species_id = request.form.get("species_id")
species = with_db_connection(
lambda conn: species_by_id(conn, species_id))
if bool(species):
return redirect(url_for(
"upload.rqtl2.select_population", species_id=species_id))
flash("Invalid species or no species selected!", "alert-error error-rqtl2")
return redirect(url_for("upload.rqtl2.select_species"))
@rqtl2.route("/upload/species/<int:species_id>/select-population",
methods=["GET", "POST"])
def select_population(species_id: int):
"""Select/Create the population to organise data under."""
with database_connection(app.config["SQL_URI"]) as conn:
species = species_by_id(conn, species_id)
if not bool(species):
flash("Invalid species selected!", "alert-error error-rqtl2")
return redirect(url_for("upload.rqtl2.select_species"))
if request.method == "GET":
return render_template(
"rqtl2/select-population.html",
species=species,
populations=populations_by_species(conn, species_id))
population = population_by_species_and_id(
conn, species["SpeciesId"], request.form.get("inbredset_id"))
if not bool(population):
flash("Invalid Population!", "alert-error error-rqtl2")
return redirect(
url_for("upload.rqtl2.select_population", pgsrc="error"),
code=307)
return redirect(url_for("upload.rqtl2.upload_rqtl2_bundle",
species_id=species["SpeciesId"],
population_id=population["InbredSetId"]))
@rqtl2.route("/upload/species/<int:species_id>/create-population",
methods=["POST"])
def create_population(species_id: int):
"""Create a new population for the given species."""
population_page = redirect(url_for("upload.rqtl2.select_population",
species_id=species_id))
with database_connection(app.config["SQL_URI"]) as conn:
species = species_by_id(conn, species_id)
population_name = request.form.get("inbredset_name", "").strip()
population_fullname = request.form.get("inbredset_fullname", "").strip()
if not bool(species):
flash("Invalid species!", "alert-error error-rqtl2")
return redirect(url_for("upload.rqtl2.select_species"))
if not bool(population_name):
flash("Invalid Population Name!", "alert-error error-rqtl2")
return population_page
if not bool(population_fullname):
flash("Invalid Population Full Name!", "alert-error error-rqtl2")
return population_page
new_population = save_population(conn, {
"SpeciesId": species["SpeciesId"],
"Name": population_name,
"InbredSetName": population_fullname,
"FullName": population_fullname,
"Family": request.form.get("inbredset_family") or None,
"Description": request.form.get("description") or None
})
flash("Population created successfully.", "alert-success")
return redirect(
url_for("upload.rqtl2.upload_rqtl2_bundle",
species_id=species_id,
population_id=new_population["population_id"],
pgsrc="create-population"),
code=307)
class __RequestError__(Exception): #pylint: disable=[invalid-name]
"""Internal class to avoid pylint's `too-many-return-statements` error."""
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle"),
methods=["GET", "POST"])
def upload_rqtl2_bundle(species_id: int, population_id: int):
"""Allow upload of R/qtl2 bundle."""
with database_connection(app.config["SQL_URI"]) as conn:
species = species_by_id(conn, species_id)
population = population_by_species_and_id(
conn, species["SpeciesId"], population_id)
if not bool(species):
flash("Invalid species!", "alert-error error-rqtl2")
return redirect(url_for("upload.rqtl2.select_species"))
if not bool(population):
flash("Invalid Population!", "alert-error error-rqtl2")
return redirect(
url_for("upload.rqtl2.select_population", pgsrc="error"),
code=307)
if request.method == "GET" or (
request.method == "POST"
and bool(request.args.get("pgsrc"))):
return render_template("rqtl2/upload-rqtl2-bundle-step-01.html",
species=species,
population=population)
try:
app.logger.debug("Files in the form: %s", request.files)
the_file = save_file(request.files["rqtl2_bundle_file"],
Path(app.config["UPLOAD_FOLDER"]))
except AssertionError:
app.logger.debug(traceback.format_exc())
flash("Please provide a valid R/qtl2 zip bundle.",
"alert-error error-rqtl2")
return redirect(url_for("upload.rqtl2.upload_rqtl2_bundle",
species_id=species_id,
population_id=population_id))
if not is_zipfile(str(the_file)):
app.logger.debug("The file is not a zip file.")
raise __RequestError__("Invalid file! Expected a zip file.")
redisuri = app.config["REDIS_URL"]
with Redis.from_url(redisuri, decode_responses=True) as rconn:
jobid = str(uuid4())
redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"]
jobs.launch_job(
jobs.initialise_job(
rconn,
jobs.jobsnamespace(),
jobid,
[sys.executable, "-m", "scripts.qc_on_rqtl2_bundle",
app.config["SQL_URI"], app.config["REDIS_URL"],
jobs.jobsnamespace(), jobid, "--redisexpiry",
str(redis_ttl_seconds)],
"rqtl2-bundle-qc-job",
redis_ttl_seconds,
{"job-metadata": json.dumps({
"speciesid": species_id,
"populationid": population_id,
"rqtl2-bundle-file": str(the_file.absolute()),
"original-filename": request.files["rqtl2_bundle_file"].filename})}),
redisuri,
f"{app.config['UPLOAD_FOLDER']}/job_errors")
return redirect(url_for(
"upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid))
@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>",
methods=["GET", "POST"])
def rqtl2_bundle_qc_status(jobid: UUID):
"""Check the status of the QC jobs."""
with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn,
database_connection(app.config["SQL_URI"]) as dbconn):
try:
thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid)
messagelistname = thejob.get("log-messagelist")
logmessages = (rconn.lrange(messagelistname, 0, -1)
if bool(messagelistname) else [])
jobstatus = thejob["status"]
if jobstatus == "error":
return render_template("rqtl2/rqtl2-qc-job-error.html",
job=thejob,
errorsgeneric=json.loads(
thejob.get("errors-generic", "[]")),
errorsgeno=json.loads(
thejob.get("errors-geno", "[]")),
errorspheno=json.loads(
thejob.get("errors-pheno", "[]")),
errorsphenose=json.loads(
thejob.get("errors-phenose", "[]")),
errorsphenocovar=json.loads(
thejob.get("errors-phenocovar", "[]")),
messages=logmessages)
if jobstatus == "success":
jobmeta = json.loads(thejob["job-metadata"])
species = species_by_id(dbconn, jobmeta["speciesid"])
return render_template(
"rqtl2/rqtl2-qc-job-results.html",
species=species,
population=population_by_species_and_id(
dbconn, species["SpeciesId"], jobmeta["populationid"]),
rqtl2bundle=Path(jobmeta["rqtl2-bundle-file"]).name,
rqtl2bundleorig=jobmeta["original-filename"])
def compute_percentage(thejob, filetype) -> Union[str, None]:
if f"{filetype}-linecount" in thejob:
return "100"
if f"{filetype}-filesize" in thejob:
percent = ((int(thejob.get(f"{filetype}-checked", 0))
/
int(thejob.get(f"{filetype}-filesize", 1)))
* 100)
return f"{percent:.2f}"
return None
return render_template(
"rqtl2/rqtl2-qc-job-status.html",
job=thejob,
geno_percent=compute_percentage(thejob, "geno"),
pheno_percent=compute_percentage(thejob, "pheno"),
phenose_percent=compute_percentage(thejob, "phenose"),
messages=logmessages)
except jobs.JobNotFound:
return render_template("rqtl2/no-such-job.html", jobid=jobid)
def redirect_on_error(flaskroute, **kwargs):
"""Utility to redirect on error"""
return redirect(url_for(flaskroute, **kwargs, pgsrc="error"),
code=(307 if request.method == "POST" else 302))
def check_species(conn: mdb.Connection, formargs: dict) -> Optional[
tuple[str, Response]]:
"""
Check whether the 'species_id' value is provided, and whether a
corresponding species exists in the database.
Maybe give the function a better name..."""
speciespage = redirect_on_error("upload.rqtl2.select_species")
if "species_id" not in formargs:
return "You MUST provide the Species identifier.", speciespage
if not bool(species_by_id(conn, formargs["species_id"])):
return "No species with the provided identifier exists.", speciespage
return None
def check_population(conn: mdb.Connection,
formargs: dict,
species_id) -> Optional[tuple[str, Response]]:
"""
Check whether the 'population_id' value is provided, and whether a
corresponding population exists in the database.
Maybe give the function a better name..."""
poppage = redirect_on_error(
"upload.rqtl2.select_species", species_id=species_id)
if "population_id" not in formargs:
return "You MUST provide the Population identifier.", poppage
if not bool(population_by_species_and_id(
conn, species_id, formargs["population_id"])):
return "No population with the provided identifier exists.", poppage
return None
def check_r_qtl2_bundle(formargs: dict,
species_id,
population_id) -> Optional[tuple[str, Response]]:
"""Check for the existence of the R/qtl2 bundle."""
fileuploadpage = redirect_on_error("upload.rqtl2.upload_rqtl2_bundle",
species_id=species_id,
population_id=population_id)
if not "rqtl2_bundle_file" in formargs:
return (
"You MUST provide a R/qtl2 zip bundle for upload.", fileuploadpage)
if not Path(fullpath(formargs["rqtl2_bundle_file"])).exists():
return "No R/qtl2 bundle with the given name exists.", fileuploadpage
return None
def check_geno_dataset(conn: mdb.Connection,
formargs: dict,
species_id,
population_id) -> Optional[tuple[str, Response]]:
"""Check for the Genotype dataset."""
genodsetpg = redirect_on_error("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id)
if not bool(formargs.get("geno-dataset-id")):
return (
"You MUST provide a valid Genotype dataset identifier", genodsetpg)
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s",
(formargs["geno-dataset-id"],))
results = cursor.fetchall()
if not bool(results):
return ("No genotype dataset with the provided identifier exists.",
genodsetpg)
if len(results) > 1:
return (
"Data corruption: More than one genotype dataset with the same "
"identifier.",
genodsetpg)
return None
def check_tissue(
conn: mdb.Connection,formargs: dict) -> Optional[tuple[str, Response]]:
"""Check for tissue/organ/biological material."""
selectdsetpg = redirect_on_error("upload.rqtl2.select_dataset_info",
species_id=formargs["species_id"],
population_id=formargs["population_id"])
if not bool(formargs.get("tissueid", "").strip()):
return ("No tissue/organ/biological material provided.", selectdsetpg)
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute("SELECT * FROM Tissue WHERE Id=%s",
(formargs["tissueid"],))
results = cursor.fetchall()
if not bool(results):
return ("No tissue/organ with the provided identifier exists.",
selectdsetpg)
if len(results) > 1:
return (
"Data corruption: More than one tissue/organ with the same "
"identifier.",
selectdsetpg)
return None
def check_probe_study(conn: mdb.Connection,
formargs: dict,
species_id,
population_id) -> Optional[tuple[str, Response]]:
"""Check for the ProbeSet study."""
dsetinfopg = redirect_on_error("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id)
if not bool(formargs.get("probe-study-id")):
return "No probeset study was selected!", dsetinfopg
if not bool(probeset_study_by_id(conn, formargs["probe-study-id"])):
return ("No probeset study with the provided identifier exists",
dsetinfopg)
return None
def check_probe_dataset(conn: mdb.Connection,
formargs: dict,
species_id,
population_id) -> Optional[tuple[str, Response]]:
"""Check for the ProbeSet dataset."""
dsetinfopg = redirect_on_error("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id)
if not bool(formargs.get("probe-dataset-id")):
return "No probeset dataset was selected!", dsetinfopg
if not bool(probeset_dataset_by_id(conn, formargs["probe-dataset-id"])):
return ("No probeset dataset with the provided identifier exists",
dsetinfopg)
return None
def with_errors(endpointthunk: Callable, *checkfns):
"""Run 'endpointthunk' with error checking."""
formargs = {**dict(request.args), **dict(request.form)}
errors = tuple(item for item in (_fn(formargs=formargs) for _fn in checkfns)
if item is not None)
if len(errors) > 0:
flash(errors[0][0], "alert-error error-rqtl2")
return errors[0][1]
return endpointthunk()
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/select-geno-dataset"),
methods=["POST"])
def select_geno_dataset(species_id: int, population_id: int):
"""Select from existing geno datasets."""
with database_connection(app.config["SQL_URI"]) as conn:
def __thunk__():
geno_dset = geno_datasets_by_species_and_population(
conn, species_id, population_id)
if not bool(geno_dset):
flash("No genotype dataset was provided!",
"alert-error error-rqtl2")
return redirect(url_for("upload.rqtl2.select_geno_dataset",
species_id=species_id,
population_id=population_id,
pgsrc="error"),
code=307)
flash("Genotype accepted", "alert-success error-rqtl2")
return redirect(url_for("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id,
pgsrc="upload.rqtl2.select_geno_dataset"),
code=307)
return with_errors(__thunk__,
partial(check_species, conn=conn),
partial(check_population, conn=conn,
species_id=species_id),
partial(check_r_qtl2_bundle,
species_id=species_id,
population_id=population_id),
partial(check_geno_dataset,
conn=conn,
species_id=species_id,
population_id=population_id))
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/create-geno-dataset"),
methods=["POST"])
def create_geno_dataset(species_id: int, population_id: int):
"""Create a new geno dataset."""
with database_connection(app.config["SQL_URI"]) as conn:
def __thunk__():
sgeno_page = redirect(url_for("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id,
pgsrc="error"),
code=307)
errorclasses = "alert-error error-rqtl2 error-rqtl2-create-geno-dataset"
if not bool(request.form.get("dataset-name")):
flash("You must provide the dataset name", errorclasses)
return sgeno_page
if not bool(request.form.get("dataset-fullname")):
flash("You must provide the dataset full name", errorclasses)
return sgeno_page
public = 2 if request.form.get("dataset-public") == "on" else 0
with conn.cursor(cursorclass=DictCursor) as cursor:
datasetname = request.form["dataset-name"]
new_dataset = {
"name": datasetname,
"fname": request.form.get("dataset-fullname"),
"sname": request.form.get("dataset-shortname") or datasetname,
"today": date.today().isoformat(),
"pub": public,
"isetid": population_id
}
cursor.execute("SELECT * FROM GenoFreeze WHERE Name=%s",
(datasetname,))
results = cursor.fetchall()
if bool(results):
flash(
f"A genotype dataset with name '{escape(datasetname)}' "
"already exists.",
errorclasses)
return redirect(url_for("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id,
pgsrc="error"),
code=307)
cursor.execute(
"INSERT INTO GenoFreeze("
"Name, FullName, ShortName, CreateTime, public, InbredSetId"
") "
"VALUES("
"%(name)s, %(fname)s, %(sname)s, %(today)s, %(pub)s, %(isetid)s"
")",
new_dataset)
flash("Created dataset successfully.", "alert-success")
return render_template(
"rqtl2/create-geno-dataset-success.html",
species=species_by_id(conn, species_id),
population=population_by_species_and_id(
conn, species_id, population_id),
rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
geno_dataset={**new_dataset, "id": cursor.lastrowid})
return with_errors(__thunk__,
partial(check_species, conn=conn),
partial(check_population, conn=conn, species_id=species_id),
partial(check_r_qtl2_bundle,
species_id=species_id,
population_id=population_id))
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/select-tissue"),
methods=["POST"])
def select_tissue(species_id: int, population_id: int):
"""Select from existing tissues."""
with database_connection(app.config["SQL_URI"]) as conn:
def __thunk__():
if not bool(request.form.get("tissueid", "").strip()):
flash("Invalid tissue selection!",
"alert-error error-select-tissue error-rqtl2")
return redirect(url_for("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id,
pgsrc="upload.rqtl2.select_geno_dataset"),
code=307)
return with_errors(__thunk__,
partial(check_species, conn=conn),
partial(check_population,
conn=conn,
species_id=species_id),
partial(check_r_qtl2_bundle,
species_id=species_id,
population_id=population_id),
partial(check_geno_dataset,
conn=conn,
species_id=species_id,
population_id=population_id))
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/create-tissue"),
methods=["POST"])
def create_tissue(species_id: int, population_id: int):
"""Add new tissue, organ or biological material to the system."""
form = request.form
datasetinfopage = redirect(
url_for("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id,
pgsrc="upload.rqtl2.select_geno_dataset"),
code=307)
with database_connection(app.config["SQL_URI"]) as conn:
tissuename = form.get("tissuename", "").strip()
tissueshortname = form.get("tissueshortname", "").strip()
if not bool(tissuename):
flash("Organ/Tissue name MUST be provided.",
"alert-error error-create-tissue error-rqtl2")
return datasetinfopage
if not bool(tissueshortname):
flash("Organ/Tissue short name MUST be provided.",
"alert-error error-create-tissue error-rqtl2")
return datasetinfopage
try:
tissue = create_new_tissue(conn, tissuename, tissueshortname)
flash("Tissue created successfully!", "alert-success")
return render_template(
"rqtl2/create-tissue-success.html",
species=species_by_id(conn, species_id),
population=population_by_species_and_id(
conn, species_id, population_id),
rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
geno_dataset=geno_dataset_by_id(
conn,
int(request.form["geno-dataset-id"])),
tissue=tissue)
except mdb.IntegrityError as _ierr:
flash("Tissue/Organ with that short name already exists!",
"alert-error error-create-tissue error-rqtl2")
return datasetinfopage
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/select-probeset-study"),
methods=["POST"])
def select_probeset_study(species_id: int, population_id: int):
"""Select or create a probeset study."""
with database_connection(app.config["SQL_URI"]) as conn:
def __thunk__():
summary_page = redirect(url_for("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id),
code=307)
if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))):
flash("Invalid study selected!", "alert-error error-rqtl2")
return summary_page
return summary_page
return with_errors(__thunk__,
partial(check_species, conn=conn),
partial(check_population,
conn=conn,
species_id=species_id),
partial(check_r_qtl2_bundle,
species_id=species_id,
population_id=population_id),
partial(check_geno_dataset,
conn=conn,
species_id=species_id,
population_id=population_id),
partial(check_tissue, conn=conn),
partial(check_probe_study,
conn=conn,
species_id=species_id,
population_id=population_id))
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/select-probeset-dataset"),
methods=["POST"])
def select_probeset_dataset(species_id: int, population_id: int):
"""Select or create a probeset dataset."""
with database_connection(app.config["SQL_URI"]) as conn:
def __thunk__():
summary_page = redirect(url_for("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id),
code=307)
if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))):
flash("Invalid study selected!", "alert-error error-rqtl2")
return summary_page
return summary_page
return with_errors(__thunk__,
partial(check_species, conn=conn),
partial(check_population,
conn=conn,
species_id=species_id),
partial(check_r_qtl2_bundle,
species_id=species_id,
population_id=population_id),
partial(check_geno_dataset,
conn=conn,
species_id=species_id,
population_id=population_id),
partial(check_tissue, conn=conn),
partial(check_probe_study,
conn=conn,
species_id=species_id,
population_id=population_id),
partial(check_probe_dataset,
conn=conn,
species_id=species_id,
population_id=population_id))
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/create-probeset-study"),
methods=["POST"])
def create_probeset_study(species_id: int, population_id: int):
"""Create a new probeset study."""
errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-study"
with database_connection(app.config["SQL_URI"]) as conn:
def __thunk__():
form = request.form
dataset_info_page = redirect(
url_for("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id),
code=307)
if not (bool(form.get("platformid")) and
bool(platform_by_id(conn, int(form["platformid"])))):
flash("Invalid platform selected.", errorclasses)
return dataset_info_page
if not (bool(form.get("tissueid")) and
bool(tissue_by_id(conn, int(form["tissueid"])))):
flash("Invalid tissue selected.", errorclasses)
return dataset_info_page
studyname = form["studyname"]
try:
study = probeset_create_study(
conn, population_id, int(form["platformid"]), int(form["tissueid"]),
studyname, form.get("studyfullname") or "",
form.get("studyshortname") or "")
except mdb.IntegrityError as _ierr:
flash(f"ProbeSet study with name '{escape(studyname)}' already "
"exists.",
errorclasses)
return dataset_info_page
return render_template(
"rqtl2/create-probe-study-success.html",
species=species_by_id(conn, species_id),
population=population_by_species_and_id(
conn, species_id, population_id),
rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
geno_dataset=geno_dataset_by_id(
conn,
int(request.form["geno-dataset-id"])),
study=study)
return with_errors(__thunk__,
partial(check_species, conn=conn),
partial(check_population,
conn=conn,
species_id=species_id),
partial(check_r_qtl2_bundle,
species_id=species_id,
population_id=population_id),
partial(check_geno_dataset,
conn=conn,
species_id=species_id,
population_id=population_id),
partial(check_tissue, conn=conn))
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/create-probeset-dataset"),
methods=["POST"])
def create_probeset_dataset(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements]
"""Create a new probeset dataset."""
errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-dataset"
with database_connection(app.config["SQL_URI"]) as conn:
def __thunk__():#pylint: disable=[too-many-return-statements]
form = request.form
summary_page = redirect(url_for("upload.rqtl2.select_dataset_info",
species_id=species_id,
population_id=population_id),
code=307)
if not bool(form.get("averageid")):
flash("Averaging method not selected!", errorclasses)
return summary_page
if not bool(form.get("datasetname")):
flash("Dataset name not provided!", errorclasses)
return summary_page
if not bool(form.get("datasetfullname")):
flash("Dataset full name not provided!", errorclasses)
return summary_page
tissue = tissue_by_id(conn, form.get("tissueid", "").strip())
study = probeset_study_by_id(conn, int(form["probe-study-id"]))
if not bool(study):
flash("Invalid ProbeSet study provided!", errorclasses)
return summary_page
avgmethod = averaging_method_by_id(conn, int(form["averageid"]))
if not bool(avgmethod):
flash("Invalid averaging method provided!", errorclasses)
return summary_page
try:
dset = probeset_create_dataset(conn,
int(form["probe-study-id"]),
int(form["averageid"]),
form["datasetname"],
form["datasetfullname"],
form["datasetshortname"],
form["datasetpublic"] == "on",
form.get(
"datasetdatascale", "log2"))
except mdb.IntegrityError as _ierr:
app.logger.debug("Possible integrity error: %s", traceback.format_exc())
flash(("IntegrityError: The data you provided has some errors: "
f"{_ierr.args}"),
errorclasses)
return summary_page
except Exception as _exc:# pylint: disable=[broad-except]
app.logger.debug("Error creating ProbeSet dataset: %s",
traceback.format_exc())
flash(("There was a problem creating your dataset. Please try "
"again."),
errorclasses)
return summary_page
return render_template(
"rqtl2/create-probe-dataset-success.html",
species=species_by_id(conn, species_id),
population=population_by_species_and_id(
conn, species_id, population_id),
rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
geno_dataset=geno_dataset_by_id(
conn,
int(request.form["geno-dataset-id"])),
tissue=tissue,
study=study,
avgmethod=avgmethod,
dataset=dset)
return with_errors(__thunk__,
partial(check_species, conn=conn),
partial(check_population,
conn=conn,
species_id=species_id),
partial(check_r_qtl2_bundle,
species_id=species_id,
population_id=population_id),
partial(check_geno_dataset,
conn=conn,
species_id=species_id,
population_id=population_id),
partial(check_tissue, conn=conn),
partial(check_probe_study,
conn=conn,
species_id=species_id,
population_id=population_id))
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/dataset-info"),
methods=["POST"])
def select_dataset_info(species_id: int, population_id: int):
"""
If `geno` files exist in the R/qtl2 bundle, prompt user to provide the
dataset the genotypes belong to.
"""
form = request.form
with database_connection(app.config["SQL_URI"]) as conn:
def __thunk__():
species = species_by_id(conn, species_id)
population = population_by_species_and_id(
conn, species_id, population_id)
thefile = fullpath(form["rqtl2_bundle_file"])
with ZipFile(str(thefile), "r") as zfile:
cdata = r_qtl2.control_data(zfile)
geno_dataset = geno_dataset_by_id(
conn,form.get("geno-dataset-id", "").strip())
if "geno" in cdata and not bool(form.get("geno-dataset-id")):
return render_template(
"rqtl2/select-geno-dataset.html",
species=species,
population=population,
rqtl2_bundle_file=thefile.name,
datasets=geno_datasets_by_species_and_population(
conn, species_id, population_id))
tissue = tissue_by_id(conn, form.get("tissueid", "").strip())
if "pheno" in cdata and not bool(tissue):
return render_template(
"rqtl2/select-tissue.html",
species=species,
population=population,
rqtl2_bundle_file=thefile.name,
geno_dataset=geno_dataset,
studies=probeset_studies_by_species_and_population(
conn, species_id, population_id),
platforms=platforms_by_species(conn, species_id),
tissues=all_tissues(conn))
probeset_study = probeset_study_by_id(
conn, form.get("probe-study-id", "").strip())
if "pheno" in cdata and not bool(probeset_study):
return render_template(
"rqtl2/select-probeset-study-id.html",
species=species,
population=population,
rqtl2_bundle_file=thefile.name,
geno_dataset=geno_dataset,
studies=probeset_studies_by_species_and_population(
conn, species_id, population_id),
platforms=platforms_by_species(conn, species_id),
tissue=tissue)
probeset_study = probeset_study_by_id(
conn, int(form["probe-study-id"]))
probeset_dataset = probeset_dataset_by_id(
conn, form.get("probe-dataset-id", "").strip())
if "pheno" in cdata and not bool(probeset_dataset):
return render_template(
"rqtl2/select-probeset-dataset.html",
species=species,
population=population,
rqtl2_bundle_file=thefile.name,
geno_dataset=geno_dataset,
probe_study=probeset_study,
tissue=tissue,
datasets=probeset_datasets_by_study(
conn, int(form["probe-study-id"])),
avgmethods=averaging_methods(conn))
return render_template("rqtl2/summary-info.html",
species=species,
population=population,
rqtl2_bundle_file=thefile.name,
geno_dataset=geno_dataset,
tissue=tissue,
probe_study=probeset_study,
probe_dataset=probeset_dataset)
return with_errors(__thunk__,
partial(check_species, conn=conn),
partial(check_population,
conn=conn,
species_id=species_id),
partial(check_r_qtl2_bundle,
species_id=species_id,
population_id=population_id))
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/confirm-bundle-details"),
methods=["POST"])
def confirm_bundle_details(species_id: int, population_id: int):
"""Confirm the details and trigger R/qtl2 bundle processing..."""
redisuri = app.config["REDIS_URL"]
with (database_connection(app.config["SQL_URI"]) as conn,
Redis.from_url(redisuri, decode_responses=True) as rconn):
def __thunk__():
redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"]
jobid = str(uuid4())
_job = jobs.launch_job(
jobs.initialise_job(
rconn,
jobs.jobsnamespace(),
jobid,
[
sys.executable, "-m", "scripts.process_rqtl2_bundle",
app.config["SQL_URI"], app.config["REDIS_URL"],
jobs.jobsnamespace(), jobid, "--redisexpiry",
str(redis_ttl_seconds)],
"R/qtl2 Bundle Upload",
redis_ttl_seconds,
{
"bundle-metadata": json.dumps({
"speciesid": species_id,
"populationid": population_id,
"rqtl2-bundle-file": str(fullpath(
request.form["rqtl2_bundle_file"])),
"geno-dataset-id": request.form.get(
"geno-dataset-id", ""),
"probe-study-id": request.form.get(
"probe-study-id", ""),
"probe-dataset-id": request.form.get(
"probe-dataset-id", ""),
**({
"platformid": probeset_study_by_id(
conn,
int(request.form["probe-study-id"]))["ChipId"]
} if bool(request.form.get("probe-study-id")) else {})
})
}),
redisuri,
f"{app.config['UPLOAD_FOLDER']}/job_errors")
return redirect(url_for("upload.rqtl2.rqtl2_processing_status",
jobid=jobid))
return with_errors(__thunk__,
partial(check_species, conn=conn),
partial(check_population,
conn=conn,
species_id=species_id),
partial(check_r_qtl2_bundle,
species_id=species_id,
population_id=population_id),
partial(check_geno_dataset,
conn=conn,
species_id=species_id,
population_id=population_id),
partial(check_probe_study,
conn=conn,
species_id=species_id,
population_id=population_id),
partial(check_probe_dataset,
conn=conn,
species_id=species_id,
population_id=population_id))
@rqtl2.route("/status/<uuid:jobid>")
def rqtl2_processing_status(jobid: UUID):
"""Retrieve the status of the job processing the uploaded R/qtl2 bundle."""
with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
try:
thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid)
messagelistname = thejob.get("log-messagelist")
logmessages = (rconn.lrange(messagelistname, 0, -1)
if bool(messagelistname) else [])
if thejob["status"] == "error":
return render_template(
"rqtl2/rqtl2-job-error.html", job=thejob, messages=logmessages)
if thejob["status"] == "success":
return render_template("rqtl2/rqtl2-job-results.html",
job=thejob,
messages=logmessages)
return render_template(
"rqtl2/rqtl2-job-status.html", job=thejob, messages=logmessages)
except jobs.JobNotFound as _exc:
return render_template("rqtl2/no-such-job.html", jobid=jobid)
