aboutsummaryrefslogtreecommitdiff
path: root/uploader/expression_data
diff options
context:
space:
mode:
authorFrederick Muriuki Muriithi2024-09-23 14:30:15 -0500
committerFrederick Muriuki Muriithi2024-09-23 16:35:39 -0500
commit4285cc10e24d6410206329ba079406e9aa21cc30 (patch)
tree70f785460b14731634d0a4bd93ad5bbb9c5ba833 /uploader/expression_data
parent0b37b9b3fa4fead86787a705713645fa14530a54 (diff)
downloadgn-uploader-4285cc10e24d6410206329ba079406e9aa21cc30.tar.gz
Move R/qtl2 upload code under the populations package.
Diffstat (limited to 'uploader/expression_data')
-rw-r--r--uploader/expression_data/rqtl2.py1175
1 files changed, 0 insertions, 1175 deletions
diff --git a/uploader/expression_data/rqtl2.py b/uploader/expression_data/rqtl2.py
deleted file mode 100644
index a855699..0000000
--- a/uploader/expression_data/rqtl2.py
+++ /dev/null
@@ -1,1175 +0,0 @@
-"""Module to handle uploading of R/qtl2 bundles."""#pylint: disable=[too-many-lines]
-import sys
-import json
-import traceback
-from pathlib import Path
-from datetime import date
-from uuid import UUID, uuid4
-from functools import partial
-from zipfile import ZipFile, is_zipfile
-from typing import Union, Callable, Optional
-
-import MySQLdb as mdb
-from redis import Redis
-from MySQLdb.cursors import DictCursor
-from werkzeug.utils import secure_filename
-from flask import (
- flash,
- escape,
- request,
- jsonify,
- url_for,
- redirect,
- Response,
- Blueprint,
- render_template,
- current_app as app)
-
-from r_qtl import r_qtl2
-
-from uploader import jobs
-from uploader.files import save_file, fullpath
-from uploader.dbinsert import species as all_species
-from uploader.db_utils import with_db_connection, database_connection
-
-from uploader.authorisation import require_login
-from uploader.db.platforms import platform_by_id, platforms_by_species
-from uploader.db.averaging import averaging_methods, averaging_method_by_id
-from uploader.db.tissues import all_tissues, tissue_by_id, create_new_tissue
-from uploader.population.models import (save_population,
- populations_by_species,
- population_by_species_and_id)
-from uploader.species.models import species_by_id
-from uploader.db.datasets import (
- geno_dataset_by_id,
- geno_datasets_by_species_and_population,
-
- probeset_study_by_id,
- probeset_create_study,
- probeset_dataset_by_id,
- probeset_create_dataset,
- probeset_datasets_by_study,
- probeset_studies_by_species_and_population)
-
-rqtl2 = Blueprint("rqtl2", __name__)
-
-
-@rqtl2.route("/", methods=["GET", "POST"])
-@rqtl2.route("/select-species", methods=["GET", "POST"])
-@require_login
-def select_species():
- """Select the species."""
- if request.method == "GET":
- return render_template("rqtl2/index.html", species=with_db_connection(all_species))
-
- species_id = request.form.get("species_id")
- species = with_db_connection(
- lambda conn: species_by_id(conn, species_id))
- if bool(species):
- return redirect(url_for(
- "expression-data.rqtl2.select_population", species_id=species_id))
- flash("Invalid species or no species selected!", "alert-error error-rqtl2")
- return redirect(url_for("expression-data.rqtl2.select_species"))
-
-
-@rqtl2.route("/upload/species/<int:species_id>/select-population",
- methods=["GET", "POST"])
-@require_login
-def select_population(species_id: int):
- """Select/Create the population to organise data under."""
- with database_connection(app.config["SQL_URI"]) as conn:
- species = species_by_id(conn, species_id)
- if not bool(species):
- flash("Invalid species selected!", "alert-error error-rqtl2")
- return redirect(url_for("expression-data.rqtl2.select_species"))
-
- if request.method == "GET":
- return render_template(
- "rqtl2/select-population.html",
- species=species,
- populations=populations_by_species(conn, species_id))
-
- population = population_by_species_and_id(
- conn, species["SpeciesId"], request.form.get("inbredset_id"))
- if not bool(population):
- flash("Invalid Population!", "alert-error error-rqtl2")
- return redirect(
- url_for("expression-data.rqtl2.select_population", pgsrc="error"),
- code=307)
-
- return redirect(url_for("expression-data.rqtl2.upload_rqtl2_bundle",
- species_id=species["SpeciesId"],
- population_id=population["InbredSetId"]))
-
-
-@rqtl2.route("/upload/species/<int:species_id>/create-population",
- methods=["POST"])
-@require_login
-def create_population(species_id: int):
- """Create a new population for the given species."""
- population_page = redirect(url_for("expression-data.rqtl2.select_population",
- species_id=species_id))
- with database_connection(app.config["SQL_URI"]) as conn:
- species = species_by_id(conn, species_id)
- population_name = request.form.get("inbredset_name", "").strip()
- population_fullname = request.form.get("inbredset_fullname", "").strip()
- if not bool(species):
- flash("Invalid species!", "alert-error error-rqtl2")
- return redirect(url_for("expression-data.rqtl2.select_species"))
- if not bool(population_name):
- flash("Invalid Population Name!", "alert-error error-rqtl2")
- return population_page
- if not bool(population_fullname):
- flash("Invalid Population Full Name!", "alert-error error-rqtl2")
- return population_page
- new_population = save_population(conn, {
- "SpeciesId": species["SpeciesId"],
- "Name": population_name,
- "InbredSetName": population_fullname,
- "FullName": population_fullname,
- "Family": request.form.get("inbredset_family") or None,
- "Description": request.form.get("description") or None
- })
-
- flash("Population created successfully.", "alert-success")
- return redirect(
- url_for("expression-data.rqtl2.upload_rqtl2_bundle",
- species_id=species_id,
- population_id=new_population["population_id"],
- pgsrc="create-population"),
- code=307)
-
-
-class __RequestError__(Exception): #pylint: disable=[invalid-name]
- """Internal class to avoid pylint's `too-many-return-statements` error."""
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle"),
- methods=["GET", "POST"])
-@require_login
-def upload_rqtl2_bundle(species_id: int, population_id: int):
- """Allow upload of R/qtl2 bundle."""
- with database_connection(app.config["SQL_URI"]) as conn:
- species = species_by_id(conn, species_id)
- population = population_by_species_and_id(
- conn, species["SpeciesId"], population_id)
- if not bool(species):
- flash("Invalid species!", "alert-error error-rqtl2")
- return redirect(url_for("expression-data.rqtl2.select_species"))
- if not bool(population):
- flash("Invalid Population!", "alert-error error-rqtl2")
- return redirect(
- url_for("expression-data.rqtl2.select_population", pgsrc="error"),
- code=307)
- if request.method == "GET" or (
- request.method == "POST"
- and bool(request.args.get("pgsrc"))):
- return render_template("rqtl2/upload-rqtl2-bundle-step-01.html",
- species=species,
- population=population)
-
- try:
- app.logger.debug("Files in the form: %s", request.files)
- the_file = save_file(request.files["rqtl2_bundle_file"],
- Path(app.config["UPLOAD_FOLDER"]))
- except AssertionError:
- app.logger.debug(traceback.format_exc())
- flash("Please provide a valid R/qtl2 zip bundle.",
- "alert-error error-rqtl2")
- return redirect(url_for("expression-data.rqtl2.upload_rqtl2_bundle",
- species_id=species_id,
- population_id=population_id))
-
- if not is_zipfile(str(the_file)):
- app.logger.debug("The file is not a zip file.")
- raise __RequestError__("Invalid file! Expected a zip file.")
-
- jobid = trigger_rqtl2_bundle_qc(
- species_id,
- population_id,
- the_file,
- request.files["rqtl2_bundle_file"].filename)#type: ignore[arg-type]
- return redirect(url_for(
- "expression-data.rqtl2.rqtl2_bundle_qc_status", jobid=jobid))
-
-
-def trigger_rqtl2_bundle_qc(
- species_id: int,
- population_id: int,
- rqtl2bundle: Path,
- originalfilename: str
-) -> UUID:
- """Trigger QC on the R/qtl2 bundle."""
- redisuri = app.config["REDIS_URL"]
- with Redis.from_url(redisuri, decode_responses=True) as rconn:
- jobid = uuid4()
- redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"]
- jobs.launch_job(
- jobs.initialise_job(
- rconn,
- jobs.jobsnamespace(),
- str(jobid),
- [sys.executable, "-m", "scripts.qc_on_rqtl2_bundle",
- app.config["SQL_URI"], app.config["REDIS_URL"],
- jobs.jobsnamespace(), str(jobid), str(species_id),
- str(population_id), "--redisexpiry",
- str(redis_ttl_seconds)],
- "rqtl2-bundle-qc-job",
- redis_ttl_seconds,
- {"job-metadata": json.dumps({
- "speciesid": species_id,
- "populationid": population_id,
- "rqtl2-bundle-file": str(rqtl2bundle.absolute()),
- "original-filename": originalfilename})}),
- redisuri,
- f"{app.config['UPLOAD_FOLDER']}/job_errors")
- return jobid
-
-
-def chunk_name(uploadfilename: str, chunkno: int) -> str:
- """Generate chunk name from original filename and chunk number"""
- if uploadfilename == "":
- raise ValueError("Name cannot be empty!")
- if chunkno < 1:
- raise ValueError("Chunk number must be greater than zero")
- return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}"
-
-
-def chunks_directory(uniqueidentifier: str) -> Path:
- """Compute the directory where chunks are temporarily stored."""
- if uniqueidentifier == "":
- raise ValueError("Unique identifier cannot be empty!")
- return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}")
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle-chunked"),
- methods=["GET"])
-@require_login
-def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"]
- species_id: int,
- population_id: int
-):
- """
- Extension to the `upload_rqtl2_bundle` endpoint above that provides a way
- for testing whether all the chunks have been uploaded and to assist with
- resuming a failed expression-data.
- """
- fileid = request.args.get("resumableIdentifier", type=str) or ""
- filename = request.args.get("resumableFilename", type=str) or ""
- chunk = request.args.get("resumableChunkNumber", type=int) or 0
- if not(fileid or filename or chunk):
- return jsonify({
- "message": "At least one required query parameter is missing.",
- "error": "BadRequest",
- "statuscode": 400
- }), 400
-
- if Path(chunks_directory(fileid),
- chunk_name(filename, chunk)).exists():
- return "OK"
-
- return jsonify({
- "message": f"Chunk {chunk} was not found.",
- "error": "NotFound",
- "statuscode": 404
- }), 404
-
-
-def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path:
- """Merge the chunks into a single file."""
- with open(targetfile, "ab") as _target:
- for chunkfile in chunkpaths:
- with open(chunkfile, "rb") as _chunkdata:
- _target.write(_chunkdata.read())
-
- chunkfile.unlink()
- return targetfile
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle-chunked"),
- methods=["POST"])
-@require_login
-def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int):
- """
- Extension to the `upload_rqtl2_bundle` endpoint above that allows large
- files to be uploaded in chunks.
-
- This should hopefully speed up uploads, and if done right, even enable
- resumable uploads
- """
- _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0
- _chunk = request.form.get("resumableChunkNumber", default=1, type=int)
- _uploadfilename = request.form.get(
- "resumableFilename", default="", type=str) or ""
- _fileid = request.form.get(
- "resumableIdentifier", default="", type=str) or ""
- _targetfile = Path(app.config["UPLOAD_FOLDER"], _fileid)
-
- if _targetfile.exists():
- return jsonify({
- "message": (
- "A file with a similar unique identifier has previously been "
- "uploaded and possibly is/has being/been processed."),
- "error": "BadRequest",
- "statuscode": 400
- }), 400
-
- try:
- # save chunk data
- chunks_directory(_fileid).mkdir(exist_ok=True, parents=True)
- request.files["file"].save(Path(chunks_directory(_fileid),
- chunk_name(_uploadfilename, _chunk)))
-
- # Check whether upload is complete
- chunkpaths = tuple(
- Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk))
- for _achunk in range(1, _totalchunks+1))
- if all(_file.exists() for _file in chunkpaths):
- # merge_files and clean up chunks
- __merge_chunks__(_targetfile, chunkpaths)
- chunks_directory(_fileid).rmdir()
- jobid = trigger_rqtl2_bundle_qc(
- species_id, population_id, _targetfile, _uploadfilename)
- return url_for(
- "expression-data.rqtl2.rqtl2_bundle_qc_status", jobid=jobid)
- except Exception as exc:# pylint: disable=[broad-except]
- msg = "Error processing uploaded file chunks."
- app.logger.error(msg, exc_info=True, stack_info=True)
- return jsonify({
- "message": msg,
- "error": type(exc).__name__,
- "error-description": " ".join(str(arg) for arg in exc.args),
- "error-trace": traceback.format_exception(exc)
- }), 500
-
- return "OK"
-
-
-@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>",
- methods=["GET", "POST"])
-@require_login
-def rqtl2_bundle_qc_status(jobid: UUID):
- """Check the status of the QC jobs."""
- with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn,
- database_connection(app.config["SQL_URI"]) as dbconn):
- try:
- thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid)
- messagelistname = thejob.get("log-messagelist")
- logmessages = (rconn.lrange(messagelistname, 0, -1)
- if bool(messagelistname) else [])
- jobstatus = thejob["status"]
- if jobstatus == "error":
- return render_template("rqtl2/rqtl2-qc-job-error.html",
- job=thejob,
- errorsgeneric=json.loads(
- thejob.get("errors-generic", "[]")),
- errorsgeno=json.loads(
- thejob.get("errors-geno", "[]")),
- errorspheno=json.loads(
- thejob.get("errors-pheno", "[]")),
- errorsphenose=json.loads(
- thejob.get("errors-phenose", "[]")),
- errorsphenocovar=json.loads(
- thejob.get("errors-phenocovar", "[]")),
- messages=logmessages)
- if jobstatus == "success":
- jobmeta = json.loads(thejob["job-metadata"])
- species = species_by_id(dbconn, jobmeta["speciesid"])
- return render_template(
- "rqtl2/rqtl2-qc-job-results.html",
- species=species,
- population=population_by_species_and_id(
- dbconn, species["SpeciesId"], jobmeta["populationid"]),
- rqtl2bundle=Path(jobmeta["rqtl2-bundle-file"]).name,
- rqtl2bundleorig=jobmeta["original-filename"])
-
- def compute_percentage(thejob, filetype) -> Union[str, None]:
- if f"{filetype}-linecount" in thejob:
- return "100"
- if f"{filetype}-filesize" in thejob:
- percent = ((int(thejob.get(f"{filetype}-checked", 0))
- /
- int(thejob.get(f"{filetype}-filesize", 1)))
- * 100)
- return f"{percent:.2f}"
- return None
-
- return render_template(
- "rqtl2/rqtl2-qc-job-status.html",
- job=thejob,
- geno_percent=compute_percentage(thejob, "geno"),
- pheno_percent=compute_percentage(thejob, "pheno"),
- phenose_percent=compute_percentage(thejob, "phenose"),
- messages=logmessages)
- except jobs.JobNotFound:
- return render_template("rqtl2/no-such-job.html", jobid=jobid)
-
-
-def redirect_on_error(flaskroute, **kwargs):
- """Utility to redirect on error"""
- return redirect(url_for(flaskroute, **kwargs, pgsrc="error"),
- code=(307 if request.method == "POST" else 302))
-
-
-def check_species(conn: mdb.Connection, formargs: dict) -> Optional[
- tuple[str, Response]]:
- """
- Check whether the 'species_id' value is provided, and whether a
- corresponding species exists in the database.
-
- Maybe give the function a better name..."""
- speciespage = redirect_on_error("expression-data.rqtl2.select_species")
- if "species_id" not in formargs:
- return "You MUST provide the Species identifier.", speciespage
-
- if not bool(species_by_id(conn, formargs["species_id"])):
- return "No species with the provided identifier exists.", speciespage
-
- return None
-
-
-def check_population(conn: mdb.Connection,
- formargs: dict,
- species_id) -> Optional[tuple[str, Response]]:
- """
- Check whether the 'population_id' value is provided, and whether a
- corresponding population exists in the database.
-
- Maybe give the function a better name..."""
- poppage = redirect_on_error(
- "expression-data.rqtl2.select_species", species_id=species_id)
- if "population_id" not in formargs:
- return "You MUST provide the Population identifier.", poppage
-
- if not bool(population_by_species_and_id(
- conn, species_id, formargs["population_id"])):
- return "No population with the provided identifier exists.", poppage
-
- return None
-
-
-def check_r_qtl2_bundle(formargs: dict,
- species_id,
- population_id) -> Optional[tuple[str, Response]]:
- """Check for the existence of the R/qtl2 bundle."""
- fileuploadpage = redirect_on_error("expression-data.rqtl2.upload_rqtl2_bundle",
- species_id=species_id,
- population_id=population_id)
- if not "rqtl2_bundle_file" in formargs:
- return (
- "You MUST provide a R/qtl2 zip bundle for expression-data.", fileuploadpage)
-
- if not Path(fullpath(formargs["rqtl2_bundle_file"])).exists():
- return "No R/qtl2 bundle with the given name exists.", fileuploadpage
-
- return None
-
-
-def check_geno_dataset(conn: mdb.Connection,
- formargs: dict,
- species_id,
- population_id) -> Optional[tuple[str, Response]]:
- """Check for the Genotype dataset."""
- genodsetpg = redirect_on_error("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id)
- if not bool(formargs.get("geno-dataset-id")):
- return (
- "You MUST provide a valid Genotype dataset identifier", genodsetpg)
-
- with conn.cursor(cursorclass=DictCursor) as cursor:
- cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s",
- (formargs["geno-dataset-id"],))
- results = cursor.fetchall()
- if not bool(results):
- return ("No genotype dataset with the provided identifier exists.",
- genodsetpg)
- if len(results) > 1:
- return (
- "Data corruption: More than one genotype dataset with the same "
- "identifier.",
- genodsetpg)
-
- return None
-
-def check_tissue(
- conn: mdb.Connection,formargs: dict) -> Optional[tuple[str, Response]]:
- """Check for tissue/organ/biological material."""
- selectdsetpg = redirect_on_error("expression-data.rqtl2.select_dataset_info",
- species_id=formargs["species_id"],
- population_id=formargs["population_id"])
- if not bool(formargs.get("tissueid", "").strip()):
- return ("No tissue/organ/biological material provided.", selectdsetpg)
-
- with conn.cursor(cursorclass=DictCursor) as cursor:
- cursor.execute("SELECT * FROM Tissue WHERE Id=%s",
- (formargs["tissueid"],))
- results = cursor.fetchall()
- if not bool(results):
- return ("No tissue/organ with the provided identifier exists.",
- selectdsetpg)
-
- if len(results) > 1:
- return (
- "Data corruption: More than one tissue/organ with the same "
- "identifier.",
- selectdsetpg)
-
- return None
-
-
-def check_probe_study(conn: mdb.Connection,
- formargs: dict,
- species_id,
- population_id) -> Optional[tuple[str, Response]]:
- """Check for the ProbeSet study."""
- dsetinfopg = redirect_on_error("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id)
- if not bool(formargs.get("probe-study-id")):
- return "No probeset study was selected!", dsetinfopg
-
- if not bool(probeset_study_by_id(conn, formargs["probe-study-id"])):
- return ("No probeset study with the provided identifier exists",
- dsetinfopg)
-
- return None
-
-
-def check_probe_dataset(conn: mdb.Connection,
- formargs: dict,
- species_id,
- population_id) -> Optional[tuple[str, Response]]:
- """Check for the ProbeSet dataset."""
- dsetinfopg = redirect_on_error("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id)
- if not bool(formargs.get("probe-dataset-id")):
- return "No probeset dataset was selected!", dsetinfopg
-
- if not bool(probeset_dataset_by_id(conn, formargs["probe-dataset-id"])):
- return ("No probeset dataset with the provided identifier exists",
- dsetinfopg)
-
- return None
-
-
-def with_errors(endpointthunk: Callable, *checkfns):
- """Run 'endpointthunk' with error checking."""
- formargs = {**dict(request.args), **dict(request.form)}
- errors = tuple(item for item in (_fn(formargs=formargs) for _fn in checkfns)
- if item is not None)
- if len(errors) > 0:
- flash(errors[0][0], "alert-error error-rqtl2")
- return errors[0][1]
-
- return endpointthunk()
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle/select-geno-dataset"),
- methods=["POST"])
-@require_login
-def select_geno_dataset(species_id: int, population_id: int):
- """Select from existing geno datasets."""
- with database_connection(app.config["SQL_URI"]) as conn:
- def __thunk__():
- geno_dset = geno_datasets_by_species_and_population(
- conn, species_id, population_id)
- if not bool(geno_dset):
- flash("No genotype dataset was provided!",
- "alert-error error-rqtl2")
- return redirect(url_for("expression-data.rqtl2.select_geno_dataset",
- species_id=species_id,
- population_id=population_id,
- pgsrc="error"),
- code=307)
-
- flash("Genotype accepted", "alert-success error-rqtl2")
- return redirect(url_for("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id,
- pgsrc="expression-data.rqtl2.select_geno_dataset"),
- code=307)
-
- return with_errors(__thunk__,
- partial(check_species, conn=conn),
- partial(check_population, conn=conn,
- species_id=species_id),
- partial(check_r_qtl2_bundle,
- species_id=species_id,
- population_id=population_id),
- partial(check_geno_dataset,
- conn=conn,
- species_id=species_id,
- population_id=population_id))
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle/create-geno-dataset"),
- methods=["POST"])
-@require_login
-def create_geno_dataset(species_id: int, population_id: int):
- """Create a new geno dataset."""
- with database_connection(app.config["SQL_URI"]) as conn:
- def __thunk__():
- sgeno_page = redirect(url_for("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id,
- pgsrc="error"),
- code=307)
- errorclasses = "alert-error error-rqtl2 error-rqtl2-create-geno-dataset"
- if not bool(request.form.get("dataset-name")):
- flash("You must provide the dataset name", errorclasses)
- return sgeno_page
- if not bool(request.form.get("dataset-fullname")):
- flash("You must provide the dataset full name", errorclasses)
- return sgeno_page
- public = 2 if request.form.get("dataset-public") == "on" else 0
-
- with conn.cursor(cursorclass=DictCursor) as cursor:
- datasetname = request.form["dataset-name"]
- new_dataset = {
- "name": datasetname,
- "fname": request.form.get("dataset-fullname"),
- "sname": request.form.get("dataset-shortname") or datasetname,
- "today": date.today().isoformat(),
- "pub": public,
- "isetid": population_id
- }
- cursor.execute("SELECT * FROM GenoFreeze WHERE Name=%s",
- (datasetname,))
- results = cursor.fetchall()
- if bool(results):
- flash(
- f"A genotype dataset with name '{escape(datasetname)}' "
- "already exists.",
- errorclasses)
- return redirect(url_for("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id,
- pgsrc="error"),
- code=307)
- cursor.execute(
- "INSERT INTO GenoFreeze("
- "Name, FullName, ShortName, CreateTime, public, InbredSetId"
- ") "
- "VALUES("
- "%(name)s, %(fname)s, %(sname)s, %(today)s, %(pub)s, %(isetid)s"
- ")",
- new_dataset)
- flash("Created dataset successfully.", "alert-success")
- return render_template(
- "rqtl2/create-geno-dataset-success.html",
- species=species_by_id(conn, species_id),
- population=population_by_species_and_id(
- conn, species_id, population_id),
- rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
- geno_dataset={**new_dataset, "id": cursor.lastrowid})
-
- return with_errors(__thunk__,
- partial(check_species, conn=conn),
- partial(check_population, conn=conn, species_id=species_id),
- partial(check_r_qtl2_bundle,
- species_id=species_id,
- population_id=population_id))
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle/select-tissue"),
- methods=["POST"])
-@require_login
-def select_tissue(species_id: int, population_id: int):
- """Select from existing tissues."""
- with database_connection(app.config["SQL_URI"]) as conn:
- def __thunk__():
- if not bool(request.form.get("tissueid", "").strip()):
- flash("Invalid tissue selection!",
- "alert-error error-select-tissue error-rqtl2")
-
- return redirect(url_for("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id,
- pgsrc="expression-data.rqtl2.select_geno_dataset"),
- code=307)
-
- return with_errors(__thunk__,
- partial(check_species, conn=conn),
- partial(check_population,
- conn=conn,
- species_id=species_id),
- partial(check_r_qtl2_bundle,
- species_id=species_id,
- population_id=population_id),
- partial(check_geno_dataset,
- conn=conn,
- species_id=species_id,
- population_id=population_id))
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle/create-tissue"),
- methods=["POST"])
-@require_login
-def create_tissue(species_id: int, population_id: int):
- """Add new tissue, organ or biological material to the system."""
- form = request.form
- datasetinfopage = redirect(
- url_for("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id,
- pgsrc="expression-data.rqtl2.select_geno_dataset"),
- code=307)
- with database_connection(app.config["SQL_URI"]) as conn:
- tissuename = form.get("tissuename", "").strip()
- tissueshortname = form.get("tissueshortname", "").strip()
- if not bool(tissuename):
- flash("Organ/Tissue name MUST be provided.",
- "alert-error error-create-tissue error-rqtl2")
- return datasetinfopage
-
- if not bool(tissueshortname):
- flash("Organ/Tissue short name MUST be provided.",
- "alert-error error-create-tissue error-rqtl2")
- return datasetinfopage
-
- try:
- tissue = create_new_tissue(conn, tissuename, tissueshortname)
- flash("Tissue created successfully!", "alert-success")
- return render_template(
- "rqtl2/create-tissue-success.html",
- species=species_by_id(conn, species_id),
- population=population_by_species_and_id(
- conn, species_id, population_id),
- rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
- geno_dataset=geno_dataset_by_id(
- conn,
- int(request.form["geno-dataset-id"])),
- tissue=tissue)
- except mdb.IntegrityError as _ierr:
- flash("Tissue/Organ with that short name already exists!",
- "alert-error error-create-tissue error-rqtl2")
- return datasetinfopage
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle/select-probeset-study"),
- methods=["POST"])
-@require_login
-def select_probeset_study(species_id: int, population_id: int):
- """Select or create a probeset study."""
- with database_connection(app.config["SQL_URI"]) as conn:
- def __thunk__():
- summary_page = redirect(url_for("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id),
- code=307)
- if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))):
- flash("Invalid study selected!", "alert-error error-rqtl2")
- return summary_page
-
- return summary_page
- return with_errors(__thunk__,
- partial(check_species, conn=conn),
- partial(check_population,
- conn=conn,
- species_id=species_id),
- partial(check_r_qtl2_bundle,
- species_id=species_id,
- population_id=population_id),
- partial(check_geno_dataset,
- conn=conn,
- species_id=species_id,
- population_id=population_id),
- partial(check_tissue, conn=conn),
- partial(check_probe_study,
- conn=conn,
- species_id=species_id,
- population_id=population_id))
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle/select-probeset-dataset"),
- methods=["POST"])
-@require_login
-def select_probeset_dataset(species_id: int, population_id: int):
- """Select or create a probeset dataset."""
- with database_connection(app.config["SQL_URI"]) as conn:
- def __thunk__():
- summary_page = redirect(url_for("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id),
- code=307)
- if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))):
- flash("Invalid study selected!", "alert-error error-rqtl2")
- return summary_page
-
- return summary_page
-
- return with_errors(__thunk__,
- partial(check_species, conn=conn),
- partial(check_population,
- conn=conn,
- species_id=species_id),
- partial(check_r_qtl2_bundle,
- species_id=species_id,
- population_id=population_id),
- partial(check_geno_dataset,
- conn=conn,
- species_id=species_id,
- population_id=population_id),
- partial(check_tissue, conn=conn),
- partial(check_probe_study,
- conn=conn,
- species_id=species_id,
- population_id=population_id),
- partial(check_probe_dataset,
- conn=conn,
- species_id=species_id,
- population_id=population_id))
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle/create-probeset-study"),
- methods=["POST"])
-@require_login
-def create_probeset_study(species_id: int, population_id: int):
- """Create a new probeset study."""
- errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-study"
- with database_connection(app.config["SQL_URI"]) as conn:
- def __thunk__():
- form = request.form
- dataset_info_page = redirect(
- url_for("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id),
- code=307)
-
- if not (bool(form.get("platformid")) and
- bool(platform_by_id(conn, int(form["platformid"])))):
- flash("Invalid platform selected.", errorclasses)
- return dataset_info_page
-
- if not (bool(form.get("tissueid")) and
- bool(tissue_by_id(conn, int(form["tissueid"])))):
- flash("Invalid tissue selected.", errorclasses)
- return dataset_info_page
-
- studyname = form["studyname"]
- try:
- study = probeset_create_study(
- conn, population_id, int(form["platformid"]), int(form["tissueid"]),
- studyname, form.get("studyfullname") or "",
- form.get("studyshortname") or "")
- except mdb.IntegrityError as _ierr:
- flash(f"ProbeSet study with name '{escape(studyname)}' already "
- "exists.",
- errorclasses)
- return dataset_info_page
- return render_template(
- "rqtl2/create-probe-study-success.html",
- species=species_by_id(conn, species_id),
- population=population_by_species_and_id(
- conn, species_id, population_id),
- rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
- geno_dataset=geno_dataset_by_id(
- conn,
- int(request.form["geno-dataset-id"])),
- study=study)
-
- return with_errors(__thunk__,
- partial(check_species, conn=conn),
- partial(check_population,
- conn=conn,
- species_id=species_id),
- partial(check_r_qtl2_bundle,
- species_id=species_id,
- population_id=population_id),
- partial(check_geno_dataset,
- conn=conn,
- species_id=species_id,
- population_id=population_id),
- partial(check_tissue, conn=conn))
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle/create-probeset-dataset"),
- methods=["POST"])
-@require_login
-def create_probeset_dataset(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements]
- """Create a new probeset dataset."""
- errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-dataset"
- with database_connection(app.config["SQL_URI"]) as conn:
- def __thunk__():#pylint: disable=[too-many-return-statements]
- form = request.form
- summary_page = redirect(url_for("expression-data.rqtl2.select_dataset_info",
- species_id=species_id,
- population_id=population_id),
- code=307)
- if not bool(form.get("averageid")):
- flash("Averaging method not selected!", errorclasses)
- return summary_page
- if not bool(form.get("datasetname")):
- flash("Dataset name not provided!", errorclasses)
- return summary_page
- if not bool(form.get("datasetfullname")):
- flash("Dataset full name not provided!", errorclasses)
- return summary_page
-
- tissue = tissue_by_id(conn, form.get("tissueid", "").strip())
-
- study = probeset_study_by_id(conn, int(form["probe-study-id"]))
- if not bool(study):
- flash("Invalid ProbeSet study provided!", errorclasses)
- return summary_page
-
- avgmethod = averaging_method_by_id(conn, int(form["averageid"]))
- if not bool(avgmethod):
- flash("Invalid averaging method provided!", errorclasses)
- return summary_page
-
- try:
- dset = probeset_create_dataset(conn,
- int(form["probe-study-id"]),
- int(form["averageid"]),
- form["datasetname"],
- form["datasetfullname"],
- form["datasetshortname"],
- form["datasetpublic"] == "on",
- form.get(
- "datasetdatascale", "log2"))
- except mdb.IntegrityError as _ierr:
- app.logger.debug("Possible integrity error: %s", traceback.format_exc())
- flash(("IntegrityError: The data you provided has some errors: "
- f"{_ierr.args}"),
- errorclasses)
- return summary_page
- except Exception as _exc:# pylint: disable=[broad-except]
- app.logger.debug("Error creating ProbeSet dataset: %s",
- traceback.format_exc())
- flash(("There was a problem creating your dataset. Please try "
- "again."),
- errorclasses)
- return summary_page
- return render_template(
- "rqtl2/create-probe-dataset-success.html",
- species=species_by_id(conn, species_id),
- population=population_by_species_and_id(
- conn, species_id, population_id),
- rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
- geno_dataset=geno_dataset_by_id(
- conn,
- int(request.form["geno-dataset-id"])),
- tissue=tissue,
- study=study,
- avgmethod=avgmethod,
- dataset=dset)
-
- return with_errors(__thunk__,
- partial(check_species, conn=conn),
- partial(check_population,
- conn=conn,
- species_id=species_id),
- partial(check_r_qtl2_bundle,
- species_id=species_id,
- population_id=population_id),
- partial(check_geno_dataset,
- conn=conn,
- species_id=species_id,
- population_id=population_id),
- partial(check_tissue, conn=conn),
- partial(check_probe_study,
- conn=conn,
- species_id=species_id,
- population_id=population_id))
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle/dataset-info"),
- methods=["POST"])
-@require_login
-def select_dataset_info(species_id: int, population_id: int):
- """
- If `geno` files exist in the R/qtl2 bundle, prompt user to provide the
- dataset the genotypes belong to.
- """
- form = request.form
- with database_connection(app.config["SQL_URI"]) as conn:
- def __thunk__():
- species = species_by_id(conn, species_id)
- population = population_by_species_and_id(
- conn, species_id, population_id)
- thefile = fullpath(form["rqtl2_bundle_file"])
- with ZipFile(str(thefile), "r") as zfile:
- cdata = r_qtl2.control_data(zfile)
-
- geno_dataset = geno_dataset_by_id(
- conn,form.get("geno-dataset-id", "").strip())
- if "geno" in cdata and not bool(form.get("geno-dataset-id")):
- return render_template(
- "rqtl2/select-geno-dataset.html",
- species=species,
- population=population,
- rqtl2_bundle_file=thefile.name,
- datasets=geno_datasets_by_species_and_population(
- conn, species_id, population_id))
-
- tissue = tissue_by_id(conn, form.get("tissueid", "").strip())
- if "pheno" in cdata and not bool(tissue):
- return render_template(
- "rqtl2/select-tissue.html",
- species=species,
- population=population,
- rqtl2_bundle_file=thefile.name,
- geno_dataset=geno_dataset,
- studies=probeset_studies_by_species_and_population(
- conn, species_id, population_id),
- platforms=platforms_by_species(conn, species_id),
- tissues=all_tissues(conn))
-
- probeset_study = probeset_study_by_id(
- conn, form.get("probe-study-id", "").strip())
- if "pheno" in cdata and not bool(probeset_study):
- return render_template(
- "rqtl2/select-probeset-study-id.html",
- species=species,
- population=population,
- rqtl2_bundle_file=thefile.name,
- geno_dataset=geno_dataset,
- studies=probeset_studies_by_species_and_population(
- conn, species_id, population_id),
- platforms=platforms_by_species(conn, species_id),
- tissue=tissue)
- probeset_study = probeset_study_by_id(
- conn, int(form["probe-study-id"]))
-
- probeset_dataset = probeset_dataset_by_id(
- conn, form.get("probe-dataset-id", "").strip())
- if "pheno" in cdata and not bool(probeset_dataset):
- return render_template(
- "rqtl2/select-probeset-dataset.html",
- species=species,
- population=population,
- rqtl2_bundle_file=thefile.name,
- geno_dataset=geno_dataset,
- probe_study=probeset_study,
- tissue=tissue,
- datasets=probeset_datasets_by_study(
- conn, int(form["probe-study-id"])),
- avgmethods=averaging_methods(conn))
-
- return render_template("rqtl2/summary-info.html",
- species=species,
- population=population,
- rqtl2_bundle_file=thefile.name,
- geno_dataset=geno_dataset,
- tissue=tissue,
- probe_study=probeset_study,
- probe_dataset=probeset_dataset)
-
- return with_errors(__thunk__,
- partial(check_species, conn=conn),
- partial(check_population,
- conn=conn,
- species_id=species_id),
- partial(check_r_qtl2_bundle,
- species_id=species_id,
- population_id=population_id))
-
-
-@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
- "/rqtl2-bundle/confirm-bundle-details"),
- methods=["POST"])
-@require_login
-def confirm_bundle_details(species_id: int, population_id: int):
- """Confirm the details and trigger R/qtl2 bundle processing..."""
- redisuri = app.config["REDIS_URL"]
- with (database_connection(app.config["SQL_URI"]) as conn,
- Redis.from_url(redisuri, decode_responses=True) as rconn):
- def __thunk__():
- redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"]
- jobid = str(uuid4())
- _job = jobs.launch_job(
- jobs.initialise_job(
- rconn,
- jobs.jobsnamespace(),
- jobid,
- [
- sys.executable, "-m", "scripts.process_rqtl2_bundle",
- app.config["SQL_URI"], app.config["REDIS_URL"],
- jobs.jobsnamespace(), jobid, "--redisexpiry",
- str(redis_ttl_seconds)],
- "R/qtl2 Bundle Upload",
- redis_ttl_seconds,
- {
- "bundle-metadata": json.dumps({
- "speciesid": species_id,
- "populationid": population_id,
- "rqtl2-bundle-file": str(fullpath(
- request.form["rqtl2_bundle_file"])),
- "geno-dataset-id": request.form.get(
- "geno-dataset-id", ""),
- "probe-study-id": request.form.get(
- "probe-study-id", ""),
- "probe-dataset-id": request.form.get(
- "probe-dataset-id", ""),
- **({
- "platformid": probeset_study_by_id(
- conn,
- int(request.form["probe-study-id"]))["ChipId"]
- } if bool(request.form.get("probe-study-id")) else {})
- })
- }),
- redisuri,
- f"{app.config['UPLOAD_FOLDER']}/job_errors")
-
- return redirect(url_for("expression-data.rqtl2.rqtl2_processing_status",
- jobid=jobid))
-
- return with_errors(__thunk__,
- partial(check_species, conn=conn),
- partial(check_population,
- conn=conn,
- species_id=species_id),
- partial(check_r_qtl2_bundle,
- species_id=species_id,
- population_id=population_id),
- partial(check_geno_dataset,
- conn=conn,
- species_id=species_id,
- population_id=population_id),
- partial(check_probe_study,
- conn=conn,
- species_id=species_id,
- population_id=population_id),
- partial(check_probe_dataset,
- conn=conn,
- species_id=species_id,
- population_id=population_id))
-
-
-@rqtl2.route("/status/<uuid:jobid>")
-def rqtl2_processing_status(jobid: UUID):
- """Retrieve the status of the job processing the uploaded R/qtl2 bundle."""
- with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
- try:
- thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid)
-
- messagelistname = thejob.get("log-messagelist")
- logmessages = (rconn.lrange(messagelistname, 0, -1)
- if bool(messagelistname) else [])
-
- if thejob["status"] == "error":
- return render_template(
- "rqtl2/rqtl2-job-error.html", job=thejob, messages=logmessages)
- if thejob["status"] == "success":
- return render_template("rqtl2/rqtl2-job-results.html",
- job=thejob,
- messages=logmessages)
-
- return render_template(
- "rqtl2/rqtl2-job-status.html", job=thejob, messages=logmessages)
- except jobs.JobNotFound as _exc:
- return render_template("rqtl2/no-such-job.html", jobid=jobid)