diff options
Diffstat (limited to 'uploader/population')
| -rw-r--r-- | uploader/population/models.py | 24 | ||||
| -rw-r--r-- | uploader/population/rqtl2.py | 125 | ||||
| -rw-r--r-- | uploader/population/views.py | 72 |
3 files changed, 78 insertions, 143 deletions
diff --git a/uploader/population/models.py b/uploader/population/models.py index 6dcd85e..4d95065 100644 --- a/uploader/population/models.py +++ b/uploader/population/models.py @@ -26,13 +26,23 @@ def populations_by_species(conn: mdb.Connection, speciesid) -> tuple: return tuple() +__GENERIC_POPULATION_FAMILIES__ = ( + "Reference Populations (replicate average, SE, N)", + "Crosses and Heterogeneous Stock (individuals)", + "Groups Without Genotypes") -def population_families(conn) -> tuple: +def population_families(conn, species_id: int) -> tuple[str]: """Fetch the families under which populations are grouped.""" with conn.cursor(cursorclass=DictCursor) as cursor: + paramstr = ", ".join(["%s"] * len(__GENERIC_POPULATION_FAMILIES__)) cursor.execute( - "SELECT DISTINCT(Family) FROM InbredSet WHERE Family IS NOT NULL") - return tuple(row["Family"] for row in cursor.fetchall()) + "SELECT DISTINCT(Family) FROM InbredSet " + "WHERE SpeciesId=%s " + "AND Family IS NOT NULL " + f"AND Family NOT IN ({paramstr})", + (species_id, *__GENERIC_POPULATION_FAMILIES__)) + return __GENERIC_POPULATION_FAMILIES__ + tuple( + row["Family"] for row in cursor.fetchall()) def population_genetic_types(conn) -> tuple: @@ -47,9 +57,11 @@ def population_genetic_types(conn) -> tuple: def save_population(cursor: mdb.cursors.Cursor, population_details: dict) -> dict: """Save the population details to the db.""" cursor.execute("SELECT DISTINCT(Family), FamilyOrder FROM InbredSet " - "WHERE Family IS NOT NULL AND Family != '' " + "WHERE SpeciesId=%s " + "AND Family IS NOT NULL AND Family != '' " "AND FamilyOrder IS NOT NULL " - "ORDER BY FamilyOrder ASC") + "ORDER BY FamilyOrder ASC", + (population_details["SpeciesId"],)) _families = { row["Family"]: int(row["FamilyOrder"]) for row in cursor.fetchall() @@ -61,7 +73,7 @@ def save_population(cursor: mdb.cursors.Cursor, population_details: dict) -> dic **population_details, "FamilyOrder": _families.get( population_details["Family"], - max(_families.values())+1) + max((0,) + tuple(_families.values()))+1) } cursor.execute( "INSERT INTO InbredSet(" diff --git a/uploader/population/rqtl2.py b/uploader/population/rqtl2.py index 436eca0..97d4854 100644 --- a/uploader/population/rqtl2.py +++ b/uploader/population/rqtl2.py @@ -11,13 +11,11 @@ from typing import Union, Callable, Optional import MySQLdb as mdb from redis import Redis from MySQLdb.cursors import DictCursor -from werkzeug.utils import secure_filename from gn_libs.mysqldb import database_connection +from markupsafe import escape from flask import ( flash, - escape, request, - jsonify, url_for, redirect, Response, @@ -191,127 +189,6 @@ def trigger_rqtl2_bundle_qc( return jobid -def chunk_name(uploadfilename: str, chunkno: int) -> str: - """Generate chunk name from original filename and chunk number""" - if uploadfilename == "": - raise ValueError("Name cannot be empty!") - if chunkno < 1: - raise ValueError("Chunk number must be greater than zero") - return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}" - - -def chunks_directory(uniqueidentifier: str) -> Path: - """Compute the directory where chunks are temporarily stored.""" - if uniqueidentifier == "": - raise ValueError("Unique identifier cannot be empty!") - return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}") - - -@rqtl2.route(("<int:species_id>/populations/<int:population_id>/rqtl2/" - "/rqtl2-bundle-chunked"), - methods=["GET"]) -@require_login -def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"] - species_id: int, - population_id: int -): - """ - Extension to the `upload_rqtl2_bundle` endpoint above that provides a way - for testing whether all the chunks have been uploaded and to assist with - resuming a failed expression-data. - """ - fileid = request.args.get("resumableIdentifier", type=str) or "" - filename = request.args.get("resumableFilename", type=str) or "" - chunk = request.args.get("resumableChunkNumber", type=int) or 0 - if not(fileid or filename or chunk): - return jsonify({ - "message": "At least one required query parameter is missing.", - "error": "BadRequest", - "statuscode": 400 - }), 400 - - if Path(chunks_directory(fileid), - chunk_name(filename, chunk)).exists(): - return "OK" - - return jsonify({ - "message": f"Chunk {chunk} was not found.", - "error": "NotFound", - "statuscode": 404 - }), 404 - - -def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path: - """Merge the chunks into a single file.""" - with open(targetfile, "ab") as _target: - for chunkfile in chunkpaths: - with open(chunkfile, "rb") as _chunkdata: - _target.write(_chunkdata.read()) - - chunkfile.unlink() - return targetfile - - -@rqtl2.route(("<int:species_id>/population/<int:population_id>/rqtl2/upload/" - "/rqtl2-bundle-chunked"), - methods=["POST"]) -@require_login -def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int): - """ - Extension to the `upload_rqtl2_bundle` endpoint above that allows large - files to be uploaded in chunks. - - This should hopefully speed up uploads, and if done right, even enable - resumable uploads - """ - _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0 - _chunk = request.form.get("resumableChunkNumber", default=1, type=int) - _uploadfilename = request.form.get( - "resumableFilename", default="", type=str) or "" - _fileid = request.form.get( - "resumableIdentifier", default="", type=str) or "" - _targetfile = Path(app.config["UPLOAD_FOLDER"], _fileid) - - if _targetfile.exists(): - return jsonify({ - "message": ( - "A file with a similar unique identifier has previously been " - "uploaded and possibly is/has being/been processed."), - "error": "BadRequest", - "statuscode": 400 - }), 400 - - try: - # save chunk data - chunks_directory(_fileid).mkdir(exist_ok=True, parents=True) - request.files["file"].save(Path(chunks_directory(_fileid), - chunk_name(_uploadfilename, _chunk))) - - # Check whether upload is complete - chunkpaths = tuple( - Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk)) - for _achunk in range(1, _totalchunks+1)) - if all(_file.exists() for _file in chunkpaths): - # merge_files and clean up chunks - __merge_chunks__(_targetfile, chunkpaths) - chunks_directory(_fileid).rmdir() - jobid = trigger_rqtl2_bundle_qc( - species_id, population_id, _targetfile, _uploadfilename) - return url_for( - "expression-data.rqtl2.rqtl2_bundle_qc_status", jobid=jobid) - except Exception as exc:# pylint: disable=[broad-except] - msg = "Error processing uploaded file chunks." - app.logger.error(msg, exc_info=True, stack_info=True) - return jsonify({ - "message": msg, - "error": type(exc).__name__, - "error-description": " ".join(str(arg) for arg in exc.args), - "error-trace": traceback.format_exception(exc) - }), 500 - - return "OK" - - @rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>", methods=["GET", "POST"]) @require_login diff --git a/uploader/population/views.py b/uploader/population/views.py index 4f985f5..795ce81 100644 --- a/uploader/population/views.py +++ b/uploader/population/views.py @@ -2,16 +2,17 @@ import json import base64 +from markupsafe import escape from MySQLdb.cursors import DictCursor from gn_libs.mysqldb import database_connection from flask import (flash, request, - url_for, redirect, Blueprint, current_app as app) from uploader.samples.views import samplesbp +from uploader.flask_extensions import url_for from uploader.oauth2.client import oauth2_post from uploader.ui import make_template_renderer from uploader.authorisation import require_login @@ -19,11 +20,11 @@ from uploader.genotypes.views import genotypesbp from uploader.datautils import enumerate_sequence from uploader.phenotypes.views import phenotypesbp from uploader.expression_data.views import exprdatabp +from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler from uploader.input_validation import is_valid_representative_name -from uploader.species.models import (all_species, - species_by_id, - order_species_by_family) +from uploader.phenotypes.models import (dataset_phenotypes, + datasets_by_population) from .models import (save_population, population_families, @@ -48,7 +49,15 @@ def index(): if not bool(request.args.get("species_id")): return render_template( "populations/index.html", - species=order_species_by_family(all_species(conn))) + species=all_species(conn), + activelink="populations") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.list_species_populations")) + species = species_by_id(conn, request.args.get("species_id")) if not bool(species): flash("Invalid species identifier provided!", "alert-danger") @@ -93,7 +102,7 @@ def create_population(species_id: int): return render_template( "populations/create-population.html", species=species, - families = population_families(conn), + families = population_families(conn, species["SpeciesId"]), genetic_types = population_genetic_types(conn), mapping_methods=( {"id": "0", "value": "No mapping support"}, @@ -101,6 +110,7 @@ def create_population(species_id: int): {"id": "2", "value": "GEMMA"}, {"id": "3", "value": "R/qtl"}, {"id": "4", "value": "GEMMA, PLINK"}), + return_to=(request.args.get("return_to") or ""), activelink="create-population", **error_values) @@ -145,13 +155,21 @@ def create_population(species_id: int): "FullName": population_fullname, "InbredSetCode": request.form.get("population_code") or None, "Description": request.form.get("population_description") or None, - "Family": request.form.get("population_family") or None, + "Family": request.form.get("population_family", "").strip() or None, "MappingMethodId": request.form.get("population_mapping_method_id"), "GeneticType": request.form.get("population_genetic_type") or None }) def __flash_success__(_success): - flash("Successfully created resource.", "alert-success") + flash("Successfully created population " + f"{escape(new_population['FullName'])}.", + "alert-success") + return_to = request.form.get("return_to") or "" + if return_to: + return redirect(url_for( + return_to, + species_id=species["SpeciesId"], + population_id=new_population["InbredSetId"])) return redirect(url_for( "species.populations.view_population", species_id=species["SpeciesId"], @@ -177,10 +195,15 @@ def create_population(species_id: int): @require_login def view_population(species_id: int, population_id: int): """View the details of a population.""" + streamlined_ui = request.args.get("streamlined_ui") with database_connection(app.config["SQL_URI"]) as conn: species = species_by_id(conn, species_id) population = population_by_species_and_id(conn, species_id, population_id) + datasets = datasets_by_population(conn, species_id, population_id) error = False + if len(datasets) > 1: + error = True + flash("Got more than one dataset for the population.", "alert alert-danger") if not bool(species): flash("You must select a species.", "alert-danger") @@ -191,9 +214,32 @@ def view_population(species_id: int, population_id: int): error = True if error: - return redirect(url_for("species.populations.index")) + return redirect(url_for(("species.view_species" + if bool(streamlined_ui) + else "species.populations.index"), + species_id=species["SpeciesId"], + streamlined_ui=streamlined_ui)) + + _datasets = datasets_by_population( + conn, species["SpeciesId"], population["Id"]) + assert len(datasets) == 0 or len(datasets) == 1, ( + "We expect only one phenotypes dataset per population.") + _kwargs = { + "species": species, + "population": population, + "activelink": "view-population", + "streamlined_ui": streamlined_ui, + "view_under_construction": request.args.get( + "view_under_construction", False) + } + + if len(_datasets) == 1: + _dataset = _datasets[0] + _kwargs = { + **_kwargs, + "dataset": _dataset, + "phenotypes": enumerate_sequence( + dataset_phenotypes(conn, population["Id"], _dataset["Id"])) + } - return render_template("populations/view-population.html", - species=species, - population=population, - activelink="view-population") + return render_template("populations/view-population.html", **_kwargs) |