diff options
Diffstat (limited to 'uploader/phenotypes')
| -rw-r--r-- | uploader/phenotypes/models.py | 44 | ||||
| -rw-r--r-- | uploader/phenotypes/views.py | 182 |
2 files changed, 204 insertions, 22 deletions
diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py index eb5a189..9324601 100644 --- a/uploader/phenotypes/models.py +++ b/uploader/phenotypes/models.py @@ -1,6 +1,7 @@ """Database and utility functions for phenotypes.""" from typing import Optional from functools import reduce +from datetime import datetime import MySQLdb as mdb from MySQLdb.cursors import Cursor, DictCursor @@ -78,7 +79,7 @@ def __phenotype_se__(cursor: Cursor, xref_id: str) -> dict: """Fetch standard-error values (if they exist) for a phenotype.""" _sequery = ( - "SELECT pxr.Id AS xref_id, pxr.DataId, pse.error, nst.count " + "SELECT pxr.Id AS xref_id, pxr.DataId, str.Id AS StrainId, pse.error, nst.count " "FROM Phenotype AS pheno " "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " "INNER JOIN PublishSE AS pse ON pxr.DataId=pse.DataId " @@ -90,7 +91,12 @@ def __phenotype_se__(cursor: Cursor, "WHERE (str.SpeciesId, pxr.InbredSetId, pf.Id, pxr.Id)=(%s, %s, %s, %s)") cursor.execute(_sequery, (species_id, population_id, dataset_id, xref_id)) - return {row["xref_id"]: dict(row) for row in cursor.fetchall()} + return {(row["DataId"], row["StrainId"]): { + "xref_id": row["xref_id"], + "DataId": row["DataId"], + "error": row["error"], + "count": row["count"] + } for row in cursor.fetchall()} def __organise_by_phenotype__(pheno, row): """Organise disparate data rows into phenotype 'objects'.""" @@ -105,11 +111,10 @@ def __organise_by_phenotype__(pheno, row): "Units": row["Units"], "Pre_publication_abbreviation": row["Pre_publication_abbreviation"], "Post_publication_abbreviation": row["Post_publication_abbreviation"], + "xref_id": row["pxr.Id"], "data": { - #TOD0: organise these by DataId and StrainId **(_pheno["data"] if bool(_pheno) else {}), - row["pxr.Id"]: { - "xref_id": row["pxr.Id"], + (row["DataId"], row["StrainId"]): { "DataId": row["DataId"], "mean": row["mean"], "Locus": row["Locus"], @@ -168,7 +173,7 @@ def phenotype_by_id( species_id, population_id, dataset_id, - xref_id)).items()) + xref_id)).values()) } if bool(_pheno) and len(_pheno.keys()) > 1: raise Exception( @@ -198,3 +203,30 @@ def phenotypes_data(conn: mdb.Connection, cursor.execute(_query, (population_id, dataset_id)) debug_query(cursor) return tuple(dict(row) for row in cursor.fetchall()) + + +def save_new_dataset(cursor: Cursor, + population_id: int, + dataset_name: str, + dataset_fullname: str, + dataset_shortname: str) -> dict: + """Create a new phenotype dataset.""" + params = { + "population_id": population_id, + "dataset_name": dataset_name, + "dataset_fullname": dataset_fullname, + "dataset_shortname": dataset_shortname, + "created": datetime.now().date().isoformat(), + "public": 2, + "confidentiality": 0, + "users": None + } + cursor.execute( + "INSERT INTO PublishFreeze(Name, FullName, ShortName, CreateTime, " + "public, InbredSetId, confidentiality, AuthorisedUsers) " + "VALUES(%(dataset_name)s, %(dataset_fullname)s, %(dataset_shortname)s, " + "%(created)s, %(public)s, %(population_id)s, %(confidentiality)s, " + "%(users)s)", + params) + debug_query(cursor) + return {**params, "Id": cursor.lastrowid} diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py index 7fc3b08..02e8078 100644 --- a/uploader/phenotypes/views.py +++ b/uploader/phenotypes/views.py @@ -1,6 +1,13 @@ """Views handling ('classical') phenotypes.""" +import sys +import uuid +import json +from pathlib import Path from functools import wraps +from redis import Redis +from requests.models import Response +from MySQLdb.cursors import DictCursor from flask import (flash, request, url_for, @@ -9,6 +16,12 @@ from flask import (flash, render_template, current_app as app) +# from r_qtl import r_qtl2 as rqtl2 +from r_qtl import r_qtl2_qc as rqc +from r_qtl import exceptions as rqe + +from uploader import jobs +from uploader.files import save_file#, fullpath from uploader.oauth2.client import oauth2_post from uploader.authorisation import require_login from uploader.db_utils import database_connection @@ -18,10 +31,14 @@ from uploader.request_checks import with_species, with_population from uploader.datautils import safe_int, order_by_family, enumerate_sequence from uploader.population.models import (populations_by_species, population_by_species_and_id) +from uploader.input_validation import (encode_errors, + decode_errors, + is_valid_representative_name) from .models import (dataset_by_id, phenotype_by_id, phenotypes_count, + save_new_dataset, dataset_phenotypes, datasets_by_population) @@ -170,6 +187,8 @@ def view_dataset(# pylint: disable=[unused-argument] offset=start_at, limit=count), start=start_at+1), + start_from=start_at, + count=count, activelink="view-dataset") @@ -190,6 +209,31 @@ def view_phenotype(# pylint: disable=[unused-argument] **kwargs ): """View an individual phenotype from the dataset.""" + def __render__(privileges): + return render_template( + "phenotypes/view-phenotype.html", + species=species, + population=population, + dataset=dataset, + phenotype=phenotype_by_id(conn, + species["SpeciesId"], + population["Id"], + dataset["Id"], + xref_id), + privileges=(privileges + ### For demo! Do not commit this part + + ("group:resource:edit-resource", + "group:resource:delete-resource",) + ### END: For demo! Do not commit this part + ), + activelink="view-phenotype") + + def __fail__(error): + if isinstance(error, Response) and error.json() == "No linked resource!": + return __render__(tuple()) + return make_either_error_handler( + "There was an error fetching the roles and privileges.")(error) + with database_connection(app.config["SQL_URI"]) as conn: return oauth2_post( "/auth/resource/phenotypes/individual/linked-resource", @@ -200,19 +244,125 @@ def view_phenotype(# pylint: disable=[unused-argument] "xref_id": xref_id } ).then( - lambda resource: render_template( - "phenotypes/view-phenotype.html", - species=species, - population=population, - dataset=dataset, - phenotype=phenotype_by_id(conn, - species["SpeciesId"], - population["Id"], - dataset["Id"], - xref_id), - resource=resource, - activelink="view-phenotype") - ).either( - make_either_error_handler( - "There was an error fetching the roles and privileges."), - lambda resp: resp) + lambda resource: tuple( + privilege["privilege_id"] for role in resource["roles"] + for privilege in role["privileges"]) + ).then(__render__).either(__fail__, lambda resp: resp) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets/create", + methods=["GET", "POST"]) +@require_login +@with_population( + species_redirect_uri="species.populations.phenotypes.index", + redirect_uri="species.populations.phenotypes.select_population") +def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] + """Create a new phenotype dataset.""" + with (database_connection(app.config["SQL_URI"]) as conn, + conn.cursor(cursorclass=DictCursor) as cursor): + if request.method == "GET": + return render_template("phenotypes/create-dataset.html", + activelink="create-dataset", + species=species, + population=population, + **decode_errors( + request.args.get("error_values", ""))) + + form = request.form + _errors: tuple[tuple[str, str], ...] = tuple() + if not is_valid_representative_name( + (form.get("dataset-name") or "").strip()): + _errors = _errors + (("dataset-name", "Invalid dataset name."),) + + if not bool((form.get("dataset-fullname") or "").strip()): + _errors = _errors + (("dataset-fullname", + "You must provide a value for 'Full Name'."),) + + if bool(_errors) > 0: + return redirect(url_for( + "species.populations.phenotypes.create_dataset", + species_id=species["SpeciesId"], + population_id=population["Id"], + error_values=encode_errors(_errors, form))) + + dataset_shortname = ( + form["dataset-shortname"] or form["dataset-name"]).strip() + _pheno_dataset = save_new_dataset( + cursor, + population["Id"], + form["dataset-name"].strip(), + form["dataset-fullname"].strip(), + dataset_shortname) + return redirect(url_for("species.populations.phenotypes.list_datasets", + species_id=species["SpeciesId"], + population_id=population["Id"])) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/add-phenotypes", + methods=["GET", "POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# pylint: disable=[unused-argument, too-many-locals] + """Add one or more phenotypes to the dataset.""" + add_phenos_uri = redirect(url_for( + "species.populations.phenotypes.add_phenotypes", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"])) + _redisuri = app.config["REDIS_URL"] + _sqluri = app.config["SQL_URI"] + with (Redis.from_url(_redisuri, decode_responses=True) as rconn, + # database_connection(_sqluri) as conn, + # conn.cursor(cursorclass=DictCursor) as cursor + ): + if request.method == "GET": + return render_template("phenotypes/add-phenotypes.html", + species=species, + population=population, + dataset=dataset, + activelink="add-phenotypes") + + try: + ## Handle huge files here... + phenobundle = save_file(request.files["phenotypes-bundle"], + Path(app.config["UPLOAD_FOLDER"])) + rqc.validate_bundle(phenobundle) + except AssertionError as _aerr: + app.logger.debug("File upload error!", exc_info=True) + flash("Expected a zipped bundle of files with phenotypes' " + "information.", + "alert-danger") + return add_phenos_uri + except rqe.RQTLError as rqtlerr: + app.logger.debug("Bundle validation error!", exc_info=True) + flash("R/qtl2 Error: " + " ".join(rqtlerr.args), "alert-danger") + return add_phenos_uri + + _jobid = uuid.uuid4() + _namespace = jobs.jobsnamespace() + _ttl_seconds = app.config["JOBS_TTL_SECONDS"] + _job = jobs.initialise_job( + rconn, + _namespace, + str(_jobid), + [sys.executable, "-m", "scripts.rqtl2.phenotypes_qc", _sqluri, + _redisuri, _namespace, str(_jobid), str(species["SpeciesId"]), + str(population["Id"]), str(dataset["Id"]), "--redisexpiry", + str(_ttl_seconds)], "phenotype_qc", _ttl_seconds, + {"job-metadata": json.dumps({ + "speciesid": species["SpeciesId"], + "populationid": population["Id"], + "datasetid": dataset["Id"], + "bundle": str(phenobundle.absolute())})}) + # jobs.launch_job( + # _job, + # redisuri, + # f"{app.config['UPLOAD_FOLDER']}/job_errors") + + raise NotImplementedError("Please implement this...") |