diff options
25 files changed, 521 insertions, 303 deletions
diff --git a/scripts/cli/options.py b/scripts/cli/options.py index 70d2a27..58d3df4 100644 --- a/scripts/cli/options.py +++ b/scripts/cli/options.py @@ -44,3 +44,13 @@ def add_population_id(parser: ArgumentParser) -> ArgumentParser: type=int, help="The ID for the population to operate on.") return parser + + +def add_dataset_id(parser: ArgumentParser) -> ArgumentParser: + """Add dataset-id as a mandatory argument.""" + parser = add_population_id(parser) + parser.add_argument("dataset_id", + metavar="DATASET-ID", + type=int, + help="The ID for the dataset to operate on.") + return parser diff --git a/scripts/load_phenotypes_to_db.py b/scripts/load_phenotypes_to_db.py index e303bb3..31eb715 100644 --- a/scripts/load_phenotypes_to_db.py +++ b/scripts/load_phenotypes_to_db.py @@ -5,9 +5,9 @@ import json import time import logging import argparse -import datetime from pathlib import Path from zipfile import ZipFile +from datetime import datetime from typing import Any, Iterable from urllib.parse import urljoin from functools import reduce, partial @@ -198,13 +198,16 @@ save_phenotypes_n = partial(save_numeric_data, def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments,too-many-arguments] - authserver, - token, + auth_details, + resource_details, species, population, dataset, xrefdata): """Grant the user access to their data.""" + logger.info("Updating authorisation for the data.") + logger.debug("Resource details for the authorisation: %s", resource_details) + authserver, token = auth_details _tries = 0 _delay = 1 headers = { @@ -215,14 +218,14 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments return urljoin(authserver, endpoint) def __fetch_user_details__(): - logger.debug("… Fetching user details") + logger.info("… Fetching user details") return mrequests.get( authserveruri("/auth/user/"), headers=headers ) def __link_data__(user): - logger.debug("… linking uploaded data to user's group") + logger.info("… linking uploaded data to user's group") return mrequests.post( authserveruri("/auth/data/link/phenotype"), headers=headers, @@ -245,7 +248,7 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments }).then(lambda ld_results: (user, ld_results)) def __fetch_phenotype_category_details__(user, linkeddata): - logger.debug("… fetching phenotype category details") + logger.info("… fetching phenotype category details") return mrequests.get( authserveruri("/auth/resource/categories"), headers=headers @@ -258,20 +261,18 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments ) def __create_resource__(user, linkeddata, category): - logger.debug("… creating authorisation resource object") - now = datetime.datetime.now().isoformat() + logger.info("… creating authorisation resource object") return mrequests.post( authserveruri("/auth/resource/create"), headers=headers, json={ + **resource_details, "resource_category": category["resource_category_id"], - "resource_name": (f"{user['email']}—{dataset['Name']}—{now}—" - f"{len(xrefdata)} phenotypes"), "public": "off" }).then(lambda cr_results: (user, linkeddata, cr_results)) def __attach_data_to_resource__(user, linkeddata, resource): - logger.debug("… attaching data to authorisation resource object") + logger.info("… attaching data to authorisation resource object") return mrequests.post( authserveruri("/auth/resource/data/link"), headers=headers, @@ -288,8 +289,8 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments # This is hacky. If the auth already exists, something went wrong # somewhere. # This needs investigation to recover correctly. - logger.info( - "The authorisation for the data was already set up.") + logger.error( + "Error: The authorisation for the data was already set up.") return 0 logger.error("ERROR: Updating the authorisation for the data failed.") logger.debug( @@ -461,6 +462,25 @@ if __name__ == "__main__": logging.getLogger("uploader.phenotypes.models").setLevel(log_level) + def __parse_resource_details__(meta) -> dict: + """Parse out details regarding the wrapper resource from the metadata.""" + _key_mappings_ = { + # allow both 'data_*' and 'data*' for the metadata. + "data_description": "description", + "datadescription": "description" + } + return { + "resource_name": meta.get( + "dataname", + meta.get("data_name", + "Unnamed phenotypes - " + datetime.now().isoformat())), + "resource_metadata": { + rkey: meta[mkey] + for mkey, rkey in _key_mappings_.items() if mkey in meta + } + } + + def main(): """Entry-point for this script.""" args = parse_args() @@ -516,8 +536,10 @@ if __name__ == "__main__": # Update authorisations (break this down) — maybe loop until it works? logger.info("Updating authorisation.") _job_metadata = job["metadata"] - return update_auth(_job_metadata["authserver"], - _job_metadata["token"], + + return update_auth((_job_metadata["authserver"], + _job_metadata["token"]), + __parse_resource_details__(_job_metadata), *db_results) diff --git a/scripts/rqtl2/phenotypes_qc.py b/scripts/rqtl2/phenotypes_qc.py index 72d6c83..084c876 100644 --- a/scripts/rqtl2/phenotypes_qc.py +++ b/scripts/rqtl2/phenotypes_qc.py @@ -198,7 +198,7 @@ def qc_phenocovar_file( "-", "-", (f"File {filepath.name} is missing the {heading} heading " - "in the header line."))),) + "in the header row/line."))),) def collect_errors(errors_and_linecount, line): _errs, _lc = errors_and_linecount @@ -312,8 +312,9 @@ def qc_pheno_file(# pylint: disable=[too-many-locals, too-many-arguments, too-ma "header row", "-", ", ".join(_absent), - ("The following phenotype names do not exist in any of the " - f"provided phenocovar files: ({', '.join(_absent)})"))),) + ("The following trait names/identifiers do not exist in any of " + "the provided descriptions/covariates files: " + f"({', '.join(_absent)})"))),) def collect_errors(errors_and_linecount, line): _errs, _lc = errors_and_linecount diff --git a/scripts/run_qtlreaper.py b/scripts/run_qtlreaper.py index ab19da0..2269ea6 100644 --- a/scripts/run_qtlreaper.py +++ b/scripts/run_qtlreaper.py @@ -6,6 +6,7 @@ import time import secrets import logging import subprocess +import multiprocessing from pathlib import Path from functools import reduce from typing import Union, Iterator @@ -147,13 +148,17 @@ def dispatch(args: Namespace) -> int: _qtlreaper_main_output = args.working_dir.joinpath( f"main-output-{secrets.token_urlsafe(15)}.tsv")#type: ignore[attr-defined] + _qtlreaper_permu_output = args.working_dir.joinpath( + f"permu-output-{secrets.token_urlsafe(15)}.tsv") logger.debug("Main output filename: %s", _qtlreaper_main_output) with subprocess.Popen( ("qtlreaper", "--n_permutations", "1000", "--geno", _genofile, "--traits", _traitsfile, - "--main_output", _qtlreaper_main_output), + "--main_output", _qtlreaper_main_output, + "--permu_output", _qtlreaper_permu_output, + "--threads", str(int(1+(multiprocessing.cpu_count()/2)))), env=({**os.environ, "RUST_BACKTRACE": "full"} if logger.getEffectiveLevel() == logging.DEBUG else dict(os.environ))) as _qtlreaper: @@ -172,8 +177,17 @@ def dispatch(args: Namespace) -> int: logger.debug("Cleaning up temporary files.") # short-circuits to delete file if exists - _traitsfile.exists() and _traitsfile.unlink() - _qtlreaper_main_output.exists() and _qtlreaper_main_output.unlink() + if _traitsfile.exists(): + _traitsfile.unlink() + logger.info("Deleted generated traits' file for QTLReaper.") + + if _qtlreaper_main_output.exists(): + _qtlreaper_main_output.unlink() + logger.info("Deleted QTLReaper's main output file.") + + if _qtlreaper_permu_output.exists(): + _qtlreaper_permu_output.unlink() + logger.info("Deleted QTLReaper's permutations file.") if _qtlreaper.returncode != 0: return _qtlreaper.returncode diff --git a/uploader/__init__.py b/uploader/__init__.py index e00c726..afaa78d 100644 --- a/uploader/__init__.py +++ b/uploader/__init__.py @@ -131,13 +131,8 @@ def create_app(config: Optional[dict] = None): default_timeout=int(app.config["SESSION_FILESYSTEM_CACHE_TIMEOUT"])) setup_logging(app) - setup_modules_logging(app.logger, ( - "uploader.base_routes", - "uploader.flask_extensions", - "uploader.publications.models", - "uploader.publications.datatables", - "uploader.phenotypes.models", - "uploader.phenotypes.views")) + setup_modules_logging( + app.logger, tuple(app.config.get("LOGGABLE_MODULES", []))) # setup jinja2 symbols app.add_template_global(user_logged_in) diff --git a/uploader/default_settings.py b/uploader/default_settings.py index 6381a67..04e1c0a 100644 --- a/uploader/default_settings.py +++ b/uploader/default_settings.py @@ -39,3 +39,6 @@ JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before d ## --- Feature flags --- FEATURE_FLAGS_HTTP: list[str] = [] + +## --- Modules for which to log output --- +LOGGABLE_MODULES: list[str] = [] diff --git a/uploader/genotypes/models.py b/uploader/genotypes/models.py index 4c3e634..34d2cfe 100644 --- a/uploader/genotypes/models.py +++ b/uploader/genotypes/models.py @@ -31,16 +31,28 @@ def genotype_markers( species_id: int, offset: int = 0, limit: Optional[int] = None -) -> tuple[dict, ...]: +) -> tuple[tuple[dict, ...], int]: """Retrieve markers from the database.""" - _query = "SELECT * FROM Geno WHERE SpeciesId=%s" - if bool(limit) and limit > 0:# type: ignore[operator] - _query = _query + f" LIMIT {limit} OFFSET {offset}" + _query_template = ( + "SELECT %%COLS%% FROM Geno AS gno " + "WHERE gno.SpeciesId=%s " + "%%LIMIT%%") with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute(_query, (species_id,)) + cursor.execute( + _query_template.replace("%%LIMIT%%", "").replace( + "%%COLS%%", "COUNT(gno.Id) AS total_records"), + (species_id,)) + _total_records = cursor.fetchone()["total_records"] + cursor.execute( + _query_template.replace("%%COLS%%", "gno.*").replace( + "%%LIMIT%%", + (f"LIMIT {int(limit)} OFFSET {int(offset)}" + if bool(limit) and limit > 0 + else "")), + (species_id,)) debug_query(cursor, app.logger) - return tuple(dict(row) for row in cursor.fetchall()) + return tuple(dict(row) for row in cursor.fetchall()), _total_records def genotype_dataset( diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py index d991614..f27671c 100644 --- a/uploader/genotypes/views.py +++ b/uploader/genotypes/views.py @@ -1,8 +1,12 @@ """Views for the genotypes.""" +import logging + from MySQLdb.cursors import DictCursor +from pymonad.either import Left, Right, Either from gn_libs.mysqldb import database_connection from flask import (flash, request, + jsonify, redirect, Blueprint, render_template, @@ -16,8 +20,8 @@ from uploader.route_utils import generic_select_population from uploader.datautils import safe_int, enumerate_sequence from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler -from uploader.request_checks import with_species, with_population from uploader.population.models import population_by_species_and_id +from uploader.request_checks import with_species, with_dataset, with_population from .models import (genotype_markers, genotype_dataset, @@ -25,56 +29,17 @@ from .models import (genotype_markers, genotype_markers_count, genocode_by_population) +logger = logging.getLogger(__name__) genotypesbp = Blueprint("genotypes", __name__) render_template = make_template_renderer("genotypes") -@genotypesbp.route("populations/genotypes", methods=["GET"]) -@require_login -def index(): - """Direct entry-point for genotypes.""" - with database_connection(app.config["SQL_URI"]) as conn: - if not bool(request.args.get("species_id")): - return render_template("genotypes/index.html", - species=all_species(conn), - activelink="genotypes") - - species_id = request.args.get("species_id") - if species_id == "CREATE-SPECIES": - return redirect(url_for( - "species.create_species", - return_to="species.populations.genotypes.select_population")) - - species = species_by_id(conn, request.args.get("species_id")) - if not bool(species): - flash(f"Could not find species with ID '{request.args.get('species_id')}'!", - "alert-danger") - return redirect(url_for("species.populations.genotypes.index")) - return redirect(url_for("species.populations.genotypes.select_population", - species_id=species["SpeciesId"])) - - -@genotypesbp.route("/<int:species_id>/populations/genotypes/select-population", - methods=["GET"]) -@require_login -@with_species(redirect_uri="species.populations.genotypes.index") -def select_population(species: dict, species_id: int):# pylint: disable=[unused-argument] - """Select the population under which the genotypes go.""" - return generic_select_population( - species, - "genotypes/select-population.html", - request.args.get("population_id") or "", - "species.populations.genotypes.select_population", - "species.populations.genotypes.list_genotypes", - "genotypes", - "Invalid population selected!") - @genotypesbp.route( "/<int:species_id>/populations/<int:population_id>/genotypes", methods=["GET"]) @require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") +@with_population(species_redirect_uri="species.list_species", + redirect_uri="species.populations.list_species_populations") def list_genotypes(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """List genotype details for species and population.""" with database_connection(app.config["SQL_URI"]) as conn: @@ -92,34 +57,31 @@ def list_genotypes(species: dict, population: dict, **kwargs):# pylint: disable= @genotypesbp.route( - "/<int:species_id>/populations/<int:population_id>/genotypes/list-markers", + "/<int:species_id>/populations/<int:population_id>/genotypes/<int:dataset_id>/list-markers", methods=["GET"]) @require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") -def list_markers( - species: dict, - population: dict, - **kwargs -):# pylint: disable=[unused-argument] - """List a species' genetic markers.""" +@with_species(redirect_uri="species.populations.genotypes.list_genotypes") +def list_markers(species: dict, **_kwargs): + """List the markers that exist for this species.""" + args = request.args + offset = int(args.get("start") or 0) with database_connection(app.config["SQL_URI"]) as conn: - start_from = max(safe_int(request.args.get("start_from") or 0), 0) - count = safe_int(request.args.get("count") or 20) - return render_template("genotypes/list-markers.html", - species=species, - population=population, - total_markers=genotype_markers_count( - conn, species["SpeciesId"]), - start_from=start_from, - count=count, - markers=enumerate_sequence( - genotype_markers(conn, - species["SpeciesId"], - offset=start_from, - limit=count), - start=start_from+1), - activelink="list-markers") + markers, total_records = genotype_markers( + conn, + species["SpeciesId"], + offset=offset, + limit=int(args.get("length") or 0)) + return jsonify({ + **({"draw": int(args.get("draw"))} + if bool(args.get("draw") or False) + else {}), + "recordsTotal": total_records, + "recordsFiltered": len(markers), + "markers": tuple({**marker, "index": idx} + for idx, marker in + enumerate(markers, start=offset+1)) + }) + @genotypesbp.route( "/<int:species_id>/populations/<int:population_id>/genotypes/datasets/" @@ -132,14 +94,14 @@ def view_dataset(species_id: int, population_id: int, dataset_id: int): species = species_by_id(conn, species_id) if not bool(species): flash("Invalid species provided!", "alert-danger") - return redirect(url_for("species.populations.genotypes.index")) + return redirect(url_for("species.list_species")) population = population_by_species_and_id( conn, species_id, population_id) if not bool(population): flash("Invalid population selected!", "alert-danger") return redirect(url_for( - "species.populations.genotypes.select_population", + "species.populations.list_species_populations", species_id=species_id)) dataset = genotype_dataset(conn, species_id, population_id, dataset_id) @@ -162,25 +124,32 @@ def view_dataset(species_id: int, population_id: int, dataset_id: int): "create", methods=["GET", "POST"]) @require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") +@with_population(species_redirect_uri="species.list_species", + redirect_uri="species.populations.list_species_populations") def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """Create a genotype dataset.""" + if request.method == "GET": + return render_template("genotypes/create-dataset.html", + species=species, + population=population, + activelink="create-dataset") + with (database_connection(app.config["SQL_URI"]) as conn, conn.cursor(cursorclass=DictCursor) as cursor): - if request.method == "GET": - return render_template("genotypes/create-dataset.html", - species=species, - population=population, - activelink="create-dataset") - - form = request.form - new_dataset = save_new_dataset( - cursor, - population["Id"], - form["geno-dataset-name"], - form["geno-dataset-fullname"], - form["geno-dataset-shortname"]) + + def __save_dataset__() -> Either: + form = request.form + try: + return Right(save_new_dataset( + cursor, + population["Id"], + form["geno-dataset-name"], + form["geno-dataset-fullname"], + form["geno-dataset-shortname"])) + except Exception: + msg = "Error adding new Genotype dataset to database." + logger.error(msg, exc_info=True) + return Left(Exception(msg)) def __success__(_success): flash("Successfully created genotype dataset.", "alert-success") @@ -189,18 +158,20 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable= species_id=species["SpeciesId"], population_id=population["Id"])) - return oauth2_post( - "auth/resource/genotypes/create", - json={ - **dict(request.form), - "species_id": species["SpeciesId"], - "population_id": population["Id"], - "dataset_id": new_dataset["Id"], - "dataset_name": form["geno-dataset-name"], - "dataset_fullname": form["geno-dataset-fullname"], - "dataset_shortname": form["geno-dataset-shortname"], - "public": "on" - } + return __save_dataset__().then( + lambda new_dataset: oauth2_post( + "auth/resource/genotypes/create", + json={ + **dict(request.form), + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": new_dataset["Id"], + "dataset_name": new_dataset["Name"], + "dataset_fullname": new_dataset["FullName"], + "dataset_shortname": new_dataset["ShortName"], + "public": "on" + } + ) ).either( make_either_error_handler( "There was an error creating the genotype dataset."), diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py index 3946a0f..3d656d2 100644 --- a/uploader/phenotypes/models.py +++ b/uploader/phenotypes/models.py @@ -96,20 +96,34 @@ def dataset_phenotypes(# pylint: disable=[too-many-arguments, too-many-positiona xref_ids: tuple[int, ...] = tuple() ) -> tuple[dict, ...]: """Fetch the actual phenotypes.""" - _query = ( - "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, ist.InbredSetCode " + _narrow_by_ids = ( + f" AND pxr.Id IN ({', '.join(['%s'] * len(xref_ids))})" + if len(xref_ids) > 0 else "") + _narrow_by_limit = ( + f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + _pub_query = ( + "SELECT pub.* " + "FROM PublishXRef AS pxr " + "INNER JOIN Publication AS pub ON pxr.PublicationId=pub.Id " + "WHERE pxr.InbredSetId=%s") + _narrow_by_ids + _pheno_query = (( + "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, pxr.PublicationId, " + "ist.InbredSetCode " "FROM Phenotype AS pheno " "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " "INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId " "INNER JOIN InbredSet AS ist ON pf.InbredSetId=ist.Id " - "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + ( - f" AND pxr.Id IN ({', '.join(['%s'] * len(xref_ids))})" - if len(xref_ids) > 0 else "") + ( - f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + + _narrow_by_ids + + _narrow_by_limit) with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute(_query, (population_id, dataset_id) + xref_ids) + cursor.execute(_pub_query, (population_id,) + xref_ids) debug_query(cursor, logger) - return tuple(dict(row) for row in cursor.fetchall()) + _pubs = {row["Id"]: dict(row) for row in cursor.fetchall()} + cursor.execute(_pheno_query, (population_id, dataset_id) + xref_ids) + debug_query(cursor, logger) + return tuple({**dict(row), "publication": _pubs[row["PublicationId"]]} + for row in cursor.fetchall()) def __phenotype_se__(cursor: BaseCursor, xref_id, dataids_and_strainids): diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py index 23bc682..c03f3f5 100644 --- a/uploader/phenotypes/views.py +++ b/uploader/phenotypes/views.py @@ -317,6 +317,11 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable= dataset_shortname = ( form["dataset-shortname"] or form["dataset-name"]).strip() _pheno_dataset = save_new_dataset( + # It's not necessary to update the authorisation server to register + # new phenotype resource here, since each phenotype trait can, in + # theory, have its own access control allowing/disallowing access to + # it. In practice, however, we tend to gather multiple traits into a + # single resource for access control. cursor, population["Id"], form["dataset-name"].strip(), @@ -662,6 +667,8 @@ def review_job_data( conn, int(_job_metadata["publicationid"])) if _job_metadata.get("publicationid") else None), + user=session.user_details(), + timestamp=datetime.datetime.now().isoformat(), activelink="add-phenotypes") @@ -741,9 +748,15 @@ def load_data_to_database( "publication_id": _meta["publicationid"], "authserver": oauth2client.authserver_uri(), "token": token["access_token"], + "dataname": request.form["data_name"].strip(), "success_handler": ( "uploader.phenotypes.views" - ".load_phenotypes_success_handler") + ".load_phenotypes_success_handler"), + **{ + key: request.form[key] + for key in ("data_description",) + if key in request.form.keys() + } }, external_id=session.logged_in_user_id()) ).then( diff --git a/uploader/publications/models.py b/uploader/publications/models.py index dcfa02b..d913144 100644 --- a/uploader/publications/models.py +++ b/uploader/publications/models.py @@ -101,6 +101,20 @@ def fetch_publication_by_id(conn: Connection, publication_id: int) -> dict: return dict(_res) if _res else {} +def fetch_publications_by_ids( + conn: Connection, publications_ids: tuple[int, ...] +) -> tuple[dict, ...]: + """Fetch publications with the given IDs.""" + if len(publications_ids) == 0: + return tuple() + + with conn.cursor(cursorclass=DictCursor) as cursor: + paramstr = ", ".join(["%s"] * len(publications_ids)) + cursor.execute(f"SELECT * FROM Publication WHERE Id IN ({paramstr})", + tuple(publications_ids)) + return tuple(dict(row) for row in cursor.fetchall()) + + def fetch_publication_phenotypes( conn: Connection, publication_id: int) -> Iterable[dict]: """Fetch all phenotypes linked to this publication.""" diff --git a/uploader/publications/views.py b/uploader/publications/views.py index d9eb294..89e9f5d 100644 --- a/uploader/publications/views.py +++ b/uploader/publications/views.py @@ -1,5 +1,6 @@ """Endpoints for publications""" import json +import datetime from gn_libs.mysqldb import database_connection from flask import ( @@ -89,9 +90,12 @@ def create_publication(): } if request.method == "GET": + now = datetime.datetime.now() return render_template( "publications/create-publication.html", - get_args=_get_args) + get_args=_get_args, + current_year=now.year, + current_month=now.strftime("%B")) form = request.form authors = form.get("publication-authors").encode("utf8") if authors is None or authors == "": diff --git a/uploader/request_checks.py b/uploader/request_checks.py index f1d8027..84935f9 100644 --- a/uploader/request_checks.py +++ b/uploader/request_checks.py @@ -2,14 +2,20 @@ These are useful for reusability, and hence maintainability of the code. """ +import logging + +from typing import Callable from functools import wraps -from gn_libs.mysqldb import database_connection +from gn_libs.mysqldb import Connection, database_connection from flask import flash, url_for, redirect, current_app as app from uploader.species.models import species_by_id from uploader.population.models import population_by_species_and_id +logger = logging.getLogger(__name__) + + def with_species(redirect_uri: str): """Ensure the species actually exists.""" def __decorator__(function): @@ -28,7 +34,7 @@ def with_species(redirect_uri: str): "alert-danger") return redirect(url_for(redirect_uri)) except ValueError as _verr: - app.logger.debug( + logger.debug( "Exception converting value to integer: %s", kwargs.get("species_id"), exc_info=True) @@ -63,7 +69,7 @@ def with_population(species_redirect_uri: str, redirect_uri: str): "alert-danger") return select_population_uri except ValueError as _verr: - app.logger.debug( + logger.debug( "Exception converting value to integer: %s", kwargs.get("population_id"), exc_info=True) @@ -73,3 +79,45 @@ def with_population(species_redirect_uri: str, redirect_uri: str): return function(**{**kwargs, "population": population}) return __with_population__ return __decorator__ + + +def with_dataset( + species_redirect_uri: str, + population_redirect_uri: str, + redirect_uri: str, + dataset_by_id: Callable[ + [Connection, int, int, int], + dict] +): + """Ensure the dataset actually exists.""" + def __decorator__(func): + @wraps(func) + @with_population(species_redirect_uri, population_redirect_uri) + def __with_dataset__(**kwargs): + try: + _spcid = int(kwargs["species_id"]) + _popid = int(kwargs["population_id"]) + _dsetid = int(kwargs.get("dataset_id")) + select_dataset_uri = redirect(url_for( + redirect_uri, species_id=_spcid, population_id=_popid)) + if not bool(_dsetid): + flash("You need to select a valid 'dataset_id' value.", + "alert-danger") + return select_dataset_uri + with database_connection(app.config["SQL_URI"]) as conn: + dataset = dataset_by_id(conn, _spcid, _popid, _dsetid) + if not bool(dataset): + flash("You must select a valid dataset.", + "alert-danger") + return select_dataset_uri + except ValueError as _verr: + logger.debug( + "Exception converting 'dataset_id' to integer: %s", + kwargs.get("dataset_id"), + exc_info=True) + flash("Expected 'dataset_id' value to be an integer." + "alert-danger") + return select_dataset_uri + return func(**{**kwargs, "dataset": dataset}) + return __with_dataset__ + return __decorator__ diff --git a/uploader/static/css/theme.css b/uploader/static/css/theme.css index 276978f..6f5cb0c 100644 --- a/uploader/static/css/theme.css +++ b/uploader/static/css/theme.css @@ -81,10 +81,10 @@ table.dataTable tbody tr.selected td { background-color: #ffee99 !important; } -.form-group { +#frm-add-phenotypes .form-group { margin-bottom: 2em; padding-bottom: 0.2em; - border-bottom: solid gray 1px; + border-bottom: solid #A9A9A9 1px; } diff --git a/uploader/templates/genotypes/create-dataset.html b/uploader/templates/genotypes/create-dataset.html index 10331c1..ff174fb 100644 --- a/uploader/templates/genotypes/create-dataset.html +++ b/uploader/templates/genotypes/create-dataset.html @@ -35,13 +35,15 @@ id="txt-geno-dataset-name" name="geno-dataset-name" required="required" - class="form-control" /> + class="form-control" + value="{{population.Name}}Geno" + readonly="readonly" /> <small class="form-text text-muted"> <p>This is a short representative, but constrained name for the genotype dataset.<br /> - The field will only accept letters ('A-Za-z'), numbers (0-9), hyphens - and underscores. Any other character will cause the name to be - rejected.</p></small> + It is used internally by the Genenetwork system. Do not change this + value.</p> + </small> </div> <div class="form-group"> @@ -50,7 +52,8 @@ id="txt-geno-dataset-fullname" name="geno-dataset-fullname" required="required" - class="form-control" /> + class="form-control" + value="{{population.Name}} Genotypes" /> <small class="form-text text-muted"> <p>This is a longer, more descriptive name for your dataset.</p></small> </div> @@ -61,7 +64,8 @@ <input type="text" id="txt-geno-dataset-shortname" name="geno-dataset-shortname" - class="form-control" /> + class="form-control" + value="{{population.Name}}Geno" /> <small class="form-text text-muted"> <p>A short name for your dataset. If you leave this field blank, the short name will be set to the same value as the diff --git a/uploader/templates/genotypes/index.html b/uploader/templates/genotypes/index.html deleted file mode 100644 index b50ebc5..0000000 --- a/uploader/templates/genotypes/index.html +++ /dev/null @@ -1,32 +0,0 @@ -{%extends "genotypes/base.html"%} -{%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-select-species.html" import select_species_form%} - -{%block title%}Genotypes{%endblock%} - -{%block pagetitle%}Genotypes{%endblock%} - - -{%block contents%} -{{flash_all_messages()}} - -<div class="row"> - <p> - This section allows you to upload genotype information for your experiments, - in the case that you have not previously done so. - </p> - <p> - We'll need to link the genotypes to the species and population, so do please - go ahead and select those in the next two steps. - </p> -</div> - -<div class="row"> - {{select_species_form(url_for("species.populations.genotypes.index"), - species)}} -</div> -{%endblock%} - -{%block javascript%} -<script type="text/javascript" src="/static/js/species.js"></script> -{%endblock%} diff --git a/uploader/templates/genotypes/list-genotypes.html b/uploader/templates/genotypes/list-genotypes.html index a2b98c8..131576f 100644 --- a/uploader/templates/genotypes/list-genotypes.html +++ b/uploader/templates/genotypes/list-genotypes.html @@ -9,19 +9,6 @@ {{flash_all_messages()}} <div class="row"> - <h2>Genetic Markers</h2> - <p>There are a total of {{total_markers}} currently registered genetic markers - for the "{{species.FullName}}" species. You can click - <a href="{{url_for('species.populations.genotypes.list_markers', - species_id=species.SpeciesId, - population_id=population.Id)}}" - title="View genetic markers for species '{{species.FullName}}"> - this link to view the genetic markers - </a>. - </p> -</div> - -<div class="row"> <h2>Genotype Encoding</h2> <p> The genotype encoding used for the "{{population.FullName}}" population from @@ -56,59 +43,101 @@ </table> {%if genocode | length < 1%} - <a href="#add-genotype-encoding" - title="Add a genotype encoding system for this population" - class="btn btn-primary not-implemented"> - add genotype encoding + <div class="col"> + <a href="#add-genotype-encoding" + title="Add a genotype encoding system for this population" + class="btn btn-primary not-implemented"> + define genotype encoding </a> + </div> {%endif%} </div> -<div class="row text-danger"> - <h3>Some Important Concepts to Consider/Remember</h3> - <ul> - <li>Reference vs. Non-reference alleles</li> - <li>In <em>GenoCode</em> table, items are ordered by <strong>InbredSet</strong></li> - </ul> - <h3>Possible references</h3> - <ul> - <li>https://mr-dictionary.mrcieu.ac.uk/term/genotype/</li> - <li>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7363099/</li> - </ul> +<div class="row"> + <h2>Genotype Dataset</h2> +</div> + +{%if dataset is not none%} + +<div class="row"> + <h3>Dataset Details</h3> + <table class="table"> + <thead> + <tr> + <th>Name</th> + <th>Full Name</th> + </tr> + </thead> + + <tbody> + <tr> + <td>{{dataset.Name}}</td> + <td><a href="{{url_for('species.populations.genotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + title="View details regarding and manage dataset '{{dataset.FullName}}'" + target="_blank"> + {{dataset.FullName}}</a></td> + </tr> + </tbody> + </table> + + <p> + To see more information regarding this dataset (e.g. which markers have + sample allele data, the allele data itself, etc) click on the "Full Name" + link above.</p> +</div> + +<div class="row"> + <h3>Genotype Markers</h3> + + <div class="row"> + <p> + The table below lists all of the markers that exist for species + {{species.SpeciesName}} ({{species.FullName}}), regardless of whether + (or not) we have corresponding sample allele data for a particular marker. + </p> + <table id="tbl-genetic-markers" class="table compact stripe cell-border"> + <thead> + <tr> + <th title="">#</th> + <th title="">Index</th> + <th title="">Marker Name</th> + <th title="Chromosome">Chr</th> + <th title="Physical location of the marker in megabasepairs"> + Location (Mb)</th> + <th title="">Source</th> + <th title="">Source2</th> + </thead> + + <tbody> + {%for marker in markers%} + <tr> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + </tr> + {%endfor%} + </tbody> + </table> </div> +{%else%} + <div class="row"> - <h2>Genotype Datasets</h2> - - <p>The genotype data is organised under various genotype datasets. You can - click on the link for the relevant dataset to view a little more information - about it.</p> - - {%if dataset is not none%} - <table class="table"> - <thead> - <tr> - <th>Name</th> - <th>Full Name</th> - </tr> - </thead> - - <tbody> - <tr> - <td>{{dataset.Name}}</td> - <td><a href="{{url_for('species.populations.genotypes.view_dataset', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id)}}" - title="View details regarding and manage dataset '{{dataset.FullName}}'"> - {{dataset.FullName}}</a></td> - </tr> - </tbody> - </table> - {%else%} + <p> + Your genotype data will need to be under a dataset. Unfortunately there is + currently no dataset defined for this population. + </p> + <p class="text-warning"> <span class="glyphicon glyphicon-exclamation-sign"></span> - There is no genotype dataset defined for this population. + Click the button below to define the genotype dataset for this population. </p> <p> <a href="{{url_for('species.populations.genotypes.create_dataset', @@ -117,16 +146,81 @@ title="Create a new genotype dataset for the '{{population.FullName}}' population for the '{{species.FullName}}' species." class="btn btn-primary"> create new genotype dataset</a></p> - {%endif%} </div> -<div class="row text-warning"> - <p> - <span class="glyphicon glyphicon-exclamation-sign"></span> - <strong>NOTE</strong>: Currently the GN2 (and related) system(s) expect a - single genotype dataset. If there is more than one, the system apparently - fails in unpredictable ways. - </p> - <p>Fix this to allow multiple datasets, each with a different assembly from - all the rest.</p> + +{%endif%} + +<div class="row"> + <h2>Notes</h2> + <div class="row text-danger"> + <h3>Genetic Markers: Some Important Concepts to Consider/Remember</h3> + <ul> + <li>Reference vs. Non-reference alleles</li> + <li>In <em>GenoCode</em> table, items are ordered by <strong>InbredSet</strong></li> + </ul> + <h3>Possible references</h3> + <ul> + <li>https://mr-dictionary.mrcieu.ac.uk/term/genotype/</li> + <li>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7363099/</li> + </ul> + </div> + + <div class="row text-warning"> + <h3>Genotype Dataset</h3> + <p> + <span class="glyphicon glyphicon-exclamation-sign"></span> + <strong>NOTE</strong>: Currently the GN2 (and related) system(s) expect a + single genotype dataset per population. If there is more than one, the + system apparently fails in unpredictable ways. + </p> + </div> </div> + +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + + $(function() { + var dtGeneticMarkers = buildDataTable( + "#tbl-genetic-markers", + [], + [ + { + data: function(marker) { + return `<input type="checkbox" name="selected-markers" ` + + `id="chk-selected-markers-${marker.Id}-${marker.GenoFreezeId}" ` + + `value="${marker.Id}_${marker.GenoFreezeId}" ` + + `class="chk-row-select" />`; + } + }, + {data: 'index'}, + {data: "Name", searchable: true}, + {data: "Chr", searchable: true}, + {data: "Mb", searchable: true}, + {data: "Source", searchable: true}, + {data: "Source2", searchable: true} + ], + { + ajax: { + url: "{{url_for('species.populations.genotypes.list_markers', species_id=species.SpeciesId, population_id=population.Id, dataset_id=dataset.Id)}}", + dataSrc: "markers" + }, + paging: true, + scroller: true, + scrollY: "50vh", + scrollCollapse: true, + layout: { + top: "info", + topStart: null, + topEnd: null, + bottom: null, + bottomStart: null, + bottomEnd: null + } + }); + }); + +</script> {%endblock%} diff --git a/uploader/templates/genotypes/select-population.html b/uploader/templates/genotypes/select-population.html deleted file mode 100644 index acdd063..0000000 --- a/uploader/templates/genotypes/select-population.html +++ /dev/null @@ -1,25 +0,0 @@ -{%extends "genotypes/base.html"%} -{%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-display-species-card.html" import display_species_card%} -{%from "populations/macro-select-population.html" import select_population_form%} - -{%block title%}Genotypes{%endblock%} - -{%block pagetitle%}Genotypes{%endblock%} - - -{%block contents%} -{{flash_all_messages()}} - -<div class="row"> - {{select_population_form(url_for("species.populations.genotypes.select_population", species_id=species.SpeciesId), species, populations)}} -</div> -{%endblock%} - -{%block sidebarcontents%} -{{display_species_card(species)}} -{%endblock%} - -{%block javascript%} -<script type="text/javascript" src="/static/js/populations.js"></script> -{%endblock%} diff --git a/uploader/templates/genotypes/view-dataset.html b/uploader/templates/genotypes/view-dataset.html index 1c4eccf..d95a8e3 100644 --- a/uploader/templates/genotypes/view-dataset.html +++ b/uploader/templates/genotypes/view-dataset.html @@ -46,8 +46,9 @@ <div class="row"> <h2>Genotype Data</h2> - <p class="text-danger"> - Provide link to enable uploading of genotype data here.</p> + <div class="col" style="margin-bottom: 3px;"> + <a href="#" class="btn btn-primary not-implemented">upload genotypes</a> + </div> </div> {%endblock%} diff --git a/uploader/templates/index.html b/uploader/templates/index.html index 4b15eb1..6e9c777 100644 --- a/uploader/templates/index.html +++ b/uploader/templates/index.html @@ -129,7 +129,7 @@ you through fixing any issues before submission.</p> </div> -<div class="row" style="text-align: right;"> +<div class="row"> <div class="col"> <a href="{{authserver_authorise_uri()}}" title="Sign in to the system" diff --git a/uploader/templates/phenotypes/create-dataset.html b/uploader/templates/phenotypes/create-dataset.html index 6eced05..9963953 100644 --- a/uploader/templates/phenotypes/create-dataset.html +++ b/uploader/templates/phenotypes/create-dataset.html @@ -49,7 +49,7 @@ class="form-control" {%endif%} required="required" - disabled="disabled" /> + readonly="readonly" /> <small class="form-text text-muted"> <p>A short representative name for the dataset.</p> <p>Recommended: Use the population name and append "Publish" at the end. diff --git a/uploader/templates/phenotypes/review-job-data.html b/uploader/templates/phenotypes/review-job-data.html index c8355b2..0e8f119 100644 --- a/uploader/templates/phenotypes/review-job-data.html +++ b/uploader/templates/phenotypes/review-job-data.html @@ -70,6 +70,9 @@ {%endif%} {%endfor%} </ul> +</div> + +<div class="row"> <form id="frm-review-phenotype-data" method="POST" @@ -78,10 +81,38 @@ population_id=population.Id, dataset_id=dataset.Id)}}"> <input type="hidden" name="data-qc-job-id" value="{{job.jobid}}" /> - <input type="submit" - value="continue" - class="btn btn-primary" /> + <div class="form-group"> + <label for="txt-data-name">data name</label> + <input type="text" + id="txt-data-name" + class="form-control" + name="data_name" + title="A short, descriptive name for this data." + placeholder="{{user.email}} - {{dataset.Name}} - {{timestamp}}" + value="{{user.email}} - {{dataset.Name}} - {{timestamp}}" + required="required"> + <span class="form-text text-muted"> + This is a short, descriptive name for the data. It is useful to humans, + enabling them identify what traits each data "resource" wraps around. + </span> + </div> + + {%if view_under_construction%} + <div class="form-group"> + <label for="txt-data-description">data description</label> + <textarea id="txt-data-description" + class="form-control" + name="data_description" + title="A longer description for this data." + rows="5"></textarea> + <span class="form-text text-muted"> + </span> + </div> + {%endif%} + + <button type="submit" class="btn btn-primary">continue</button> </form> + </div> {%else%} <div class="row"> diff --git a/uploader/templates/phenotypes/view-dataset.html b/uploader/templates/phenotypes/view-dataset.html index 3bb2586..fc84757 100644 --- a/uploader/templates/phenotypes/view-dataset.html +++ b/uploader/templates/phenotypes/view-dataset.html @@ -148,14 +148,36 @@ return `<a href="${url.toString()}" target="_blank">` + `${pheno.InbredSetCode}_${pheno.xref_id}` + `</a>`; - } + }, + title: "Record", + visible: true, + searchable: true }, { data: function(pheno) { return (pheno.Post_publication_description || pheno.Original_description || pheno.Pre_publication_description); - } + }, + title: "Description", + visible: true, + searchable: true + }, + { + data: function(pheno) { + return pheno.publication.Title; + }, + title: "Publication Title", + visible: false, + searchable: true + }, + { + data: function(pheno) { + return pheno.publication.Authors; + }, + title: "Authors", + visible: false, + searchable: true } ], { diff --git a/uploader/templates/populations/view-population.html b/uploader/templates/populations/view-population.html index ac89bc7..29add29 100644 --- a/uploader/templates/populations/view-population.html +++ b/uploader/templates/populations/view-population.html @@ -95,14 +95,13 @@ id="genotypes-content" role="tabpanel" aria-labelledby="genotypes-content-tab"> - <p>This allows you to upload the data that concerns your genotypes.</p> - <p>Any samples/individuals/cases/strains that do not already exist in the - system will be added. This does not delete any existing data.</p> + <p>Click the button to view and manage genetic data for individuals in + this population.</p> <a href="{{url_for('species.populations.genotypes.list_genotypes', species_id=species.SpeciesId, population_id=population.Id)}}" title="Upload genotype information for the '{{population.FullName}}' population of the '{{species.FullName}}' species." - class="btn btn-primary">upload genotypes</a> + class="btn btn-primary">manage genotypes</a> </div> <div class="tab-pane fade" id="expression-data-content" role="tabpanel" aria-labelledby="expression-data-content-tab"> <p>Upload expression data (mRNA data) for this population.</p> diff --git a/uploader/templates/publications/create-publication.html b/uploader/templates/publications/create-publication.html index fb0127d..da5889e 100644 --- a/uploader/templates/publications/create-publication.html +++ b/uploader/templates/publications/create-publication.html @@ -91,22 +91,22 @@ class="col-sm-2 col-form-label"> Month</label> <div class="col-sm-4"> - <select class="form-control" + <select class="form-select" id="select-publication-month" name="publication-month"> <option value="">Select a month</option> - <option value="january">January</option> - <option value="february">February</option> - <option value="march">March</option> - <option value="april">April</option> - <option value="may">May</option> - <option value="june">June</option> - <option value="july">July</option> - <option value="august">August</option> - <option value="september">September</option> - <option value="october">October</option> - <option value="november">November</option> - <option value="december">December</option> + <option {%if current_month | lower == "january"%}selected="selected"{%endif%}value="january">January</option> + <option {%if current_month | lower == "february"%}selected="selected"{%endif%}value="february">February</option> + <option {%if current_month | lower == "march"%}selected="selected"{%endif%}value="march">March</option> + <option {%if current_month | lower == "april"%}selected="selected"{%endif%}value="april">April</option> + <option {%if current_month | lower == "may"%}selected="selected"{%endif%}value="may">May</option> + <option {%if current_month | lower == "june"%}selected="selected"{%endif%}value="june">June</option> + <option {%if current_month | lower == "july"%}selected="selected"{%endif%}value="july">July</option> + <option {%if current_month | lower == "august"%}selected="selected"{%endif%}value="august">August</option> + <option {%if current_month | lower == "september"%}selected="selected"{%endif%}value="september">September</option> + <option {%if current_month | lower == "october"%}selected="selected"{%endif%}value="october">October</option> + <option {%if current_month | lower == "november"%}selected="selected"{%endif%}value="november">November</option> + <option {%if current_month | lower == "december"%}selected="selected"{%endif%}value="december">December</option> </select> <span class="form-text text-muted">Month of publication</span> </div> @@ -119,7 +119,10 @@ id="txt-publication-year" name="publication-year" class="form-control" - min="1960" /> + min="1960" + max="{{current_year}}" + value="{{current_year or ''}}" + required="required" /> <span class="form-text text-muted">Year of publication</span> </div> </div> |