diff options
Diffstat (limited to 'uploader')
40 files changed, 803 insertions, 402 deletions
diff --git a/uploader/__init__.py b/uploader/__init__.py index 46689c5..afaa78d 100644 --- a/uploader/__init__.py +++ b/uploader/__init__.py @@ -131,12 +131,8 @@ def create_app(config: Optional[dict] = None): default_timeout=int(app.config["SESSION_FILESYSTEM_CACHE_TIMEOUT"])) setup_logging(app) - setup_modules_logging(app.logger, ( - "uploader.base_routes", - "uploader.flask_extensions", - "uploader.publications.models", - "uploader.publications.datatables", - "uploader.phenotypes.models")) + setup_modules_logging( + app.logger, tuple(app.config.get("LOGGABLE_MODULES", []))) # setup jinja2 symbols app.add_template_global(user_logged_in) diff --git a/uploader/default_settings.py b/uploader/default_settings.py index 6381a67..04e1c0a 100644 --- a/uploader/default_settings.py +++ b/uploader/default_settings.py @@ -39,3 +39,6 @@ JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before d ## --- Feature flags --- FEATURE_FLAGS_HTTP: list[str] = [] + +## --- Modules for which to log output --- +LOGGABLE_MODULES: list[str] = [] diff --git a/uploader/expression_data/dbinsert.py b/uploader/expression_data/dbinsert.py index 6d8ce80..7040698 100644 --- a/uploader/expression_data/dbinsert.py +++ b/uploader/expression_data/dbinsert.py @@ -94,7 +94,7 @@ def select_platform(): job = jobs.job(rconn, jobs.jobsnamespace(), job_id) if job: filename = job["filename"] - filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + filepath = f"{app.config['UPLOADS_DIRECTORY']}/{filename}" if os.path.exists(filepath): default_species = 1 gchips = genechips() @@ -367,7 +367,7 @@ def insert_data(): assert form.get("datasetid"), "dataset" filename = form["filename"] - filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + filepath = f"{app.config['UPLOADS_DIRECTORY']}/{filename}" redisurl = app.config["REDIS_URL"] if os.path.exists(filepath): with Redis.from_url(redisurl, decode_responses=True) as rconn: @@ -377,7 +377,7 @@ def insert_data(): form["species"], form["genechipid"], form["datasetid"], app.config["SQL_URI"], redisurl, app.config["JOBS_TTL_SECONDS"]), - redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors") + redisurl, f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return redirect(url_for("dbinsert.insert_status", job_id=job["jobid"])) return render_error(f"File '{filename}' no longer exists.") diff --git a/uploader/expression_data/views.py b/uploader/expression_data/views.py index 0b318b7..0e9b072 100644 --- a/uploader/expression_data/views.py +++ b/uploader/expression_data/views.py @@ -162,7 +162,7 @@ def upload_file(species_id: int, population_id: int): species=species, population=population) - upload_dir = app.config["UPLOAD_FOLDER"] + upload_dir = app.config["UPLOADS_DIRECTORY"] request_errors = errors(request) if request_errors: for error in request_errors: @@ -225,7 +225,7 @@ def parse_file(species_id: int, population_id: int): _errors = True if filename: - filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename) + filepath = os.path.join(app.config["UPLOADS_DIRECTORY"], filename) if not os.path.exists(filepath): flash("Selected file does not exist (any longer)", "alert-danger") _errors = True @@ -241,7 +241,7 @@ def parse_file(species_id: int, population_id: int): species_id, filepath, filetype,# type: ignore[arg-type] app.config["JOBS_TTL_SECONDS"]), redisurl, - f"{app.config['UPLOAD_FOLDER']}/job_errors") + f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return redirect(url_for("species.populations.expression-data.parse_status", species_id=species_id, @@ -263,7 +263,7 @@ def parse_status(species_id: int, population_id: int, job_id: str): return render_template("no_such_job.html", job_id=job_id), 400 error_filename = jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + job_id, f"{app.config['UPLOADS_DIRECTORY']}/job_errors") if os.path.exists(error_filename): stat = os.stat(error_filename) if stat.st_size > 0: @@ -345,7 +345,7 @@ def fail(species_id: int, population_id: int, job_id: str): if job: error_filename = jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + job_id, f"{app.config['UPLOADS_DIRECTORY']}/job_errors") if os.path.exists(error_filename): stat = os.stat(error_filename) if stat.st_size > 0: diff --git a/uploader/flask_extensions.py b/uploader/flask_extensions.py index 83d25aa..0fc774a 100644 --- a/uploader/flask_extensions.py +++ b/uploader/flask_extensions.py @@ -11,7 +11,8 @@ from flask import ( logger = logging.getLogger(__name__) -def __fetch_flags__(): +def fetch_flags(): + """Fetch get arguments that are defined as feature flags.""" flags = {} for flag in app.config["FEATURE_FLAGS_HTTP"]: flag_value = (request.args.get(flag) or request.form.get(flag) or "").strip() @@ -38,7 +39,7 @@ def url_for( _scheme=_scheme, _external=_external, **values, - **__fetch_flags__()) + **fetch_flags()) def render_template(template_name_or_list, **context: Any) -> str: @@ -47,5 +48,5 @@ def render_template(template_name_or_list, **context: Any) -> str: template_name_or_list, **{ **context, - **__fetch_flags__() # override any flag values + **fetch_flags() # override any flag values }) diff --git a/uploader/genotypes/models.py b/uploader/genotypes/models.py index 4c3e634..34d2cfe 100644 --- a/uploader/genotypes/models.py +++ b/uploader/genotypes/models.py @@ -31,16 +31,28 @@ def genotype_markers( species_id: int, offset: int = 0, limit: Optional[int] = None -) -> tuple[dict, ...]: +) -> tuple[tuple[dict, ...], int]: """Retrieve markers from the database.""" - _query = "SELECT * FROM Geno WHERE SpeciesId=%s" - if bool(limit) and limit > 0:# type: ignore[operator] - _query = _query + f" LIMIT {limit} OFFSET {offset}" + _query_template = ( + "SELECT %%COLS%% FROM Geno AS gno " + "WHERE gno.SpeciesId=%s " + "%%LIMIT%%") with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute(_query, (species_id,)) + cursor.execute( + _query_template.replace("%%LIMIT%%", "").replace( + "%%COLS%%", "COUNT(gno.Id) AS total_records"), + (species_id,)) + _total_records = cursor.fetchone()["total_records"] + cursor.execute( + _query_template.replace("%%COLS%%", "gno.*").replace( + "%%LIMIT%%", + (f"LIMIT {int(limit)} OFFSET {int(offset)}" + if bool(limit) and limit > 0 + else "")), + (species_id,)) debug_query(cursor, app.logger) - return tuple(dict(row) for row in cursor.fetchall()) + return tuple(dict(row) for row in cursor.fetchall()), _total_records def genotype_dataset( diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py index d991614..f27671c 100644 --- a/uploader/genotypes/views.py +++ b/uploader/genotypes/views.py @@ -1,8 +1,12 @@ """Views for the genotypes.""" +import logging + from MySQLdb.cursors import DictCursor +from pymonad.either import Left, Right, Either from gn_libs.mysqldb import database_connection from flask import (flash, request, + jsonify, redirect, Blueprint, render_template, @@ -16,8 +20,8 @@ from uploader.route_utils import generic_select_population from uploader.datautils import safe_int, enumerate_sequence from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler -from uploader.request_checks import with_species, with_population from uploader.population.models import population_by_species_and_id +from uploader.request_checks import with_species, with_dataset, with_population from .models import (genotype_markers, genotype_dataset, @@ -25,56 +29,17 @@ from .models import (genotype_markers, genotype_markers_count, genocode_by_population) +logger = logging.getLogger(__name__) genotypesbp = Blueprint("genotypes", __name__) render_template = make_template_renderer("genotypes") -@genotypesbp.route("populations/genotypes", methods=["GET"]) -@require_login -def index(): - """Direct entry-point for genotypes.""" - with database_connection(app.config["SQL_URI"]) as conn: - if not bool(request.args.get("species_id")): - return render_template("genotypes/index.html", - species=all_species(conn), - activelink="genotypes") - - species_id = request.args.get("species_id") - if species_id == "CREATE-SPECIES": - return redirect(url_for( - "species.create_species", - return_to="species.populations.genotypes.select_population")) - - species = species_by_id(conn, request.args.get("species_id")) - if not bool(species): - flash(f"Could not find species with ID '{request.args.get('species_id')}'!", - "alert-danger") - return redirect(url_for("species.populations.genotypes.index")) - return redirect(url_for("species.populations.genotypes.select_population", - species_id=species["SpeciesId"])) - - -@genotypesbp.route("/<int:species_id>/populations/genotypes/select-population", - methods=["GET"]) -@require_login -@with_species(redirect_uri="species.populations.genotypes.index") -def select_population(species: dict, species_id: int):# pylint: disable=[unused-argument] - """Select the population under which the genotypes go.""" - return generic_select_population( - species, - "genotypes/select-population.html", - request.args.get("population_id") or "", - "species.populations.genotypes.select_population", - "species.populations.genotypes.list_genotypes", - "genotypes", - "Invalid population selected!") - @genotypesbp.route( "/<int:species_id>/populations/<int:population_id>/genotypes", methods=["GET"]) @require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") +@with_population(species_redirect_uri="species.list_species", + redirect_uri="species.populations.list_species_populations") def list_genotypes(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """List genotype details for species and population.""" with database_connection(app.config["SQL_URI"]) as conn: @@ -92,34 +57,31 @@ def list_genotypes(species: dict, population: dict, **kwargs):# pylint: disable= @genotypesbp.route( - "/<int:species_id>/populations/<int:population_id>/genotypes/list-markers", + "/<int:species_id>/populations/<int:population_id>/genotypes/<int:dataset_id>/list-markers", methods=["GET"]) @require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") -def list_markers( - species: dict, - population: dict, - **kwargs -):# pylint: disable=[unused-argument] - """List a species' genetic markers.""" +@with_species(redirect_uri="species.populations.genotypes.list_genotypes") +def list_markers(species: dict, **_kwargs): + """List the markers that exist for this species.""" + args = request.args + offset = int(args.get("start") or 0) with database_connection(app.config["SQL_URI"]) as conn: - start_from = max(safe_int(request.args.get("start_from") or 0), 0) - count = safe_int(request.args.get("count") or 20) - return render_template("genotypes/list-markers.html", - species=species, - population=population, - total_markers=genotype_markers_count( - conn, species["SpeciesId"]), - start_from=start_from, - count=count, - markers=enumerate_sequence( - genotype_markers(conn, - species["SpeciesId"], - offset=start_from, - limit=count), - start=start_from+1), - activelink="list-markers") + markers, total_records = genotype_markers( + conn, + species["SpeciesId"], + offset=offset, + limit=int(args.get("length") or 0)) + return jsonify({ + **({"draw": int(args.get("draw"))} + if bool(args.get("draw") or False) + else {}), + "recordsTotal": total_records, + "recordsFiltered": len(markers), + "markers": tuple({**marker, "index": idx} + for idx, marker in + enumerate(markers, start=offset+1)) + }) + @genotypesbp.route( "/<int:species_id>/populations/<int:population_id>/genotypes/datasets/" @@ -132,14 +94,14 @@ def view_dataset(species_id: int, population_id: int, dataset_id: int): species = species_by_id(conn, species_id) if not bool(species): flash("Invalid species provided!", "alert-danger") - return redirect(url_for("species.populations.genotypes.index")) + return redirect(url_for("species.list_species")) population = population_by_species_and_id( conn, species_id, population_id) if not bool(population): flash("Invalid population selected!", "alert-danger") return redirect(url_for( - "species.populations.genotypes.select_population", + "species.populations.list_species_populations", species_id=species_id)) dataset = genotype_dataset(conn, species_id, population_id, dataset_id) @@ -162,25 +124,32 @@ def view_dataset(species_id: int, population_id: int, dataset_id: int): "create", methods=["GET", "POST"]) @require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") +@with_population(species_redirect_uri="species.list_species", + redirect_uri="species.populations.list_species_populations") def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """Create a genotype dataset.""" + if request.method == "GET": + return render_template("genotypes/create-dataset.html", + species=species, + population=population, + activelink="create-dataset") + with (database_connection(app.config["SQL_URI"]) as conn, conn.cursor(cursorclass=DictCursor) as cursor): - if request.method == "GET": - return render_template("genotypes/create-dataset.html", - species=species, - population=population, - activelink="create-dataset") - - form = request.form - new_dataset = save_new_dataset( - cursor, - population["Id"], - form["geno-dataset-name"], - form["geno-dataset-fullname"], - form["geno-dataset-shortname"]) + + def __save_dataset__() -> Either: + form = request.form + try: + return Right(save_new_dataset( + cursor, + population["Id"], + form["geno-dataset-name"], + form["geno-dataset-fullname"], + form["geno-dataset-shortname"])) + except Exception: + msg = "Error adding new Genotype dataset to database." + logger.error(msg, exc_info=True) + return Left(Exception(msg)) def __success__(_success): flash("Successfully created genotype dataset.", "alert-success") @@ -189,18 +158,20 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable= species_id=species["SpeciesId"], population_id=population["Id"])) - return oauth2_post( - "auth/resource/genotypes/create", - json={ - **dict(request.form), - "species_id": species["SpeciesId"], - "population_id": population["Id"], - "dataset_id": new_dataset["Id"], - "dataset_name": form["geno-dataset-name"], - "dataset_fullname": form["geno-dataset-fullname"], - "dataset_shortname": form["geno-dataset-shortname"], - "public": "on" - } + return __save_dataset__().then( + lambda new_dataset: oauth2_post( + "auth/resource/genotypes/create", + json={ + **dict(request.form), + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": new_dataset["Id"], + "dataset_name": new_dataset["Name"], + "dataset_fullname": new_dataset["FullName"], + "dataset_shortname": new_dataset["ShortName"], + "public": "on" + } + ) ).either( make_either_error_handler( "There was an error creating the genotype dataset."), diff --git a/uploader/oauth2/client.py b/uploader/oauth2/client.py index 4e81afd..e37816d 100644 --- a/uploader/oauth2/client.py +++ b/uploader/oauth2/client.py @@ -4,7 +4,7 @@ import time import uuid import random from datetime import datetime, timedelta -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin, urlparse, urlencode import requests from flask import request, current_app as app @@ -18,6 +18,7 @@ from authlib.integrations.requests_client import OAuth2Session from uploader import session import uploader.monadic_requests as mrequests +from uploader.flask_extensions import fetch_flags SCOPE = ("profile group role resource register-client user masquerade " "introspect migrate-data") @@ -176,11 +177,13 @@ def authserver_authorise_uri(): """Build up the authorisation URI.""" req_baseurl = urlparse(request.base_url, scheme=request.scheme) host_uri = f"{req_baseurl.scheme}://{req_baseurl.netloc}/" - return urljoin( - authserver_uri(), - "auth/authorise?response_type=code" - f"&client_id={oauth2_clientid()}" - f"&redirect_uri={urljoin(host_uri, 'oauth2/code')}") + args = { + "response_type": "code", + "client_id": oauth2_clientid(), + "redirect_uri": ( + f"{urljoin(host_uri, 'oauth2/code')}?{urlencode(fetch_flags())}") + } + return f"{urljoin(authserver_uri(), 'auth/authorise')}?{urlencode(args)}" def __no_token__(_err) -> Left: diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py index b9841aa..3d656d2 100644 --- a/uploader/phenotypes/models.py +++ b/uploader/phenotypes/models.py @@ -87,24 +87,43 @@ def phenotype_publication_data(conn, phenotype_id) -> Optional[dict]: return dict(res) -def dataset_phenotypes(conn: Connection, - population_id: int, - dataset_id: int, - offset: int = 0, - limit: Optional[int] = None) -> tuple[dict, ...]: +def dataset_phenotypes(# pylint: disable=[too-many-arguments, too-many-positional-arguments] + conn: Connection, + population_id: int, + dataset_id: int, + offset: int = 0, + limit: Optional[int] = None, + xref_ids: tuple[int, ...] = tuple() +) -> tuple[dict, ...]: """Fetch the actual phenotypes.""" - _query = ( - "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, ist.InbredSetCode " + _narrow_by_ids = ( + f" AND pxr.Id IN ({', '.join(['%s'] * len(xref_ids))})" + if len(xref_ids) > 0 else "") + _narrow_by_limit = ( + f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + _pub_query = ( + "SELECT pub.* " + "FROM PublishXRef AS pxr " + "INNER JOIN Publication AS pub ON pxr.PublicationId=pub.Id " + "WHERE pxr.InbredSetId=%s") + _narrow_by_ids + _pheno_query = (( + "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, pxr.PublicationId, " + "ist.InbredSetCode " "FROM Phenotype AS pheno " "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " "INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId " "INNER JOIN InbredSet AS ist ON pf.InbredSetId=ist.Id " - "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + ( - f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + + _narrow_by_ids + + _narrow_by_limit) with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute(_query, (population_id, dataset_id)) + cursor.execute(_pub_query, (population_id,) + xref_ids) debug_query(cursor, logger) - return tuple(dict(row) for row in cursor.fetchall()) + _pubs = {row["Id"]: dict(row) for row in cursor.fetchall()} + cursor.execute(_pheno_query, (population_id, dataset_id) + xref_ids) + debug_query(cursor, logger) + return tuple({**dict(row), "publication": _pubs[row["PublicationId"]]} + for row in cursor.fetchall()) def __phenotype_se__(cursor: BaseCursor, xref_id, dataids_and_strainids): diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py index 2cf0ca0..c03f3f5 100644 --- a/uploader/phenotypes/views.py +++ b/uploader/phenotypes/views.py @@ -1,4 +1,6 @@ """Views handling ('classical') phenotypes."""# pylint: disable=[too-many-lines] +import io +import csv import sys import uuid import json @@ -21,12 +23,14 @@ from gn_libs import jobs as gnlibs_jobs from gn_libs.jobs.jobs import JobNotFound from gn_libs.mysqldb import database_connection +from werkzeug.datastructures import Headers from flask import (flash, request, jsonify, redirect, Blueprint, - current_app as app) + current_app as app, + Response as FlaskResponse) from r_qtl import r_qtl2_qc as rqc from r_qtl import exceptions as rqe @@ -313,6 +317,11 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable= dataset_shortname = ( form["dataset-shortname"] or form["dataset-name"]).strip() _pheno_dataset = save_new_dataset( + # It's not necessary to update the authorisation server to register + # new phenotype resource here, since each phenotype trait can, in + # theory, have its own access control allowing/disallowing access to + # it. In practice, however, we tend to gather multiple traits into a + # single resource for access control. cursor, population["Id"], form["dataset-name"].strip(), @@ -522,6 +531,65 @@ def job_status( @phenotypesbp.route( "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/job/<uuid:job_id>/download-errors", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def download_errors( + species: dict, + population: dict, + dataset: dict, + job_id: uuid.UUID, + **kwargs):# pylint: disable=[unused-argument] + """Download the list of errors as a CSV file.""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id)) + _prefix_ = jobs.jobsnamespace() + _jobid_ = job['jobid'] + def __generate_chunks__(): + _errors_ = ( + json.loads(error) + for key in rconn.keys( + f"{_prefix_}:{str(_jobid_)}:*:errors:*") + for error in rconn.lrange(key, 0, -1)) + _chunk_no_ = 0 + _all_errors_printed_ = False + while not _all_errors_printed_: + _chunk_ = [] + try: + for _ in range(0, 1000): + _chunk_.append(next(_errors_)) + except StopIteration: + _all_errors_printed_ = True + if len(_chunk_) <= 0: + raise + + _out_ = io.StringIO() + _writer_ = csv.DictWriter(_out_, fieldnames=tuple(_chunk_[0].keys())) + if _chunk_no_ == 0: + _writer_.writeheader() + _writer_.writerows(_chunk_) + _chunk_no_ += 1 + yield _out_.getvalue() + if _all_errors_printed_: + return + + headers = Headers() + headers.set("Content-Disposition", + "attachment", + filename=f"{job['job-type']}_{job['jobid']}.csv") + return FlaskResponse( + __generate_chunks__(), mimetype="text/csv", headers=headers) + except jobs.JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" "/<int:dataset_id>/job/<uuid:job_id>/review", methods=["GET"]) @require_login @@ -599,6 +667,8 @@ def review_job_data( conn, int(_job_metadata["publicationid"])) if _job_metadata.get("publicationid") else None), + user=session.user_details(), + timestamp=datetime.datetime.now().isoformat(), activelink="add-phenotypes") @@ -612,6 +682,12 @@ def load_phenotypes_success_handler(job): job_id=job["job_id"])) +def proceed_to_job_status(job): + """A generic 'job success' handler for asynchronous phenotype jobs.""" + app.logger.debug("The new job: %s", job) + return redirect(url_for("background-jobs.job_status", job_id=job["job_id"])) + + @phenotypesbp.route( "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" "/<int:dataset_id>/load-data-to-database", @@ -654,11 +730,6 @@ def load_data_to_database( def __handle_error__(resp): return render_template("http-error.html", *resp.json()) - def __handle_success__(load_job): - app.logger.debug("The phenotypes loading job: %s", load_job) - return redirect(url_for( - "background-jobs.job_status", job_id=load_job["job_id"])) - return request_token( token_uri=urljoin(oauth2client.authserver_uri(), "auth/token"), @@ -677,9 +748,15 @@ def load_data_to_database( "publication_id": _meta["publicationid"], "authserver": oauth2client.authserver_uri(), "token": token["access_token"], + "dataname": request.form["data_name"].strip(), "success_handler": ( "uploader.phenotypes.views" - ".load_phenotypes_success_handler") + ".load_phenotypes_success_handler"), + **{ + key: request.form[key] + for key in ("data_description",) + if key in request.form.keys() + } }, external_id=session.logged_in_user_id()) ).then( @@ -689,7 +766,7 @@ def load_data_to_database( Path(f"{uploads_dir(app)}/job_errors"), worker_manager="gn_libs.jobs.launcher", loglevel=_loglevel) - ).either(__handle_error__, __handle_success__) + ).either(__handle_error__, proceed_to_job_status) def update_phenotype_metadata(conn, metadata: dict): @@ -1158,6 +1235,12 @@ def rerun_qtlreaper_success_handler(job): return return_to_dataset_view_handler(job, "QTLReaper ran successfully!") +def delete_phenotypes_success_handler(job): + """Handle success running the 'delete-phenotypes' script.""" + return return_to_dataset_view_handler( + job, "Phenotypes deleted successfully.") + + @phenotypesbp.route( "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" "/<int:dataset_id>/delete", @@ -1167,14 +1250,29 @@ def rerun_qtlreaper_success_handler(job): species_redirect_uri="species.populations.phenotypes.index", population_redirect_uri="species.populations.phenotypes.select_population", redirect_uri="species.populations.phenotypes.list_datasets") -def delete_phenotypes(# pylint: disable=[unused-argument] +def delete_phenotypes(# pylint: disable=[unused-argument, too-many-locals] species: dict, population: dict, dataset: dict, **kwargs ): """Delete the specified phenotype data.""" - with database_connection(app.config["SQL_URI"]) as conn: + _dataset_page = redirect(url_for( + "species.populations.phenotypes.view_dataset", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"])) + + def __handle_error__(resp): + flash( + "Error retrieving authorisation token. Phenotype deletion " + "failed. Please try again later.", + "alert alert-danger") + return _dataset_page + + _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + with (database_connection(app.config["SQL_URI"]) as conn, + sqlite3.connection(_jobs_db) as jobsconn): form = request.form xref_ids = tuple(int(item) for item in set(form.getlist("xref_ids"))) @@ -1186,16 +1284,68 @@ def delete_phenotypes(# pylint: disable=[unused-argument] population_id=population["Id"], dataset_id=dataset["Id"])) case "delete": - # delete everything - # python3 -m scripts.phenotypes.delete_phenotypes <mariadburi> <authdburi> <speciesid> <populationid> - # - # delete selected phenotypes - # python3 -m scripts.phenotypes.delete_phenotypes <mariadburi> <authdburi> <speciesid> <populationid> --xref-ids-file=/path/to/file.txt - return "Would actually delete the data!" + _loglevel = logging.getLevelName( + app.logger.getEffectiveLevel()).lower() + if form.get("confirm_delete_all_phenotypes", "") == "on": + _cmd = ["--delete-all"] + else: + # setup phenotypes xref_ids file + _xref_ids_file = Path( + app.config["SCRATCH_DIRECTORY"], + f"delete-phenotypes-{uuid.uuid4()}.txt") + with _xref_ids_file.open(mode="w", encoding="utf8") as ptr: + ptr.write("\n".join(str(_id) for _id in xref_ids)) + + _cmd = ["--xref_ids_file", str(_xref_ids_file)] + + _job_id = uuid.uuid4() + return request_token( + token_uri=urljoin( + oauth2client.authserver_uri(), "auth/token"), + user_id=session.user_details()["user_id"] + ).then( + lambda token: gnlibs_jobs.initialise_job( + jobsconn, + _job_id, + [ + sys.executable, + "-u", + "-m", + "scripts.phenotypes.delete_phenotypes", + "--log-level", _loglevel, + app.config["SQL_URI"], + str(species["SpeciesId"]), + str(population["Id"]), + str(dataset["Id"]), + app.config["AUTH_SERVER_URL"], + token["access_token"]] + _cmd, + "delete-phenotypes", + extra_meta={ + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": dataset["Id"], + "success_handler": ( + "uploader.phenotypes.views." + "delete_phenotypes_success_handler") + }, + external_id=session.logged_in_user_id()) + ).then( + lambda _job: gnlibs_jobs.launch_job( + _job, + _jobs_db, + Path(f"{uploads_dir(app)}/job_errors"), + worker_manager="gn_libs.jobs.launcher", + loglevel=_loglevel) + ).either(__handle_error__, proceed_to_job_status) case _: + _phenos: tuple[dict, ...] = tuple() + if len(xref_ids) > 0: + _phenos = dataset_phenotypes( + conn, population["Id"], dataset["Id"], xref_ids=xref_ids) + return render_template( "phenotypes/confirm-delete-phenotypes.html", species=species, population=population, dataset=dataset, - phenotypes=xref_ids) + phenotypes=_phenos) diff --git a/uploader/population/rqtl2.py b/uploader/population/rqtl2.py index 97d4854..bb5066e 100644 --- a/uploader/population/rqtl2.py +++ b/uploader/population/rqtl2.py @@ -134,7 +134,7 @@ def upload_rqtl2_bundle(species_id: int, population_id: int): try: app.logger.debug("Files in the form: %s", request.files) the_file = save_file(request.files["rqtl2_bundle_file"], - Path(app.config["UPLOAD_FOLDER"])) + Path(app.config["UPLOADS_DIRECTORY"])) except AssertionError: app.logger.debug(traceback.format_exc()) flash("Please provide a valid R/qtl2 zip bundle.", @@ -185,7 +185,7 @@ def trigger_rqtl2_bundle_qc( "rqtl2-bundle-file": str(rqtl2bundle.absolute()), "original-filename": originalfilename})}), redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") + f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return jobid @@ -895,7 +895,7 @@ def confirm_bundle_details(species_id: int, population_id: int): }) }), redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") + f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return redirect(url_for("expression-data.rqtl2.rqtl2_processing_status", jobid=jobid)) diff --git a/uploader/publications/models.py b/uploader/publications/models.py index dcfa02b..d913144 100644 --- a/uploader/publications/models.py +++ b/uploader/publications/models.py @@ -101,6 +101,20 @@ def fetch_publication_by_id(conn: Connection, publication_id: int) -> dict: return dict(_res) if _res else {} +def fetch_publications_by_ids( + conn: Connection, publications_ids: tuple[int, ...] +) -> tuple[dict, ...]: + """Fetch publications with the given IDs.""" + if len(publications_ids) == 0: + return tuple() + + with conn.cursor(cursorclass=DictCursor) as cursor: + paramstr = ", ".join(["%s"] * len(publications_ids)) + cursor.execute(f"SELECT * FROM Publication WHERE Id IN ({paramstr})", + tuple(publications_ids)) + return tuple(dict(row) for row in cursor.fetchall()) + + def fetch_publication_phenotypes( conn: Connection, publication_id: int) -> Iterable[dict]: """Fetch all phenotypes linked to this publication.""" diff --git a/uploader/publications/views.py b/uploader/publications/views.py index d9eb294..89e9f5d 100644 --- a/uploader/publications/views.py +++ b/uploader/publications/views.py @@ -1,5 +1,6 @@ """Endpoints for publications""" import json +import datetime from gn_libs.mysqldb import database_connection from flask import ( @@ -89,9 +90,12 @@ def create_publication(): } if request.method == "GET": + now = datetime.datetime.now() return render_template( "publications/create-publication.html", - get_args=_get_args) + get_args=_get_args, + current_year=now.year, + current_month=now.strftime("%B")) form = request.form authors = form.get("publication-authors").encode("utf8") if authors is None or authors == "": diff --git a/uploader/request_checks.py b/uploader/request_checks.py index f1d8027..84935f9 100644 --- a/uploader/request_checks.py +++ b/uploader/request_checks.py @@ -2,14 +2,20 @@ These are useful for reusability, and hence maintainability of the code. """ +import logging + +from typing import Callable from functools import wraps -from gn_libs.mysqldb import database_connection +from gn_libs.mysqldb import Connection, database_connection from flask import flash, url_for, redirect, current_app as app from uploader.species.models import species_by_id from uploader.population.models import population_by_species_and_id +logger = logging.getLogger(__name__) + + def with_species(redirect_uri: str): """Ensure the species actually exists.""" def __decorator__(function): @@ -28,7 +34,7 @@ def with_species(redirect_uri: str): "alert-danger") return redirect(url_for(redirect_uri)) except ValueError as _verr: - app.logger.debug( + logger.debug( "Exception converting value to integer: %s", kwargs.get("species_id"), exc_info=True) @@ -63,7 +69,7 @@ def with_population(species_redirect_uri: str, redirect_uri: str): "alert-danger") return select_population_uri except ValueError as _verr: - app.logger.debug( + logger.debug( "Exception converting value to integer: %s", kwargs.get("population_id"), exc_info=True) @@ -73,3 +79,45 @@ def with_population(species_redirect_uri: str, redirect_uri: str): return function(**{**kwargs, "population": population}) return __with_population__ return __decorator__ + + +def with_dataset( + species_redirect_uri: str, + population_redirect_uri: str, + redirect_uri: str, + dataset_by_id: Callable[ + [Connection, int, int, int], + dict] +): + """Ensure the dataset actually exists.""" + def __decorator__(func): + @wraps(func) + @with_population(species_redirect_uri, population_redirect_uri) + def __with_dataset__(**kwargs): + try: + _spcid = int(kwargs["species_id"]) + _popid = int(kwargs["population_id"]) + _dsetid = int(kwargs.get("dataset_id")) + select_dataset_uri = redirect(url_for( + redirect_uri, species_id=_spcid, population_id=_popid)) + if not bool(_dsetid): + flash("You need to select a valid 'dataset_id' value.", + "alert-danger") + return select_dataset_uri + with database_connection(app.config["SQL_URI"]) as conn: + dataset = dataset_by_id(conn, _spcid, _popid, _dsetid) + if not bool(dataset): + flash("You must select a valid dataset.", + "alert-danger") + return select_dataset_uri + except ValueError as _verr: + logger.debug( + "Exception converting 'dataset_id' to integer: %s", + kwargs.get("dataset_id"), + exc_info=True) + flash("Expected 'dataset_id' value to be an integer." + "alert-danger") + return select_dataset_uri + return func(**{**kwargs, "dataset": dataset}) + return __with_dataset__ + return __decorator__ diff --git a/uploader/samples/views.py b/uploader/samples/views.py index ee002ba..2a09f8e 100644 --- a/uploader/samples/views.py +++ b/uploader/samples/views.py @@ -138,7 +138,7 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma try: samples_file = save_file(request.files["samples_file"], - Path(app.config["UPLOAD_FOLDER"])) + Path(app.config["UPLOADS_DIRECTORY"])) except AssertionError: flash("You need to provide a file with the samples data.", "alert-error") @@ -172,12 +172,33 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma ] + (["--firstlineheading"] if firstlineheading else []), "samples_upload", extra_meta={ - "job_name": f"Samples Upload: {samples_file.name}" + "job_name": f"Samples Upload: {samples_file.name}", + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "success_handler": ( + "uploader.samples.views.samples_upload_success_handler") }, external_id=session.logged_in_user_id()), _jobs_db, - Path(f"{app.config['UPLOAD_FOLDER']}/job_errors").absolute(), + Path(f"{app.config['UPLOADS_DIRECTORY']}/job_errors").absolute(), loglevel=logging.getLevelName( app.logger.getEffectiveLevel()).lower()) return redirect( url_for("background-jobs.job_status", job_id=job["job_id"])) + + +def samples_upload_success_handler(job): + """Handler for background jobs: Successful upload of samples""" + return return_to_samples_list_view_handler( + job, "Samples uploaded successfully.") + + +def return_to_samples_list_view_handler(job, msg): + """Handler for background jobs: Return to list_samples page.""" + flash(msg, "alert alert-success") + return redirect(url_for( + "species.populations.samples." + "list_samples", + species_id=job["metadata"]["species_id"], + population_id=job["metadata"]["population_id"], + job_id=job["job_id"])) diff --git a/uploader/static/css/layout-common.css b/uploader/static/css/layout-common.css index 88e580c..9c9d034 100644 --- a/uploader/static/css/layout-common.css +++ b/uploader/static/css/layout-common.css @@ -2,20 +2,20 @@ box-sizing: border-box; } - body { - display: grid; - grid-gap: 1em; - } +body { + display: grid; + grid-gap: 1em; +} - #header { - margin: -0.7em; /* Fill entire length of screen */ - /* Define layout for the children elements */ - display: grid; - } +#header { + margin: -0.7em; /* Fill entire length of screen */ + /* Define layout for the children elements */ + display: grid; +} - #header #header-nav { - /* Place it in the parent element */ - grid-column-start: 1; - grid-column-end: 2; - display: flex; - } +#header #header-nav { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; + display: flex; +} diff --git a/uploader/static/css/theme.css b/uploader/static/css/theme.css index 45e5d3d..6f5cb0c 100644 --- a/uploader/static/css/theme.css +++ b/uploader/static/css/theme.css @@ -81,10 +81,10 @@ table.dataTable tbody tr.selected td { background-color: #ffee99 !important; } -.form-group { +#frm-add-phenotypes .form-group { margin-bottom: 2em; padding-bottom: 0.2em; - border-bottom: solid gray 1px; + border-bottom: solid #A9A9A9 1px; } @@ -95,3 +95,8 @@ table.dataTable tbody tr.selected td { .breadcrumb-item a { text-decoration: none; } + +.table thead tr th { + text-align: center; + vertical-align: middle; +} diff --git a/uploader/static/images/frontpage_banner.png b/uploader/static/images/frontpage_banner.png new file mode 100644 index 0000000..d25e1c9 --- /dev/null +++ b/uploader/static/images/frontpage_banner.png Binary files differdiff --git a/uploader/templates/background-jobs/job-status.html b/uploader/templates/background-jobs/job-status.html index 50cf6e5..2e75c6d 100644 --- a/uploader/templates/background-jobs/job-status.html +++ b/uploader/templates/background-jobs/job-status.html @@ -30,12 +30,16 @@ <div class="row"> <h3 class="subheading">STDOUT</h3> - <pre>{{job["stdout"]}}</pre> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stdout"]}}</pre> + </div> </div> <div class="row"> <h3 class="subheading">STDERR</h3> - <pre>{{job["stderr"]}}</pre> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stderr"]}}</pre> + </div> </div> {%endblock%} diff --git a/uploader/templates/background-jobs/job-summary.html b/uploader/templates/background-jobs/job-summary.html index c2c2d6b..ef9ef6c 100644 --- a/uploader/templates/background-jobs/job-summary.html +++ b/uploader/templates/background-jobs/job-summary.html @@ -50,12 +50,16 @@ <div class="row"> <h3 class="subheading">Script Errors and Logging</h3> - <pre>{{job["stderr"]}}</pre> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stderr"]}}</pre> + </div> </div> <div class="row"> <h3 class="subheading">Script Output</h3> - <pre>{{job["stdout"]}}</pre> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stdout"]}}</pre> + </div> </div> {%endblock%} diff --git a/uploader/templates/genotypes/create-dataset.html b/uploader/templates/genotypes/create-dataset.html index 10331c1..ff174fb 100644 --- a/uploader/templates/genotypes/create-dataset.html +++ b/uploader/templates/genotypes/create-dataset.html @@ -35,13 +35,15 @@ id="txt-geno-dataset-name" name="geno-dataset-name" required="required" - class="form-control" /> + class="form-control" + value="{{population.Name}}Geno" + readonly="readonly" /> <small class="form-text text-muted"> <p>This is a short representative, but constrained name for the genotype dataset.<br /> - The field will only accept letters ('A-Za-z'), numbers (0-9), hyphens - and underscores. Any other character will cause the name to be - rejected.</p></small> + It is used internally by the Genenetwork system. Do not change this + value.</p> + </small> </div> <div class="form-group"> @@ -50,7 +52,8 @@ id="txt-geno-dataset-fullname" name="geno-dataset-fullname" required="required" - class="form-control" /> + class="form-control" + value="{{population.Name}} Genotypes" /> <small class="form-text text-muted"> <p>This is a longer, more descriptive name for your dataset.</p></small> </div> @@ -61,7 +64,8 @@ <input type="text" id="txt-geno-dataset-shortname" name="geno-dataset-shortname" - class="form-control" /> + class="form-control" + value="{{population.Name}}Geno" /> <small class="form-text text-muted"> <p>A short name for your dataset. If you leave this field blank, the short name will be set to the same value as the diff --git a/uploader/templates/genotypes/index.html b/uploader/templates/genotypes/index.html deleted file mode 100644 index b50ebc5..0000000 --- a/uploader/templates/genotypes/index.html +++ /dev/null @@ -1,32 +0,0 @@ -{%extends "genotypes/base.html"%} -{%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-select-species.html" import select_species_form%} - -{%block title%}Genotypes{%endblock%} - -{%block pagetitle%}Genotypes{%endblock%} - - -{%block contents%} -{{flash_all_messages()}} - -<div class="row"> - <p> - This section allows you to upload genotype information for your experiments, - in the case that you have not previously done so. - </p> - <p> - We'll need to link the genotypes to the species and population, so do please - go ahead and select those in the next two steps. - </p> -</div> - -<div class="row"> - {{select_species_form(url_for("species.populations.genotypes.index"), - species)}} -</div> -{%endblock%} - -{%block javascript%} -<script type="text/javascript" src="/static/js/species.js"></script> -{%endblock%} diff --git a/uploader/templates/genotypes/list-genotypes.html b/uploader/templates/genotypes/list-genotypes.html index a2b98c8..131576f 100644 --- a/uploader/templates/genotypes/list-genotypes.html +++ b/uploader/templates/genotypes/list-genotypes.html @@ -9,19 +9,6 @@ {{flash_all_messages()}} <div class="row"> - <h2>Genetic Markers</h2> - <p>There are a total of {{total_markers}} currently registered genetic markers - for the "{{species.FullName}}" species. You can click - <a href="{{url_for('species.populations.genotypes.list_markers', - species_id=species.SpeciesId, - population_id=population.Id)}}" - title="View genetic markers for species '{{species.FullName}}"> - this link to view the genetic markers - </a>. - </p> -</div> - -<div class="row"> <h2>Genotype Encoding</h2> <p> The genotype encoding used for the "{{population.FullName}}" population from @@ -56,59 +43,101 @@ </table> {%if genocode | length < 1%} - <a href="#add-genotype-encoding" - title="Add a genotype encoding system for this population" - class="btn btn-primary not-implemented"> - add genotype encoding + <div class="col"> + <a href="#add-genotype-encoding" + title="Add a genotype encoding system for this population" + class="btn btn-primary not-implemented"> + define genotype encoding </a> + </div> {%endif%} </div> -<div class="row text-danger"> - <h3>Some Important Concepts to Consider/Remember</h3> - <ul> - <li>Reference vs. Non-reference alleles</li> - <li>In <em>GenoCode</em> table, items are ordered by <strong>InbredSet</strong></li> - </ul> - <h3>Possible references</h3> - <ul> - <li>https://mr-dictionary.mrcieu.ac.uk/term/genotype/</li> - <li>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7363099/</li> - </ul> +<div class="row"> + <h2>Genotype Dataset</h2> +</div> + +{%if dataset is not none%} + +<div class="row"> + <h3>Dataset Details</h3> + <table class="table"> + <thead> + <tr> + <th>Name</th> + <th>Full Name</th> + </tr> + </thead> + + <tbody> + <tr> + <td>{{dataset.Name}}</td> + <td><a href="{{url_for('species.populations.genotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + title="View details regarding and manage dataset '{{dataset.FullName}}'" + target="_blank"> + {{dataset.FullName}}</a></td> + </tr> + </tbody> + </table> + + <p> + To see more information regarding this dataset (e.g. which markers have + sample allele data, the allele data itself, etc) click on the "Full Name" + link above.</p> +</div> + +<div class="row"> + <h3>Genotype Markers</h3> + + <div class="row"> + <p> + The table below lists all of the markers that exist for species + {{species.SpeciesName}} ({{species.FullName}}), regardless of whether + (or not) we have corresponding sample allele data for a particular marker. + </p> + <table id="tbl-genetic-markers" class="table compact stripe cell-border"> + <thead> + <tr> + <th title="">#</th> + <th title="">Index</th> + <th title="">Marker Name</th> + <th title="Chromosome">Chr</th> + <th title="Physical location of the marker in megabasepairs"> + Location (Mb)</th> + <th title="">Source</th> + <th title="">Source2</th> + </thead> + + <tbody> + {%for marker in markers%} + <tr> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + </tr> + {%endfor%} + </tbody> + </table> </div> +{%else%} + <div class="row"> - <h2>Genotype Datasets</h2> - - <p>The genotype data is organised under various genotype datasets. You can - click on the link for the relevant dataset to view a little more information - about it.</p> - - {%if dataset is not none%} - <table class="table"> - <thead> - <tr> - <th>Name</th> - <th>Full Name</th> - </tr> - </thead> - - <tbody> - <tr> - <td>{{dataset.Name}}</td> - <td><a href="{{url_for('species.populations.genotypes.view_dataset', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id)}}" - title="View details regarding and manage dataset '{{dataset.FullName}}'"> - {{dataset.FullName}}</a></td> - </tr> - </tbody> - </table> - {%else%} + <p> + Your genotype data will need to be under a dataset. Unfortunately there is + currently no dataset defined for this population. + </p> + <p class="text-warning"> <span class="glyphicon glyphicon-exclamation-sign"></span> - There is no genotype dataset defined for this population. + Click the button below to define the genotype dataset for this population. </p> <p> <a href="{{url_for('species.populations.genotypes.create_dataset', @@ -117,16 +146,81 @@ title="Create a new genotype dataset for the '{{population.FullName}}' population for the '{{species.FullName}}' species." class="btn btn-primary"> create new genotype dataset</a></p> - {%endif%} </div> -<div class="row text-warning"> - <p> - <span class="glyphicon glyphicon-exclamation-sign"></span> - <strong>NOTE</strong>: Currently the GN2 (and related) system(s) expect a - single genotype dataset. If there is more than one, the system apparently - fails in unpredictable ways. - </p> - <p>Fix this to allow multiple datasets, each with a different assembly from - all the rest.</p> + +{%endif%} + +<div class="row"> + <h2>Notes</h2> + <div class="row text-danger"> + <h3>Genetic Markers: Some Important Concepts to Consider/Remember</h3> + <ul> + <li>Reference vs. Non-reference alleles</li> + <li>In <em>GenoCode</em> table, items are ordered by <strong>InbredSet</strong></li> + </ul> + <h3>Possible references</h3> + <ul> + <li>https://mr-dictionary.mrcieu.ac.uk/term/genotype/</li> + <li>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7363099/</li> + </ul> + </div> + + <div class="row text-warning"> + <h3>Genotype Dataset</h3> + <p> + <span class="glyphicon glyphicon-exclamation-sign"></span> + <strong>NOTE</strong>: Currently the GN2 (and related) system(s) expect a + single genotype dataset per population. If there is more than one, the + system apparently fails in unpredictable ways. + </p> + </div> </div> + +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + + $(function() { + var dtGeneticMarkers = buildDataTable( + "#tbl-genetic-markers", + [], + [ + { + data: function(marker) { + return `<input type="checkbox" name="selected-markers" ` + + `id="chk-selected-markers-${marker.Id}-${marker.GenoFreezeId}" ` + + `value="${marker.Id}_${marker.GenoFreezeId}" ` + + `class="chk-row-select" />`; + } + }, + {data: 'index'}, + {data: "Name", searchable: true}, + {data: "Chr", searchable: true}, + {data: "Mb", searchable: true}, + {data: "Source", searchable: true}, + {data: "Source2", searchable: true} + ], + { + ajax: { + url: "{{url_for('species.populations.genotypes.list_markers', species_id=species.SpeciesId, population_id=population.Id, dataset_id=dataset.Id)}}", + dataSrc: "markers" + }, + paging: true, + scroller: true, + scrollY: "50vh", + scrollCollapse: true, + layout: { + top: "info", + topStart: null, + topEnd: null, + bottom: null, + bottomStart: null, + bottomEnd: null + } + }); + }); + +</script> {%endblock%} diff --git a/uploader/templates/genotypes/list-markers.html b/uploader/templates/genotypes/list-markers.html index 5f3dd6f..22189c7 100644 --- a/uploader/templates/genotypes/list-markers.html +++ b/uploader/templates/genotypes/list-markers.html @@ -57,7 +57,7 @@ <table class="table"> <thead> <tr> - <th title="">#</th> + <th title="">Index</th> <th title="">Marker Name</th> <th title="Chromosome">Chr</th> <th title="Physical location of the marker in megabasepairs"> diff --git a/uploader/templates/genotypes/select-population.html b/uploader/templates/genotypes/select-population.html deleted file mode 100644 index acdd063..0000000 --- a/uploader/templates/genotypes/select-population.html +++ /dev/null @@ -1,25 +0,0 @@ -{%extends "genotypes/base.html"%} -{%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-display-species-card.html" import display_species_card%} -{%from "populations/macro-select-population.html" import select_population_form%} - -{%block title%}Genotypes{%endblock%} - -{%block pagetitle%}Genotypes{%endblock%} - - -{%block contents%} -{{flash_all_messages()}} - -<div class="row"> - {{select_population_form(url_for("species.populations.genotypes.select_population", species_id=species.SpeciesId), species, populations)}} -</div> -{%endblock%} - -{%block sidebarcontents%} -{{display_species_card(species)}} -{%endblock%} - -{%block javascript%} -<script type="text/javascript" src="/static/js/populations.js"></script> -{%endblock%} diff --git a/uploader/templates/genotypes/view-dataset.html b/uploader/templates/genotypes/view-dataset.html index 1c4eccf..d95a8e3 100644 --- a/uploader/templates/genotypes/view-dataset.html +++ b/uploader/templates/genotypes/view-dataset.html @@ -46,8 +46,9 @@ <div class="row"> <h2>Genotype Data</h2> - <p class="text-danger"> - Provide link to enable uploading of genotype data here.</p> + <div class="col" style="margin-bottom: 3px;"> + <a href="#" class="btn btn-primary not-implemented">upload genotypes</a> + </div> </div> {%endblock%} diff --git a/uploader/templates/index.html b/uploader/templates/index.html index e426732..6e9c777 100644 --- a/uploader/templates/index.html +++ b/uploader/templates/index.html @@ -20,7 +20,7 @@ <div class="col"> <a href="{{url_for('species.create_species', return_to='base.index')}}" class="btn btn-outline-primary" - title="Create a new species.">Create a new Species</a> + title="Add a new species to Genenetwork.">add a new Species</a> </div> </div> {%endmacro%} @@ -106,8 +106,8 @@ id="publications-content" role="tabpanel" aria-labelledby="publications-content-tab"> - <p>View, edit and delete existing publications, and add new - publications by clicking on the button below.</p> + <p>You can view, edit, and delete existing publications, as well as add + new ones, by clicking the button below.</p> <a href="{{url_for('publications.index')}}" title="Manage publications." @@ -116,47 +116,55 @@ </div> </div> - {%else%} +{%else%} - <div class="row"> - <p>The Genenetwork Uploader (<em>gn-uploader</em>) enables upload of new data - into the Genenetwork System. It provides Quality Control over data, and - guidance in case you data does not meet the standards for acceptance.</p> - <p> +<div class="row"> + <img src="/static/images/frontpage_banner.png" + alt="Banner image showing the process flow a user would follow." /> +</div> + +<div class="row"> + <p>The GeneNetwork Uploader (gn-uploader) lets you easily add new data to the + GeneNetwork System. It automatically checks your data for quality and walks + you through fixing any issues before submission.</p> +</div> + +<div class="row"> + <div class="col"> <a href="{{authserver_authorise_uri()}}" title="Sign in to the system" class="btn btn-primary">Sign in</a> - to get started.</p> </div> - {%endif%} +</div> +{%endif%} - {%endblock%} +{%endblock%} - {%block sidebarcontents%} - {%if view_under_construction%} - <div class="row"> - <p>The data in Genenetwork is related to one species or another. Use the form - provided to select from existing species, or click on the - "Create a New Species" button if you cannot find the species you want to - work with.</p> - </div> - <div class="row"> - <form id="frm-quick-navigation"> - <legend>Quick Navigation</legend> - <div class="form-group"> - <label for="fqn-species-id">Species</label> - <select name="species_id"> - <option value="">Select species</option> - </select> - </div> - </form> - </div> - {%endif%} - {%endblock%} +{%block sidebarcontents%} +{%if view_under_construction%} +<div class="row"> + <p>The data in Genenetwork is related to one species or another. Use the form + provided to select from existing species, or click on the + "Create a New Species" button if you cannot find the species you want to + work with.</p> +</div> +<div class="row"> + <form id="frm-quick-navigation"> + <legend>Quick Navigation</legend> + <div class="form-group"> + <label for="fqn-species-id">Species</label> + <select name="species_id"> + <option value="">Select species</option> + </select> + </div> + </form> +</div> +{%endif%} +{%endblock%} - {%block javascript%} - <script type="text/javascript" src="/static/js/species.js"></script> - {%endblock%} +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/phenotypes/add-phenotypes-base.html b/uploader/templates/phenotypes/add-phenotypes-base.html index c74a0fa..3207129 100644 --- a/uploader/templates/phenotypes/add-phenotypes-base.html +++ b/uploader/templates/phenotypes/add-phenotypes-base.html @@ -43,7 +43,7 @@ <table id="tbl-select-publication" class="table compact stripe"> <thead> <tr> - <th>#</th> + <th>Index</th> <th>PubMed ID</th> <th>Title</th> <th>Authors</th> @@ -84,7 +84,8 @@ if(pub.PubMed_ID) { return `<a href="https://pubmed.ncbi.nlm.nih.gov/` + `${pub.PubMed_ID}/" target="_blank" ` + - `title="Link to publication on NCBI.">` + + `title="Link to publication on NCBI. This will ` + + `open in a new tab.">` + `${pub.PubMed_ID}</a>`; } return ""; @@ -97,10 +98,7 @@ if(pub.Title) { title = pub.Title } - return `<a href="/publications/view/${pub.Id}" ` + - `target="_blank" ` + - `title="Link to view publication details">` + - `${title}</a>`; + return title; } }, { diff --git a/uploader/templates/phenotypes/base.html b/uploader/templates/phenotypes/base.html index fe7ccd3..5959422 100644 --- a/uploader/templates/phenotypes/base.html +++ b/uploader/templates/phenotypes/base.html @@ -3,6 +3,7 @@ {%block breadcrumbs%} {{super()}} +{%if dataset%} <li class="breadcrumb-item"> <a href="{{url_for('species.populations.phenotypes.view_dataset', species_id=species['SpeciesId'], @@ -11,6 +12,7 @@ {{dataset["Name"]}} </a> </li> +{%endif%} {%endblock%} {%block contents%} diff --git a/uploader/templates/phenotypes/confirm-delete-phenotypes.html b/uploader/templates/phenotypes/confirm-delete-phenotypes.html index b59fd7b..3cf6e65 100644 --- a/uploader/templates/phenotypes/confirm-delete-phenotypes.html +++ b/uploader/templates/phenotypes/confirm-delete-phenotypes.html @@ -47,7 +47,7 @@ <table id="tbl-delete-phenotypes" class="table"> <thead> <tr> - <th>#</th> + <th>Index</th> <th>Record ID</th> <th>Description</th> </tr> @@ -56,13 +56,16 @@ {%for phenotype in phenotypes%} <tr> <td> - <input id="chk-xref-id-{{phenotype}}" + <input id="chk-xref-id-{{phenotype.xref_id}}" name="xref_ids" type="checkbox" + value="{{phenotype.xref_id}}" class="chk-row-select" /> </td> - <td>{{phenotype}}</td> - <td>{{phenotype}} — Description</td> + <td>{{phenotype.xref_id}}</td> + <td>{{phenotype.Post_publication_description or + phenotype.Pre_publication_description or + phenotype.original_description}}</td> </tr> {%endfor%} </tbody> @@ -166,6 +169,27 @@ $("#btn-deselect-all-phenotypes").on("click", function(event) { dt.deselectAll(); }); + + $("#btn-delete-phenotypes-selected").on("click", function(event) { + event.preventDefault(); + form = $("#frm-delete-phenotypes-selected"); + form.find(".dynamically-added-element").remove(); + dt.rows({selected: true}).nodes().each(function(node, index) { + var xref_id = $(node) + .find('input[type="checkbox"]:checked') + .val(); + var chk = $('<input type="checkbox">'); + chk.attr("class", "dynamically-added-element"); + chk.attr("value", xref_id); + chk.attr("name", "xref_ids"); + chk.attr("style", "display: none"); + chk.prop("checked", true); + form.append(chk); + }); + form.append( + $('<input type="hidden" name="action" value="delete" />')); + form.submit(); + }) }); </script> {%endblock%} diff --git a/uploader/templates/phenotypes/create-dataset.html b/uploader/templates/phenotypes/create-dataset.html index 19a2b34..9963953 100644 --- a/uploader/templates/phenotypes/create-dataset.html +++ b/uploader/templates/phenotypes/create-dataset.html @@ -48,7 +48,8 @@ {%else%} class="form-control" {%endif%} - required="required" /> + required="required" + readonly="readonly" /> <small class="form-text text-muted"> <p>A short representative name for the dataset.</p> <p>Recommended: Use the population name and append "Publish" at the end. @@ -66,7 +67,7 @@ <input id="txt-dataset-fullname" name="dataset-fullname" type="text" - value="{{original_formdata.get('dataset-fullname', '')}}" + value="{{original_formdata.get('dataset-fullname', '') or population.Name + ' Phenotypes'}}" {%if errors["dataset-fullname"] is defined%} class="form-control danger" {%else%} diff --git a/uploader/templates/phenotypes/edit-phenotype.html b/uploader/templates/phenotypes/edit-phenotype.html index 115d6af..1b3ee9d 100644 --- a/uploader/templates/phenotypes/edit-phenotype.html +++ b/uploader/templates/phenotypes/edit-phenotype.html @@ -142,7 +142,7 @@ <table class="table table-striped table-responsive table-form-table"> <thead style="position: sticky; top: 0;"> <tr> - <th>#</th> + <th>Index</th> <th>Sample</th> <th>Value</th> {%if population.Family in families_with_se_and_n%} diff --git a/uploader/templates/phenotypes/job-status.html b/uploader/templates/phenotypes/job-status.html index 0bbe8e0..951907f 100644 --- a/uploader/templates/phenotypes/job-status.html +++ b/uploader/templates/phenotypes/job-status.html @@ -52,10 +52,10 @@ <p> {%if errors | length == 0%} <a href="{{url_for('species.populations.phenotypes.review_job_data', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id, - job_id=job_id)}}" + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + job_id=job_id)}}" class="btn btn-primary" title="Continue to process data">Continue</a> {%else%} @@ -70,13 +70,28 @@ </div> <h3 class="subheading">upload errors</h3> +{%if errors | length == 0 %} <div class="row" style="max-height: 20em; overflow: scroll;"> - {%if errors | length == 0 %} <p class="text-info"> <span class="glyphicon glyphicon-info-sign"></span> No errors found so far </p> - {%else%} +</div> +{%else%} +{%if errors | length > 0%} +<div class="row"> + <div class="col"> + <a href="{{url_for('species.populations.phenotypes.download_errors', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + job_id=job_id)}}" + class="btn btn-info" + title="Download the errors as a CSV file.">download errors CSV</a> + </div> +</div> +{%endif%} +<div class="row" style="max-height: 20em; overflow: scroll;"> <table class="table table-responsive"> <thead style="position: sticky; top: 0; background: white;"> <tr> @@ -111,7 +126,8 @@ {%endfor%} </tbody> </table> - {%endif%} +</div> +{%endif%} </div> <div class="row"> diff --git a/uploader/templates/phenotypes/review-job-data.html b/uploader/templates/phenotypes/review-job-data.html index c8355b2..0e8f119 100644 --- a/uploader/templates/phenotypes/review-job-data.html +++ b/uploader/templates/phenotypes/review-job-data.html @@ -70,6 +70,9 @@ {%endif%} {%endfor%} </ul> +</div> + +<div class="row"> <form id="frm-review-phenotype-data" method="POST" @@ -78,10 +81,38 @@ population_id=population.Id, dataset_id=dataset.Id)}}"> <input type="hidden" name="data-qc-job-id" value="{{job.jobid}}" /> - <input type="submit" - value="continue" - class="btn btn-primary" /> + <div class="form-group"> + <label for="txt-data-name">data name</label> + <input type="text" + id="txt-data-name" + class="form-control" + name="data_name" + title="A short, descriptive name for this data." + placeholder="{{user.email}} - {{dataset.Name}} - {{timestamp}}" + value="{{user.email}} - {{dataset.Name}} - {{timestamp}}" + required="required"> + <span class="form-text text-muted"> + This is a short, descriptive name for the data. It is useful to humans, + enabling them identify what traits each data "resource" wraps around. + </span> + </div> + + {%if view_under_construction%} + <div class="form-group"> + <label for="txt-data-description">data description</label> + <textarea id="txt-data-description" + class="form-control" + name="data_description" + title="A longer description for this data." + rows="5"></textarea> + <span class="form-text text-muted"> + </span> + </div> + {%endif%} + + <button type="submit" class="btn btn-primary">continue</button> </form> + </div> {%else%} <div class="row"> diff --git a/uploader/templates/phenotypes/view-dataset.html b/uploader/templates/phenotypes/view-dataset.html index de76cbf..fc84757 100644 --- a/uploader/templates/phenotypes/view-dataset.html +++ b/uploader/templates/phenotypes/view-dataset.html @@ -77,7 +77,6 @@ </form> </div> - {%if view_under_construction%} <div class="col"> <form id="frm-delete-phenotypes" method="POST" @@ -93,7 +92,6 @@ value="delete phenotypes" /> </form> </div> - {%endif%} </div> <div class="row" style="margin-top: 0.5em;"> @@ -150,14 +148,36 @@ return `<a href="${url.toString()}" target="_blank">` + `${pheno.InbredSetCode}_${pheno.xref_id}` + `</a>`; - } + }, + title: "Record", + visible: true, + searchable: true }, { data: function(pheno) { return (pheno.Post_publication_description || pheno.Original_description || pheno.Pre_publication_description); - } + }, + title: "Description", + visible: true, + searchable: true + }, + { + data: function(pheno) { + return pheno.publication.Title; + }, + title: "Publication Title", + visible: false, + searchable: true + }, + { + data: function(pheno) { + return pheno.publication.Authors; + }, + title: "Authors", + visible: false, + searchable: true } ], { diff --git a/uploader/templates/phenotypes/view-phenotype.html b/uploader/templates/phenotypes/view-phenotype.html index a69b024..a59949e 100644 --- a/uploader/templates/phenotypes/view-phenotype.html +++ b/uploader/templates/phenotypes/view-phenotype.html @@ -103,7 +103,7 @@ or "group:resource:delete-resource" in privileges%} <table class="table"> <thead> <tr> - <th>#</th> + <th>Index</th> <th>Sample</th> <th>Value</th> {%if has_se%} diff --git a/uploader/templates/populations/view-population.html b/uploader/templates/populations/view-population.html index ac89bc7..29add29 100644 --- a/uploader/templates/populations/view-population.html +++ b/uploader/templates/populations/view-population.html @@ -95,14 +95,13 @@ id="genotypes-content" role="tabpanel" aria-labelledby="genotypes-content-tab"> - <p>This allows you to upload the data that concerns your genotypes.</p> - <p>Any samples/individuals/cases/strains that do not already exist in the - system will be added. This does not delete any existing data.</p> + <p>Click the button to view and manage genetic data for individuals in + this population.</p> <a href="{{url_for('species.populations.genotypes.list_genotypes', species_id=species.SpeciesId, population_id=population.Id)}}" title="Upload genotype information for the '{{population.FullName}}' population of the '{{species.FullName}}' species." - class="btn btn-primary">upload genotypes</a> + class="btn btn-primary">manage genotypes</a> </div> <div class="tab-pane fade" id="expression-data-content" role="tabpanel" aria-labelledby="expression-data-content-tab"> <p>Upload expression data (mRNA data) for this population.</p> diff --git a/uploader/templates/publications/create-publication.html b/uploader/templates/publications/create-publication.html index fb0127d..da5889e 100644 --- a/uploader/templates/publications/create-publication.html +++ b/uploader/templates/publications/create-publication.html @@ -91,22 +91,22 @@ class="col-sm-2 col-form-label"> Month</label> <div class="col-sm-4"> - <select class="form-control" + <select class="form-select" id="select-publication-month" name="publication-month"> <option value="">Select a month</option> - <option value="january">January</option> - <option value="february">February</option> - <option value="march">March</option> - <option value="april">April</option> - <option value="may">May</option> - <option value="june">June</option> - <option value="july">July</option> - <option value="august">August</option> - <option value="september">September</option> - <option value="october">October</option> - <option value="november">November</option> - <option value="december">December</option> + <option {%if current_month | lower == "january"%}selected="selected"{%endif%}value="january">January</option> + <option {%if current_month | lower == "february"%}selected="selected"{%endif%}value="february">February</option> + <option {%if current_month | lower == "march"%}selected="selected"{%endif%}value="march">March</option> + <option {%if current_month | lower == "april"%}selected="selected"{%endif%}value="april">April</option> + <option {%if current_month | lower == "may"%}selected="selected"{%endif%}value="may">May</option> + <option {%if current_month | lower == "june"%}selected="selected"{%endif%}value="june">June</option> + <option {%if current_month | lower == "july"%}selected="selected"{%endif%}value="july">July</option> + <option {%if current_month | lower == "august"%}selected="selected"{%endif%}value="august">August</option> + <option {%if current_month | lower == "september"%}selected="selected"{%endif%}value="september">September</option> + <option {%if current_month | lower == "october"%}selected="selected"{%endif%}value="october">October</option> + <option {%if current_month | lower == "november"%}selected="selected"{%endif%}value="november">November</option> + <option {%if current_month | lower == "december"%}selected="selected"{%endif%}value="december">December</option> </select> <span class="form-text text-muted">Month of publication</span> </div> @@ -119,7 +119,10 @@ id="txt-publication-year" name="publication-year" class="form-control" - min="1960" /> + min="1960" + max="{{current_year}}" + value="{{current_year or ''}}" + required="required" /> <span class="form-text text-muted">Year of publication</span> </div> </div> diff --git a/uploader/templates/publications/index.html b/uploader/templates/publications/index.html index 54d3fc0..eb2e81b 100644 --- a/uploader/templates/publications/index.html +++ b/uploader/templates/publications/index.html @@ -17,7 +17,7 @@ </div> <div class="row"> - <p>Click on title to view more details and to edit details for that + <p>Click on the title to view more details or to edit the information for that publication.</p> </div> @@ -25,7 +25,7 @@ <table id="tbl-list-publications" class="table compact stripe"> <thead> <tr> - <th>#</th> + <th>Index</th> <th>PubMed ID</th> <th>Title</th> <th>Authors</th> diff --git a/uploader/templates/species/base.html b/uploader/templates/species/base.html index a7c1a8f..3be79f0 100644 --- a/uploader/templates/species/base.html +++ b/uploader/templates/species/base.html @@ -2,9 +2,11 @@ {%block breadcrumbs%} {{super()}} +{%if species%} <li class="breadcrumb-item"> <a href="{{url_for('species.view_species', species_id=species['SpeciesId'])}}"> {{species["Name"]|title}} </a> </li> +{%endif%} {%endblock%} |
