diff options
Diffstat (limited to 'uploader')
53 files changed, 1488 insertions, 549 deletions
diff --git a/uploader/__init__.py b/uploader/__init__.py index cae531b..23e66c1 100644 --- a/uploader/__init__.py +++ b/uploader/__init__.py @@ -6,6 +6,9 @@ from pathlib import Path from flask import Flask, request from flask_session import Session +from cachelib import FileSystemCache + +from gn_libs import jobs as gnlibs_jobs from uploader.oauth2.client import user_logged_in, authserver_authorise_uri @@ -51,9 +54,15 @@ def setup_logging(app: Flask) -> Flask: return __log_gunicorn__(app) if bool(software) else __log_dev__(app) -def create_app(): - """The application factory""" +def create_app(config: dict = {}): + """The application factory. + + config: dict + Useful to override settings in the settings files and environment + especially in environments such as testing.""" app = Flask(__name__) + + ### BEGIN: Application configuration app.config.from_pyfile( Path(__file__).parent.joinpath("default_settings.py")) if "UPLOADER_CONF" in os.environ: @@ -68,6 +77,13 @@ def create_app(): if secretsfile.exists(): # Silently ignore secrets if the file does not exist. app.config.from_pyfile(secretsfile) + app.config.update(config) # Override everything with passed in config + ### END: Application configuration + + app.config["SESSION_CACHELIB"] = FileSystemCache( + cache_dir=Path(app.config["SESSION_FILESYSTEM_CACHE_PATH"]).absolute(), + threshold=int(app.config["SESSION_FILESYSTEM_CACHE_THRESHOLD"]), + default_timeout=int(app.config["SESSION_FILESYSTEM_CACHE_TIMEOUT"])) setup_logging(app) @@ -88,4 +104,5 @@ def create_app(): app.register_blueprint(speciesbp, url_prefix="/species") register_error_handlers(app) + gnlibs_jobs.init_app(app) return app diff --git a/uploader/authorisation.py b/uploader/authorisation.py index bd3454c..bc950d8 100644 --- a/uploader/authorisation.py +++ b/uploader/authorisation.py @@ -16,13 +16,12 @@ def require_login(function): @wraps(function) def __is_session_valid__(*args, **kwargs): """Check that the user is logged in and their token is valid.""" - def __clear_session__(_no_token): - session.clear_session_info() - flash("You need to be logged in.", "alert-danger big-alert") + def __alert_needs_sign_in__(_no_token): + flash("You need to be signed in.", "alert alert-danger big-alert") return redirect("/") return session.user_token().either( - __clear_session__, + __alert_needs_sign_in__, lambda token: function(*args, **kwargs)) return __is_session_valid__ diff --git a/uploader/base_routes.py b/uploader/base_routes.py index 326086f..74a3b90 100644 --- a/uploader/base_routes.py +++ b/uploader/base_routes.py @@ -35,8 +35,8 @@ def appenv(): @base.route("/bootstrap/<path:filename>") def bootstrap(filename): """Fetch bootstrap files.""" - return send_from_directory( - appenv(), f"share/genenetwork2/javascript/bootstrap/{filename}") + return send_from_directory(appenv(), f"share/web/bootstrap/{filename}") + @base.route("/jquery/<path:filename>") @@ -52,6 +52,12 @@ def datatables(filename): return send_from_directory( appenv(), f"share/genenetwork2/javascript/DataTables/{filename}") +@base.route("/datatables-extensions/<path:filename>") +def datatables_extensions(filename): + """Fetch DataTables files.""" + return send_from_directory( + appenv(), f"share/genenetwork2/javascript/DataTablesExtensions/{filename}") + @base.route("/node-modules/<path:filename>") def node_modules(filename): diff --git a/uploader/default_settings.py b/uploader/default_settings.py index 1acb247..1136ff8 100644 --- a/uploader/default_settings.py +++ b/uploader/default_settings.py @@ -2,9 +2,12 @@ The default configuration file. The values here should be overridden in the actual configuration file used for the production and staging systems. """ +import hashlib + LOG_LEVEL = "WARNING" SECRET_KEY = b"<Please! Please! Please! Change This!>" UPLOAD_FOLDER = "/tmp/qc_app_files" +TEMPORARY_DIRECTORY = "/tmp/gn-uploader-tmpdir" REDIS_URL = "redis://" JOBS_TTL_SECONDS = 1209600 # 14 days GNQC_REDIS_PREFIX="gn-uploader" @@ -12,9 +15,18 @@ SQL_URI = "" GN2_SERVER_URL = "https://genenetwork.org/" -SESSION_TYPE = "redis" SESSION_PERMANENT = True SESSION_USE_SIGNER = True +SESSION_TYPE = "cachelib" +## --- Settings for CacheLib session type --- ## +## --- These are on flask-session config variables --- ## +## --- https://cachelib.readthedocs.io/en/stable/file/ --- ## +SESSION_FILESYSTEM_CACHE_PATH = "./flask_session" +SESSION_FILESYSTEM_CACHE_THRESHOLD = 500 +SESSION_FILESYSTEM_CACHE_TIMEOUT = 300 +SESSION_FILESYSTEM_CACHE_MODE = 0o600 +SESSION_FILESYSTEM_CACHE_HASH_METHOD = hashlib.md5 +## --- END: Settings for CacheLib session type --- ## JWKS_ROTATION_AGE_DAYS = 7 # Days (from creation) to keep a JWK in use. JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before deleting it. diff --git a/uploader/files/views.py b/uploader/files/views.py index 8d81654..ddf5350 100644 --- a/uploader/files/views.py +++ b/uploader/files/views.py @@ -59,7 +59,7 @@ def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path: with open(chunkfile, "rb") as _chunkdata: _target.write(_chunkdata.read()) - chunkfile.unlink() + chunkfile.unlink(missing_ok=True) return targetfile diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py index 0433420..54c2444 100644 --- a/uploader/genotypes/views.py +++ b/uploader/genotypes/views.py @@ -12,12 +12,12 @@ from flask import (flash, from uploader.ui import make_template_renderer from uploader.oauth2.client import oauth2_post from uploader.authorisation import require_login +from uploader.route_utils import generic_select_population +from uploader.datautils import safe_int, enumerate_sequence from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler from uploader.request_checks import with_species, with_population -from uploader.datautils import safe_int, order_by_family, enumerate_sequence -from uploader.population.models import (populations_by_species, - population_by_species_and_id) +from uploader.population.models import population_by_species_and_id from .models import (genotype_markers, genotype_dataset, @@ -35,8 +35,15 @@ def index(): with database_connection(app.config["SQL_URI"]) as conn: if not bool(request.args.get("species_id")): return render_template("genotypes/index.html", - species=order_by_family(all_species(conn)), + species=all_species(conn), activelink="genotypes") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.genotypes.select_population")) + species = species_by_id(conn, request.args.get("species_id")) if not bool(species): flash(f"Could not find species with ID '{request.args.get('species_id')}'!", @@ -50,28 +57,16 @@ def index(): methods=["GET"]) @require_login @with_species(redirect_uri="species.populations.genotypes.index") -def select_population(species: dict, species_id: int): +def select_population(species: dict, species_id: int):# pylint: disable=[unused-argument] """Select the population under which the genotypes go.""" - with database_connection(app.config["SQL_URI"]) as conn: - if not bool(request.args.get("population_id")): - return render_template("genotypes/select-population.html", - species=species, - populations=order_by_family( - populations_by_species(conn, species_id), - order_key="FamilyOrder"), - activelink="genotypes") - - population = population_by_species_and_id( - conn, species_id, request.args.get("population_id")) - if not bool(population): - flash("Invalid population selected!", "alert-danger") - return redirect(url_for( - "species.populations.genotypes.select_population", - species_id=species_id)) - - return redirect(url_for("species.populations.genotypes.list_genotypes", - species_id=species_id, - population_id=population["Id"])) + return generic_select_population( + species, + "genotypes/select-population.html", + request.args.get("population_id") or "", + "species.populations.genotypes.select_population", + "species.populations.genotypes.list_genotypes", + "genotypes", + "Invalid population selected!") @genotypesbp.route( diff --git a/uploader/monadic_requests.py b/uploader/monadic_requests.py index c492df5..f1f5c77 100644 --- a/uploader/monadic_requests.py +++ b/uploader/monadic_requests.py @@ -5,12 +5,12 @@ from typing import Union, Optional, Callable import requests from requests.models import Response from pymonad.either import Left, Right, Either +from markupsafe import escape as markupsafe_escape from flask import (flash, request, redirect, render_template, - current_app as app, - escape as flask_escape) + current_app as app) # HTML Status codes indicating a successful request. SUCCESS_CODES = (200, 201, 202, 203, 204, 205, 206, 207, 208, 226) @@ -39,9 +39,9 @@ def make_error_handler( trace=traceback.format_exception(resp_or_exc)) if isinstance(resp_or_exc, Response): flash("The authorisation server responded with " - f"({flask_escape(resp_or_exc.status_code)}, " - f"{flask_escape(resp_or_exc.reason)}) for the request to " - f"'{flask_escape(resp_or_exc.request.url)}'", + f"({markupsafe_escape(resp_or_exc.status_code)}, " + f"{markupsafe_escape(resp_or_exc.reason)}) for the request to " + f"'{markupsafe_escape(resp_or_exc.request.url)}'", "alert-danger") return redirect_to diff --git a/uploader/oauth2/client.py b/uploader/oauth2/client.py index 1efa299..12fbf80 100644 --- a/uploader/oauth2/client.py +++ b/uploader/oauth2/client.py @@ -1,6 +1,7 @@ """OAuth2 client utilities.""" import json import time +import uuid import random from datetime import datetime, timedelta from urllib.parse import urljoin, urlparse @@ -146,9 +147,24 @@ def oauth2_client(): __client__) +def fetch_user_details() -> Either: + """Retrieve user details from the auth server""" + suser = session.session_info()["user"] + if suser["email"] == "anon@ymous.user": + udets = oauth2_get("auth/user/").then( + lambda usrdets: session.set_user_details({ + "user_id": uuid.UUID(usrdets["user_id"]), + "name": usrdets["name"], + "email": usrdets["email"], + "token": session.user_token()})) + return udets + return Right(suser) + + def user_logged_in(): """Check whether the user has logged in.""" suser = session.session_info()["user"] + fetch_user_details() return suser["logged_in"] and suser["token"].is_right() diff --git a/uploader/oauth2/views.py b/uploader/oauth2/views.py index 61037f3..db4ef61 100644 --- a/uploader/oauth2/views.py +++ b/uploader/oauth2/views.py @@ -24,22 +24,24 @@ from .client import ( user_logged_in, authserver_uri, oauth2_clientid, + fetch_user_details, oauth2_clientsecret) oauth2 = Blueprint("oauth2", __name__) + @oauth2.route("/code") def authorisation_code(): """Receive authorisation code from auth server and use it to get token.""" def __process_error__(resp_or_exception): app.logger.debug("ERROR: (%s)", resp_or_exception) flash("There was an error retrieving the authorisation token.", - "alert-danger") + "alert alert-danger") return redirect("/") def __fail_set_user_details__(_failure): app.logger.debug("Fetching user details fails: %s", _failure) - flash("Could not retrieve the user details", "alert-danger") + flash("Could not retrieve the user details", "alert alert-danger") return redirect("/") def __success_set_user_details__(_success): @@ -48,19 +50,13 @@ def authorisation_code(): def __success__(token): session.set_user_token(token) - return oauth2_get("auth/user/").then( - lambda usrdets: session.set_user_details({ - "user_id": uuid.UUID(usrdets["user_id"]), - "name": usrdets["name"], - "email": usrdets["email"], - "token": session.user_token(), - "logged_in": True})).either( + return fetch_user_details().either( __fail_set_user_details__, __success_set_user_details__) code = request.args.get("code", "").strip() if not bool(code): - flash("AuthorisationError: No code was provided.", "alert-danger") + flash("AuthorisationError: No code was provided.", "alert alert-danger") return redirect("/") baseurl = urlparse(request.base_url, scheme=request.scheme) @@ -116,7 +112,7 @@ def logout(): _user = session_info["user"] _user_str = f"{_user['name']} ({_user['email']})" session.clear_session_info() - flash("Successfully logged out.", "alert-success") + flash("Successfully signed out.", "alert alert-success") return redirect("/") if user_logged_in(): @@ -134,5 +130,5 @@ def logout(): cleanup_thunk=lambda: __unset_session__( session.session_info())), lambda res: __unset_session__(session.session_info())) - flash("There is no user that is currently logged in.", "alert-info") + flash("There is no user that is currently logged in.", "alert alert-info") return redirect("/") diff --git a/uploader/phenotypes/misc.py b/uploader/phenotypes/misc.py new file mode 100644 index 0000000..cbe3b7f --- /dev/null +++ b/uploader/phenotypes/misc.py @@ -0,0 +1,26 @@ +"""Miscellaneous functions handling phenotypes and phenotypes data.""" +import logging + +logger = logging.getLogger(__name__) + + +def phenotypes_data_differences( + filedata: tuple[dict, ...], dbdata: tuple[dict, ...] +) -> tuple[dict, ...]: + """Compute differences between file data and db data""" + diff = tuple() + for filerow, dbrow in zip( + sorted(filedata, key=lambda item: (item["phenotype_id"], item["xref_id"])), + sorted(dbdata, key=lambda item: (item["PhenotypeId"], item["xref_id"]))): + for samplename, value in filerow["data"].items(): + if value != dbrow["data"].get(samplename, {}).get("value"): + diff = diff + ({ + "PhenotypeId": filerow["phenotype_id"], + "xref_id": filerow["xref_id"], + "DataId": dbrow["DataId"], + "StrainId": dbrow["data"].get(samplename, {}).get("StrainId"), + "StrainName": samplename, + "value": value + },) + + return diff diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py index e1ec0c9..4a229e6 100644 --- a/uploader/phenotypes/models.py +++ b/uploader/phenotypes/models.py @@ -75,7 +75,7 @@ def dataset_phenotypes(conn: mdb.Connection, limit: Optional[int] = None) -> tuple[dict, ...]: """Fetch the actual phenotypes.""" _query = ( - "SELECT pheno.*, pxr.Id AS xref_id, ist.InbredSetCode FROM Phenotype AS pheno " + "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, ist.InbredSetCode FROM Phenotype AS pheno " "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " "INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId " "INNER JOIN InbredSet AS ist ON pf.InbredSetId=ist.Id " @@ -254,3 +254,32 @@ def save_new_dataset(cursor: Cursor, params) debug_query(cursor, app.logger) return {**params, "Id": cursor.lastrowid} + + +def phenotypes_data_by_ids( + conn: mdb.Connection, + inbred_pheno_xref: dict[str, int] +) -> tuple[dict, ...]: + """Fetch all phenotype data, filtered by the `inbred_pheno_xref` mapping.""" + _paramstr = ",".join(["(%s, %s, %s)"] * len(inbred_pheno_xref)) + _query = ("SELECT " + "pub.PubMed_ID, pheno.*, pxr.*, pd.*, str.*, iset.InbredSetCode " + "FROM Publication AS pub " + "RIGHT JOIN PublishXRef AS pxr0 ON pub.Id=pxr0.PublicationId " + "INNER JOIN Phenotype AS pheno ON pxr0.PhenotypeId=pheno.id " + "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " + "INNER JOIN PublishData AS pd ON pxr.DataId=pd.Id " + "INNER JOIN Strain AS str ON pd.StrainId=str.Id " + "INNER JOIN StrainXRef AS sxr ON str.Id=sxr.StrainId " + "INNER JOIN PublishFreeze AS pf ON sxr.InbredSetId=pf.InbredSetId " + "INNER JOIN InbredSet AS iset ON pf.InbredSetId=iset.InbredSetId " + f"WHERE (pxr.InbredSetId, pheno.Id, pxr.Id) IN ({_paramstr}) " + "ORDER BY pheno.Id") + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(_query, tuple(item for row in inbred_pheno_xref + for item in (row["population_id"], + row["phenoid"], + row["xref_id"]))) + debug_query(cursor, app.logger) + return tuple( + reduce(__organise_by_phenotype__, cursor.fetchall(), {}).values()) diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py index ddec54c..a18c44d 100644 --- a/uploader/phenotypes/views.py +++ b/uploader/phenotypes/views.py @@ -1,8 +1,11 @@ """Views handling ('classical') phenotypes.""" import sys +import csv import uuid import json +import logging import datetime +import tempfile from typing import Any from pathlib import Path from zipfile import ZipFile @@ -13,6 +16,7 @@ from redis import Redis from pymonad.either import Left from requests.models import Response from MySQLdb.cursors import DictCursor +from werkzeug.utils import secure_filename from gn_libs.mysqldb import database_connection from flask import (flash, request, @@ -20,6 +24,7 @@ from flask import (flash, jsonify, redirect, Blueprint, + send_file, current_app as app) # from r_qtl import r_qtl2 as rqtl2 @@ -31,12 +36,12 @@ from uploader.files import save_file#, fullpath from uploader.ui import make_template_renderer from uploader.oauth2.client import oauth2_post from uploader.authorisation import require_login +from uploader.route_utils import generic_select_population +from uploader.datautils import safe_int, enumerate_sequence from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler from uploader.request_checks import with_species, with_population -from uploader.datautils import safe_int, order_by_family, enumerate_sequence -from uploader.population.models import (populations_by_species, - population_by_species_and_id) +from uploader.samples.models import samples_by_species_and_population from uploader.input_validation import (encode_errors, decode_errors, is_valid_representative_name) @@ -47,6 +52,7 @@ from .models import (dataset_by_id, save_new_dataset, dataset_phenotypes, datasets_by_population, + phenotypes_data_by_ids, phenotype_publication_data) phenotypesbp = Blueprint("phenotypes", __name__) @@ -62,10 +68,16 @@ def index(): with database_connection(app.config["SQL_URI"]) as conn: if not bool(request.args.get("species_id")): return render_template("phenotypes/index.html", - species=order_by_family(all_species(conn)), + species=all_species(conn), activelink="phenotypes") - species = species_by_id(conn, request.args.get("species_id")) + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.phenotypes.select_population")) + + species = species_by_id(conn, species_id) if not bool(species): flash("No such species!", "alert-danger") return redirect(url_for("species.populations.phenotypes.index")) @@ -79,27 +91,14 @@ def index(): @with_species(redirect_uri="species.populations.phenotypes.index") def select_population(species: dict, **kwargs):# pylint: disable=[unused-argument] """Select the population for your phenotypes.""" - with database_connection(app.config["SQL_URI"]) as conn: - if not bool(request.args.get("population_id")): - return render_template("phenotypes/select-population.html", - species=species, - populations=order_by_family( - populations_by_species( - conn, species["SpeciesId"]), - order_key="FamilyOrder"), - activelink="phenotypes") - - population = population_by_species_and_id( - conn, species["SpeciesId"], int(request.args["population_id"])) - if not bool(population): - flash("No such population found!", "alert-danger") - return redirect(url_for( - "species.populations.phenotypes.select_population", - species_id=species["SpeciesId"])) - - return redirect(url_for("species.populations.phenotypes.list_datasets", - species_id=species["SpeciesId"], - population_id=population["Id"])) + return generic_select_population( + species, + "phenotypes/select-population.html", + request.args.get("population_id") or "", + "species.populations.phenotypes.select_population", + "species.populations.phenotypes.list_datasets", + "phenotypes", + "No such population found!") @@ -852,3 +851,187 @@ def edit_phenotype_data(# pylint: disable=[unused-argument] population_id=population["Id"], dataset_id=dataset["Id"], xref_id=xref_id)) + + +def process_phenotype_data_for_download(pheno: dict) -> dict: + """Sanitise data for download.""" + return { + "UniqueIdentifier": f"phId:{pheno['Id']}::xrId:{pheno['xref_id']}", + **{ + key: val for key, val in pheno.items() + if key not in ("Id", "xref_id", "data", "Units") + }, + **{ + data_item["StrainName"]: data_item["value"] + for data_item in pheno.get("data", {}).values() + } + } + + +BULK_EDIT_COMMON_FIELDNAMES = [ + "UniqueIdentifier", + "Post_publication_description", + "Pre_publication_abbreviation", + "Pre_publication_description", + "Original_description", + "Post_publication_abbreviation", + "PubMed_ID" +] + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/edit-download", + methods=["POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def edit_download_phenotype_data(# pylint: disable=[unused-argument] + species: dict, + population: dict, + dataset: dict, + **kwargs +): + formdata = request.json + with database_connection(app.config["SQL_URI"]) as conn: + samples_list = [ + sample["Name"] for sample in samples_by_species_and_population( + conn, species["SpeciesId"], population["Id"])] + data = ( + process_phenotype_data_for_download(pheno) + for pheno in phenotypes_data_by_ids(conn, tuple({ + "population_id": population["Id"], + "phenoid": row["phenotype_id"], + "xref_id": row["xref_id"] + } for row in formdata))) + + with (tempfile.TemporaryDirectory( + prefix=app.config["TEMPORARY_DIRECTORY"]) as tmpdir): + filename = Path(tmpdir).joinpath("tempfile.tsv") + with open(filename, mode="w") as outfile: + outfile.write( + "# **DO NOT** delete the 'UniqueIdentifier' row. It is used " + "by the system to identify and edit the correct rows and " + "columns in the database.\n") + outfile.write( + "# The '…_description' fields are useful for you to figure out " + "what row you are working on. Changing any of this fields will " + "also update the database, so do be careful.\n") + outfile.write( + "# Leave a field empty to delete the value in the database.\n") + outfile.write( + "# Any line beginning with a '#' character is considered a " + "comment line. This line, and all the lines above it, are " + "all comment lines. Comment lines will be ignored.\n") + writer = csv.DictWriter(outfile, + fieldnames= ( + BULK_EDIT_COMMON_FIELDNAMES + + samples_list), + dialect="excel-tab") + writer.writeheader() + writer.writerows(data) + outfile.flush() + + return send_file( + filename, + mimetype="text/csv", + as_attachment=True, + download_name=secure_filename(f"{dataset['Name']}_data")) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/edit-upload", + methods=["GET", "POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def edit_upload_phenotype_data(# pylint: disable=[unused-argument] + species: dict, + population: dict, + dataset: dict, + **kwargs +): + if request.method == "GET": + return render_template( + "phenotypes/bulk-edit-upload.html", + species=species, + population=population, + dataset=dataset, + activelink="edit-phenotype") + + edit_file = save_file(request.files["file-upload-bulk-edit-upload"], + Path(app.config["UPLOAD_FOLDER"])) + + from gn_libs import jobs as gnlibs_jobs + from gn_libs import sqlite3 + jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + with sqlite3.connection(jobs_db) as conn: + job_id = uuid.uuid4() + job_cmd = [ + sys.executable, "-u", + "-m", "scripts.phenotypes_bulk_edit", + app.config["SQL_URI"], + jobs_db, + str(job_id), + "--log-level", + logging.getLevelName( + app.logger.getEffectiveLevel() + ).lower() + ] + app.logger.debug("Phenotype-edit, bulk-upload command: %s", job_cmd) + _job = gnlibs_jobs.launch_job( + gnlibs_jobs.initialise_job(conn, + job_id, + job_cmd, + "phenotype-bulk-edit", + extra_meta = { + "edit-file": str(edit_file), + "species-id": species["SpeciesId"], + "population-id": population["Id"], + "dataset-id": dataset["Id"] + }), + jobs_db, + f"{app.config['UPLOAD_FOLDER']}/job_errors", + worker_manager="gn_libs.jobs.launcher") + + + return """ + <p>The following steps need to be performed: + <ol> + <li>Check that all IDs exist</li> + <li>Check for mandatory values</li> + <li>Update descriptions in the database (where changed)</li> + <li>Update publications in the database (where changed): + <ol> + <li>If <strong>PubMed_ID</strong> exists in our database, simply update the + 'PublicationId' value in the 'PublishXRef' table.</li> + <li>If <strong>PubMed_ID</strong> does not exists in our database: + <ol> + <li>fetch the publication's details from PubMed using the new + <strong>PubMed_ID</strong> value.</li> + <li>create a new publication in our database using the fetched data</li> + <li>Update 'PublicationId' value in 'PublishXRef' with ID of newly created + publication</li> + </ol> + </ol> + </li> + <li>Update values in the database (where changed)</li> + </ol> + </p> + + <p><strong>Note:</strong> + <ul> + <li>If a strain that did not have a value is given a value, then we need to + add a new cross-reference for the new DataId created.</li> + <li>If a strain that had a value has its value deleted and left blank, we + need to remove the cross-reference for the existing DataId — or, should we + enter the NULL value instead? Removing the cross-reference might be more + trouble than it is worth.</li> + </ul> + </p> + """ diff --git a/uploader/platforms/views.py b/uploader/platforms/views.py index c20ab44..d12a9ef 100644 --- a/uploader/platforms/views.py +++ b/uploader/platforms/views.py @@ -12,7 +12,7 @@ from flask import ( from uploader.ui import make_template_renderer from uploader.authorisation import require_login from uploader.species.models import all_species, species_by_id -from uploader.datautils import safe_int, order_by_family, enumerate_sequence +from uploader.datautils import safe_int, enumerate_sequence from .models import (save_new_platform, platforms_by_species, @@ -29,9 +29,15 @@ def index(): if not bool(request.args.get("species_id")): return render_template( "platforms/index.html", - species=order_by_family(all_species(conn)), + species=all_species(conn), activelink="platforms") + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.platforms.list_platforms")) + species = species_by_id(conn, request.args["species_id"]) if not bool(species): flash("No species selected.", "alert-danger") diff --git a/uploader/population/models.py b/uploader/population/models.py index 6dcd85e..d78a821 100644 --- a/uploader/population/models.py +++ b/uploader/population/models.py @@ -61,7 +61,7 @@ def save_population(cursor: mdb.cursors.Cursor, population_details: dict) -> dic **population_details, "FamilyOrder": _families.get( population_details["Family"], - max(_families.values())+1) + max((0,) + tuple(_families.values()))+1) } cursor.execute( "INSERT INTO InbredSet(" diff --git a/uploader/population/views.py b/uploader/population/views.py index 4f985f5..270dd5f 100644 --- a/uploader/population/views.py +++ b/uploader/population/views.py @@ -2,6 +2,7 @@ import json import base64 +from markupsafe import escape from MySQLdb.cursors import DictCursor from gn_libs.mysqldb import database_connection from flask import (flash, @@ -19,11 +20,9 @@ from uploader.genotypes.views import genotypesbp from uploader.datautils import enumerate_sequence from uploader.phenotypes.views import phenotypesbp from uploader.expression_data.views import exprdatabp +from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler from uploader.input_validation import is_valid_representative_name -from uploader.species.models import (all_species, - species_by_id, - order_species_by_family) from .models import (save_population, population_families, @@ -48,7 +47,15 @@ def index(): if not bool(request.args.get("species_id")): return render_template( "populations/index.html", - species=order_species_by_family(all_species(conn))) + species=all_species(conn), + activelink="populations") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.list_species_populations")) + species = species_by_id(conn, request.args.get("species_id")) if not bool(species): flash("Invalid species identifier provided!", "alert-danger") @@ -101,6 +108,7 @@ def create_population(species_id: int): {"id": "2", "value": "GEMMA"}, {"id": "3", "value": "R/qtl"}, {"id": "4", "value": "GEMMA, PLINK"}), + return_to=(request.args.get("return_to") or ""), activelink="create-population", **error_values) @@ -151,7 +159,15 @@ def create_population(species_id: int): }) def __flash_success__(_success): - flash("Successfully created resource.", "alert-success") + flash("Successfully created population " + f"{escape(new_population['FullName'])}.", + "alert-success") + return_to = request.form.get("return_to") or "" + if return_to: + return redirect(url_for( + return_to, + species_id=species["SpeciesId"], + population_id=new_population["InbredSetId"])) return redirect(url_for( "species.populations.view_population", species_id=species["SpeciesId"], diff --git a/uploader/publications/__init__.py b/uploader/publications/__init__.py new file mode 100644 index 0000000..57c0cbb --- /dev/null +++ b/uploader/publications/__init__.py @@ -0,0 +1 @@ +"""Package for handling publications.""" diff --git a/uploader/publications/misc.py b/uploader/publications/misc.py new file mode 100644 index 0000000..fca6f71 --- /dev/null +++ b/uploader/publications/misc.py @@ -0,0 +1,25 @@ +"""Miscellaneous functions dealing with publications.""" + + +def publications_differences( + filedata: tuple[dict, ...], + dbdata: tuple[dict, ...], + pubmedid2pubidmap: tuple[dict, ...] +) -> tuple[dict, ...]: + """Compute the differences between file data and db data""" + diff = tuple() + for filerow, dbrow in zip( + sorted(filedata, key=lambda item: ( + item["phenotype_id"], item["xref_id"])), + sorted(dbdata, key=lambda item: ( + item["PhenotypeId"], item["xref_id"]))): + if filerow["PubMed_ID"] == dbrow["PubMed_ID"]: + continue + + newpubmed = filerow["PubMed_ID"] + diff = diff + ({ + **dbrow, + "PubMed_ID": newpubmed, + "PublicationId": pubmedid2pubidmap.get(newpubmed)},) + + return diff diff --git a/uploader/publications/models.py b/uploader/publications/models.py new file mode 100644 index 0000000..3fc9542 --- /dev/null +++ b/uploader/publications/models.py @@ -0,0 +1,73 @@ +"""Module to handle persistence and retrieval of publication to/from MariaDB""" +import logging + +from MySQLdb.cursors import DictCursor + +from gn_libs.mysqldb import Connection, debug_query + +logger = logging.getLogger(__name__) + + +def fetch_phenotype_publications( + conn: Connection, + ids: tuple[tuple[int, int], ...] +) -> tuple[dict, ...]: + """Fetch publication from database by ID.""" + paramstr = ",".join(["(%s, %s)"] * len(ids)) + query = ( + "SELECT " + "pxr.PhenotypeId, pxr.Id AS xref_id, pxr.PublicationId, pub.PubMed_ID " + "FROM PublishXRef AS pxr INNER JOIN Publication AS pub " + "ON pxr.PublicationId=pub.Id " + f"WHERE (pxr.PhenotypeId, pxr.Id) IN ({paramstr})") + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(query, tuple(item for row in ids for item in row)) + return tuple(dict(row) for row in cursor.fetchall()) + + +def create_new_publications( + conn: Connection, + publications: tuple[dict, ...] +) -> tuple[dict, ...]: + if len(publications) > 0: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.executemany( + ("INSERT INTO " + "Publication( " + "PubMed_ID, Abstract, Authors, Title, Journal, Volume, Pages, " + "Month, Year" + ") " + "VALUES(" + "%(pubmed_id)s, %(abstract)s, %(authors)s, %(title)s, " + "%(journal)s, %(volume)s, %(pages)s, %(month)s, %(year)s" + ") " + "ON DUPLICATE KEY UPDATE " + "Abstract=VALUES(Abstract), Authors=VALUES(Authors), " + "Title=VALUES(Title), Journal=VALUES(Journal), " + "Volume=VALUES(Volume), Pages=VALUES(pages), " + "Month=VALUES(Month), Year=VALUES(Year) " + "RETURNING *"), + publications) + return tuple({ + **row, "PublicationId": row["Id"] + } for row in cursor.fetchall()) + return tuple() + + +def update_publications(conn: Connection , publications: tuple[dict, ...]) -> tuple[dict, ...]: + """Update details for multiple publications""" + if len(publications) > 0: + with conn.cursor(cursorclass=DictCursor) as cursor: + logger.debug("UPDATING PUBLICATIONS: %s", publications) + cursor.executemany( + ("UPDATE Publication SET " + "PubMed_ID=%(pubmed_id)s, Abstract=%(abstract)s, " + "Authors=%(authors)s, Title=%(title)s, Journal=%(journal)s, " + "Volume=%(volume)s, Pages=%(pages)s, Month=%(month)s, " + "Year=%(year)s " + "WHERE Id=%(publication_id)s"), + publications) + debug_query(cursor, logger) + return publications + return tuple() + return tuple() diff --git a/uploader/publications/pubmed.py b/uploader/publications/pubmed.py new file mode 100644 index 0000000..ed9b652 --- /dev/null +++ b/uploader/publications/pubmed.py @@ -0,0 +1,103 @@ +"""Module to interact with NCBI's PubMed""" +import logging + +import requests +from lxml import etree + +logger = logging.getLogger(__name__) + + +def __pub_date__(pubdate: etree.Element): + pubyear = pubdate.find("Year") + pubmonth = pubdate.find("Month") + pubday = pubdate.find("Day") + return { + "year": pubyear.text if pubyear is not None else None, + "month": pubmonth.text if pubmonth is not None else None, + "day": pubday.text if pubday is not None else None + } + + +def __journal__(journal: etree.Element) -> dict: + volume = journal.find("JournalIssue/Volume") + issue = journal.find("JournalIssue/Issue") + return { + "volume": volume.text if volume is not None else None, + "issue": issue.text if issue is not None else None, + **__pub_date__(journal.find("JournalIssue/PubDate")), + "journal": journal.find("Title").text + } + +def __author__(author: etree.Element) -> str: + return "%s %s" % ( + author.find("LastName").text, + author.find("Initials").text) + + +def __pages__(pagination: etree.Element) -> str: + start = pagination.find("StartPage") + end = pagination.find("EndPage") + return (start.text + ( + f"-{end.text}" if end is not None else "" + )) if start is not None else "" + + +def __abstract__(article: etree.Element) -> str: + abstract = article.find("Abstract/AbstractText") + return abstract.text if abstract is not None else None + + +def __article__(pubmed_article: etree.Element) -> dict: + article = pubmed_article.find("MedlineCitation/Article") + return { + "pubmed_id": int(pubmed_article.find("MedlineCitation/PMID").text), + "title": article.find("ArticleTitle").text, + **__journal__(article.find("Journal")), + "abstract": __abstract__(article), + "pages": __pages__(article.find("Pagination")), + "authors": ", ".join(__author__(author) + for author in article.findall("AuthorList/Author")) + } + + +def __process_pubmed_publication_data__(text) -> tuple[dict, ...]: + """Process the data from PubMed into usable data.""" + doc = etree.XML(text) + articles = doc.xpath("//PubmedArticle") + logger.debug("Retrieved %s publications from NCBI", len(articles)) + return tuple(__article__(article) for article in articles) + +def fetch_publications(pubmed_ids: tuple[int, ...]) -> tuple[dict, ...]: + """Retrieve data on new publications from NCBI.""" + # See whether we can retrieve multiple publications in one go + # Parse data and save to DB + # Return PublicationId(s) for new publication(s). + if len(pubmed_ids) == 0: + logger.debug("There are no new PubMed IDs to fetch") + return tuple() + + logger.info("Fetching publications data for the following PubMed IDs: %s", + ", ".join((str(pid) for pid in pubmed_ids))) + + # Should we, perhaps, pass this in from a config variable? + uri = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" + try: + response = requests.get( + uri, + params={ + "db": "pubmed", + "retmode": "xml", + "id": ",".join(str(item) for item in pubmed_ids) + }) + + if response.status_code == 200: + return __process_pubmed_publication_data__(response.text) + + logger.error( + "Could not fetch the new publication from %s (status code: %s)", + uri, + response.status_code) + except requests.exceptions.ConnectionError: + logger.error("Could not find the domain %s", uri) + + return tuple() diff --git a/uploader/route_utils.py b/uploader/route_utils.py new file mode 100644 index 0000000..18eadda --- /dev/null +++ b/uploader/route_utils.py @@ -0,0 +1,41 @@ +"""Generic routing utilities.""" +from flask import flash, url_for, redirect, render_template, current_app as app + +from gn_libs.mysqldb import database_connection + +from uploader.population.models import (populations_by_species, + population_by_species_and_id) + +def generic_select_population(# pylint: disable=[too-many-arguments] + species: dict, + template: str, + population_id: str, + back_to: str, + forward_to: str, + activelink: str, + error_message: str = "No such population found!" +): + """Handles common flow for 'select population' step.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(population_id): + return render_template( + template, + species=species, + populations=populations_by_species(conn, species["SpeciesId"]), + activelink=activelink) + + if population_id == "CREATE-POPULATION": + return redirect(url_for( + "species.populations.create_population", + species_id=species["SpeciesId"], + return_to=forward_to)) + + population = population_by_species_and_id( + conn, species["SpeciesId"], int(population_id)) + if not bool(population): + flash(error_message, "alert-danger") + return redirect(url_for(back_to, species_id=species["SpeciesId"])) + + return redirect(url_for(forward_to, + species_id=species["SpeciesId"], + population_id=population["Id"])) diff --git a/uploader/samples/views.py b/uploader/samples/views.py index ed79101..27e5d3c 100644 --- a/uploader/samples/views.py +++ b/uploader/samples/views.py @@ -16,16 +16,15 @@ from uploader import jobs from uploader.files import save_file from uploader.ui import make_template_renderer from uploader.authorisation import require_login -from uploader.request_checks import with_population from uploader.input_validation import is_integer_input -from uploader.datautils import safe_int, order_by_family, enumerate_sequence -from uploader.population.models import population_by_id, populations_by_species +from uploader.population.models import population_by_id +from uploader.route_utils import generic_select_population +from uploader.datautils import safe_int, enumerate_sequence +from uploader.species.models import all_species, species_by_id +from uploader.request_checks import with_species, with_population from uploader.db_utils import (with_db_connection, database_connection, with_redis_connection) -from uploader.species.models import (all_species, - species_by_id, - order_species_by_family) from .models import samples_by_species_and_population @@ -40,8 +39,15 @@ def index(): if not bool(request.args.get("species_id")): return render_template( "samples/index.html", - species=order_species_by_family(all_species(conn)), + species=all_species(conn), activelink="samples") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.samples.select_population")) + species = species_by_id(conn, request.args.get("species_id")) if not bool(species): flash("No such species!", "alert-danger") @@ -52,57 +58,31 @@ def index(): @samplesbp.route("<int:species_id>/samples/select-population", methods=["GET"]) @require_login -def select_population(species_id: int): +@with_species(redirect_uri="species.populations.samples.index") +def select_population(species: dict, **kwargs):# pylint: disable=[unused-argument] """Select the population to use for the samples.""" - with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - if not bool(species): - flash("Invalid species!", "alert-danger") - return redirect(url_for("species.populations.samples.index")) - - if not bool(request.args.get("population_id")): - return render_template("samples/select-population.html", - species=species, - populations=order_by_family( - populations_by_species( - conn, - species_id), - order_key="FamilyOrder"), - activelink="samples") - - population = population_by_id(conn, request.args.get("population_id")) - if not bool(population): - flash("Population not found!", "alert-danger") - return redirect(url_for( - "species.populations.samples.select_population", - species_id=species_id)) - - return redirect(url_for("species.populations.samples.list_samples", - species_id=species_id, - population_id=population["Id"])) + return generic_select_population( + species, + "samples/select-population.html", + request.args.get("population_id") or "", + "species.populations.samples.select_population", + "species.populations.samples.list_samples", + "samples", + "Population not found!") @samplesbp.route("<int:species_id>/populations/<int:population_id>/samples") @require_login -def list_samples(species_id: int, population_id: int): +@with_population( + species_redirect_uri="species.populations.samples.index", + redirect_uri="species.populations.samples.select_population") +def list_samples(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """ List the samples in a particular population and give the ability to upload new ones. """ with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - if not bool(species): - flash("Invalid species!", "alert-danger") - return redirect(url_for("species.populations.samples.index")) - - population = population_by_id(conn, population_id) - if not bool(population): - flash("Population not found!", "alert-danger") - return redirect(url_for( - "species.populations.samples.select_population", - species_id=species_id)) - all_samples = enumerate_sequence(samples_by_species_and_population( - conn, species_id, population_id)) + conn, species["SpeciesId"], population["Id"])) total_samples = len(all_samples) offset = max(safe_int(request.args.get("from") or 0), 0) count = int(request.args.get("count") or 20) diff --git a/uploader/session.py b/uploader/session.py index b538187..5af5827 100644 --- a/uploader/session.py +++ b/uploader/session.py @@ -77,12 +77,15 @@ def set_user_token(token: str) -> SessionInfo: """Set the user's token.""" info = session_info() return save_session_info({ - **info, "user": {**info["user"], "token": Right(token)}})#type: ignore[misc] + **info, + "user": {**info["user"], "token": Right(token), "logged_in": True} + })#type: ignore[misc] def set_user_details(userdets: UserDetails) -> SessionInfo: """Set the user details information""" - return save_session_info({**session_info(), "user": userdets})#type: ignore[misc] + info = session_info() + return save_session_info({**info, "user": {**info["user"], **userdets}})#type: ignore[misc] def user_details() -> UserDetails: """Retrieve user details.""" diff --git a/uploader/species/models.py b/uploader/species/models.py index 51f941c..db53d48 100644 --- a/uploader/species/models.py +++ b/uploader/species/models.py @@ -58,7 +58,8 @@ def save_species(conn: mdb.Connection, common_name: The species' common name. scientific_name; The species' scientific name. """ - genus, species_name = scientific_name.split(" ") + genus, *species_parts = scientific_name.split(" ") + species_name: str = " ".join(species_parts) families = species_families(conn) with conn.cursor() as cursor: cursor.execute("SELECT MAX(OrderId) FROM Species") @@ -68,7 +69,7 @@ def save_species(conn: mdb.Connection, "menu_name": f"{common_name} ({genus[0]}. {species_name.lower()})", "scientific_name": scientific_name, "family": family, - "family_order": families[family], + "family_order": families.get(family, 999999), "taxon_id": taxon_id, "species_order": cursor.fetchone()[0] + 5 } @@ -116,7 +117,8 @@ def update_species(# pylint: disable=[too-many-arguments] species_order: The ordering of this species in relation to others """ with conn.cursor(cursorclass=DictCursor) as cursor: - genus, species_name = scientific_name.split(" ") + genus, *species_parts = scientific_name.split(" ") + species_name = " ".join(species_parts) species = { "species_id": species_id, "common_name": common_name, diff --git a/uploader/species/views.py b/uploader/species/views.py index fee5c75..cea2f68 100644 --- a/uploader/species/views.py +++ b/uploader/species/views.py @@ -1,4 +1,5 @@ """Endpoints handling species.""" +from markupsafe import escape from pymonad.either import Left, Right, Either from gn_libs.mysqldb import database_connection from flask import (flash, @@ -62,6 +63,8 @@ def create_species(): if request.method == "GET": return render_template("species/create-species.html", families=species_families(conn), + return_to=( + request.args.get("return_to") or ""), activelink="create-species") error = False @@ -79,7 +82,7 @@ def create_species(): error = True parts = tuple(name.strip() for name in scientific_name.split(" ")) - if len(parts) != 2 or not all(bool(name) for name in parts): + if (len(parts) != 2 and len(parts) != 3) or not all(bool(name) for name in parts): flash("The scientific name you provided is invalid.", "alert-danger") error = True @@ -113,7 +116,15 @@ def create_species(): species = save_species( conn, common_name, scientific_name, family, taxon_id) - flash("Species saved successfully!", "alert-success") + flash( + f"You have successfully added species " + f"'{escape(species['scientific_name'])} " + f"({escape(species['common_name'])})'.", + "alert-success") + + return_to = request.form.get("return_to").strip() + if return_to: + return redirect(url_for(return_to, species_id=species["species_id"])) return redirect(url_for("species.view_species", species_id=species["species_id"])) diff --git a/uploader/static/css/styles.css b/uploader/static/css/styles.css index ef8725e..80c5a56 100644 --- a/uploader/static/css/styles.css +++ b/uploader/static/css/styles.css @@ -1,137 +1,137 @@ +* { + box-sizing: border-box; +} + body { margin: 0.7em; - box-sizing: border-box; display: grid; - grid-template-columns: 1fr 6fr; - grid-template-rows: 4em 100%; + grid-template-columns: 1fr 9fr; grid-gap: 20px; - font-family: Georgia, Garamond, serif; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; font-style: normal; + font-size: 20px; } #header { - grid-column: 1/3; - width: 100%; - /* background: cyan; */ - padding-top: 0.5em; - border-radius: 0.5em; + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 8fr 2fr; + + /* Content styling */ background-color: #336699; - border-color: #080808; color: #FFFFFF; - background-image: none; + border-radius: 3px; + min-height: 30px; } -#header .header { - font-size: 1.7em; - display: inline-block; - text-align: start; -} +#header #header-text { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; -#header .header-nav { - display: inline-block; - color: #FFFFFF; + /* Content styling */ + padding-left: 1em; } -#header .header-nav li { - border-width: 1px; - border-color: #FFFFFF; - vertical-align: middle; - margin: 0.01em; - border-style: solid; - border-width: 2px; - border-radius: 0.5em; - text-align: center; +#header #header-nav { + /* Place it in the parent element */ + grid-column-start: 2; + grid-column-end: 3; } -#header .header-nav a { +#header #header-nav .nav li a { + /* Content styling */ color: #FFFFFF; - text-decoration: none; + background: #4477AA; + border: solid 5px #336699; + border-radius: 5px; + font-size: 0.7em; + text-align: center; + padding: 1px 7px; } #nav-sidebar { - grid-column: 1/2; - /* background: #e5e5ff; */ - padding-top: 0.5em; - border-radius: 0.5em; - font-size: 1.2em; + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; } -#main { - grid-column: 2/3; - width: 100%; - /* background: gray; */ +#nav-sidebar .nav li a:hover { border-radius: 0.5em; } -.pagetitle { - line-height: 1; - padding-top: 0.2em; - /* background: pink; */ +#nav-sidebar .nav .activemenu { + border-style: solid; border-radius: 0.5em; - /* background-color: #6699CC; */ - /* background-color: #77AADD; */ - background-color: #88BBEE; + border-color: #AAAAAA; + background-color: #EFEFEF; } -.pagetitle .title { - text-align: start; - text-transform: capitalize; - padding-left: 0.5em; - font-size: 1.7em; -} +#main { + /* Place it in the parent element */ + grid-column-start: 2; + grid-column-end: 3; -.pagetitle .breadcrumb { - background: none; + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 1fr; + grid-template-rows: 4em 100%; + grid-gap: 1em; } -.pagetitle .breadcrumb .active a { - color: #333333; -} +#main #pagetitle { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; -.pagetitle .breadcrumb a { - color: #666666; + /* Content-styling */ + border-radius: 3px; + background-color: #88BBEE; } -.main-content { - font-size: 1.275em; +#main #pagetitle .title { + font-size: 1.4em; + text-transform: capitalize; + padding-left: 0.5em; } -.breadcrumb { - text-transform: capitalize; +#main #all-content { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 7fr 3fr; /* For a maximum screen width of 1366 pixels */ + grid-gap: 1.5em; } -dd { - margin-left: 3em; - font-size: 0.88em; - padding-bottom: 1em; +#main #all-content .row { + margin: 0 2px; } -input[type="submit"], .btn { - text-transform: capitalize; +#main #all-content #main-content { + background: #FFFFFF; + max-width: 950px; } -.card { - margin-top: 0.3em; - border-width: 1px; - border-style: solid; - border-radius: 0.3em; - border-color: #AAAAAA; - padding: 0.5em; +#pagetitle .breadcrumb { + background: none; + text-transform: capitalize; + font-size: 0.75em; } -.activemenu { - border-style: solid; - border-radius: 0.5em; - border-color: #AAAAAA; - background-color: #EFEFEF; +#pagetitle .breadcrumb .active a { + color: #333333; } -.danger { - color: #A94442; - border-color: #DCA7A7; - background-color: #F2DEDE; +#pagetitle .breadcrumb a { + color: #666666; } .heading { @@ -145,32 +145,20 @@ input[type="submit"], .btn { text-transform: capitalize; } -form { - margin-top: 0.3em; - background: #E5E5FF; - padding: 0.5em; - border-radius:0.5em; -} - -form .form-control { - background-color: #EAEAFF; +input[type="search"] { + border-radius: 5px; } -.table-form-table thead { - background: #E5E5FF; -} - - -.sidebar-content .card .card-title { - font-size: 1.5em; +.btn { + text-transform: Capitalize; } -.sidebar-content .card-text table tbody td:nth-child(1) { - font-weight: bolder; +table.dataTable thead th, table.dataTable tfoot th{ + border-right: 1px solid white; + color: white; + background-color: #369 !important; } -.big-alert { - line-height: 1.5em; - padding: 0.5em 0 0.5em 3em; - margin-top: 0.2em; +table.dataTable tbody tr.selected td { + background-color: #ffee99 !important; } diff --git a/uploader/static/js/datatables.js b/uploader/static/js/datatables.js new file mode 100644 index 0000000..82fd696 --- /dev/null +++ b/uploader/static/js/datatables.js @@ -0,0 +1,69 @@ +/** Handlers for events in datatables **/ + +var addTableLength = (menuList, lengthToAdd, dataLength) => { + if(dataLength >= lengthToAdd) { + newList = structuredClone(menuList);//menuList.slice(0, menuList.length); // shallow copy + newList.push(lengthToAdd); + return newList; + } + return menuList; +}; + +var defaultLengthMenu = (data) => { + menuList = [] + var lengths = [10, 25, 50, 100, 1000, data.length]; + lengths.forEach((len) => { + menuList = addTableLength(menuList, len, data.length); + }); + return menuList; +}; + +var buildDataTable = (tableId, data = [], columns = [], userSettings = {}) => { + var defaultSettings = { + responsive: true, + layout: { + topStart: null, + topEnd: null, + bottomStart: null, + bottomEnd: null, + }, + select: true, + lengthMenu: defaultLengthMenu(data), + language: { + processing: "Processing… Please wait.", + loadingRecords: "Loading table data… Please wait.", + lengthMenu: "", + info: "" + }, + data: data, + columns: columns, + drawCallback: (settings) => { + $(this[0]).find("tbody tr").each((idx, row) => { + var arow = $(row); + var checkboxOrRadio = arow.find(".chk-row-select"); + if (checkboxOrRadio) { + if (arow.hasClass("selected")) { + checkboxOrRadio.prop("checked", true); + } else { + checkboxOrRadio.prop("checked", false); + } + } + }); + } + } + var theDataTable = $(tableId).DataTable({ + ...defaultSettings, + ...userSettings + }); + theDataTable.on("select", (event, datatable, type, cell, originalEvent) => { + datatable.rows({selected: true}).nodes().each((node, index) => { + $(node).find(".chk-row-select").prop("checked", true) + }); + }); + theDataTable.on("deselect", (event, datatable, type, cell, originalEvent) => { + datatable.rows({selected: false}).nodes().each((node, index) => { + $(node).find(".chk-row-select").prop("checked", false) + }); + }); + return theDataTable; +}; diff --git a/uploader/static/js/populations.js b/uploader/static/js/populations.js new file mode 100644 index 0000000..be1231f --- /dev/null +++ b/uploader/static/js/populations.js @@ -0,0 +1,21 @@ +$(() => { + var populationsDataTable = buildDataTable( + "#tbl-select-population", + JSON.parse( + $("#tbl-select-population").attr("data-populations-list")), + [ + { + data: (apopulation) => { + return `<input type="radio" name="population_id"` + + `id="rdo_population_id_${apopulation.InbredSetId}" ` + + `value="${apopulation.InbredSetId}" ` + + `class="chk-row-select">`; + } + }, + { + data: (apopulation) => { + return `${apopulation.FullName} (${apopulation.InbredSetName})`; + } + } + ]); +}); diff --git a/uploader/static/js/species.js b/uploader/static/js/species.js new file mode 100644 index 0000000..9ea3017 --- /dev/null +++ b/uploader/static/js/species.js @@ -0,0 +1,20 @@ +$(() => { + var speciesDataTable = buildDataTable( + "#tbl-select-species", + JSON.parse( + $("#tbl-select-species").attr("data-species-list")), + [ + { + data: (aspecies) => { + return `<input type="radio" name="species_id"` + + `id="rdo_species_id_${aspecies.SpeciesId}" ` + + `value="${aspecies.SpeciesId}" class="chk-row-select">`; + } + }, + { + data: (aspecies) => { + return `${aspecies.FullName} (${aspecies.SpeciesName})`; + } + } + ]); +}); diff --git a/uploader/templates/base.html b/uploader/templates/base.html index c124b13..09e6470 100644 --- a/uploader/templates/base.html +++ b/uploader/templates/base.html @@ -8,14 +8,14 @@ <meta name="viewport" content="width=device-width, initial-scale=1.0" /> {%block extrameta%}{%endblock%} - <title>GN Uploader: {%block title%}{%endblock%}</title> + <title>Data Upload and Quality Control: {%block title%}{%endblock%}</title> <link rel="stylesheet" type="text/css" href="{{url_for('base.bootstrap', filename='css/bootstrap.min.css')}}" /> <link rel="stylesheet" type="text/css" - href="{{url_for('base.bootstrap', - filename='css/bootstrap-theme.min.css')}}" /> + href="{{url_for('base.datatables', + filename='css/dataTables.bootstrap5.min.css')}}" /> <link rel="stylesheet" type="text/css" href="/static/css/styles.css" /> {%block css%}{%endblock%} @@ -23,25 +23,26 @@ </head> <body> - <header id="header" class="container-fluid"> - <div class="row"> - <span class="header col-lg-9">GeneNetwork Data Quality Control and Upload</span> - <nav class="header-nav col-lg-3"> - <ul class="nav justify-content-end"> - <li> - {%if user_logged_in()%} - <a href="{{url_for('oauth2.logout')}}" - title="Log out of the system">{{user_email()}} — Log Out</a> - {%else%} - <a href="{{authserver_authorise_uri()}}" - title="Log in to the system">Log In</a> - {%endif%} - </li> - </ul> - </nav> + <header id="header"> + <span id="header-text">GeneNetwork</span> + <nav id="header-nav"> + <ul class="nav justify-content-end"> + <li> + {%if user_logged_in()%} + <a href="{{url_for('oauth2.logout')}}" + title="Log out of the system"> + <span class="glyphicon glyphicon-user"></span> + Sign Out</a> + {%else%} + <a href="{{authserver_authorise_uri()}}" + title="Log in to the system">Sign In</a> + {%endif%} + </li> + </ul> + </nav> </header> - <aside id="nav-sidebar" class="container-fluid"> + <aside id="nav-sidebar"> <ul class="nav flex-column"> <li {%if activemenu=="home"%}class="activemenu"{%endif%}> <a href="/" >Home</a></li> @@ -70,6 +71,7 @@ <li {%if activemenu=="phenotypes"%}class="activemenu"{%endif%}> <a href="{{url_for('species.populations.phenotypes.index')}}" title="Upload phenotype data.">Phenotype Data</a></li> + <!-- <li {%if activemenu=="expression-data"%}class="activemenu"{%endif%}> <a href="{{url_for('species.populations.expression-data.index')}}" title="Upload expression data." @@ -87,47 +89,70 @@ class="not-implemented" title="View and manage the backgroud jobs you have running"> Background Jobs</a></li> + --> </ul> </aside> - <main id="main" class="main container-fluid"> + <main id="main" class="main"> - <div class="pagetitle row"> - <span class="title">GN Uploader: {%block pagetitle%}{%endblock%}</span> - <nav> - <ol class="breadcrumb"> - <li {%if activelink is not defined or activelink=="home"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('base.index')}}">Home</a> - </li> - {%block lvl1_breadcrumbs%}{%endblock%} - </ol> - </nav> + <div id="pagetitle" class="pagetitle"> + <span class="title">Data Upload and Quality Control: {%block pagetitle%}{%endblock%}</span> + <!-- + <nav> + <ol class="breadcrumb"> + <li {%if activelink is not defined or activelink=="home"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('base.index')}}">Home</a> + </li> + {%block lvl1_breadcrumbs%}{%endblock%} + </ol> + </nav> + --> </div> - <div class="row"> - <div class="container-fluid"> - <div class="col-md-8 main-content"> - {%block contents%}{%endblock%} - </div> - <div class="sidebar-content col-md-4"> - {%block sidebarcontents%}{%endblock%} - </div> + <div id="all-content"> + <div id="main-content"> + {%block contents%}{%endblock%} + </div> + <div id="sidebar-content"> + {%block sidebarcontents%}{%endblock%} </div> </div> </main> + <!-- + Core dependencies + --> <script src="{{url_for('base.jquery', filename='jquery.min.js')}}"></script> <script src="{{url_for('base.bootstrap', filename='js/bootstrap.min.js')}}"></script> + + <!-- + DataTables dependencies + --> + <script type="text/javascript" + src="{{url_for('base.datatables', + filename='js/dataTables.min.js')}}"></script> + <script type="text/javascript" + src="{{url_for('base.datatables_extensions', + filename='scroller/js/dataTables.scroller.min.js')}}"></script> + <script type="text/javascript" + src="{{url_for('base.datatables_extensions', + filename='buttons/js/dataTables.buttons.min.js')}}"></script> + <script type="text/javascript" + src="{{url_for('base.datatables_extensions', + filename='select/js/dataTables.select.min.js')}}"></script> + + <!-- + local dependencies + --> <script type="text/javascript" src="/static/js/misc.js"></script> + <script type="text/javascript" src="/static/js/datatables.js"></script> {%block javascript%}{%endblock%} - </body> - </html> diff --git a/uploader/templates/genotypes/index.html b/uploader/templates/genotypes/index.html index e749f5a..b50ebc5 100644 --- a/uploader/templates/genotypes/index.html +++ b/uploader/templates/genotypes/index.html @@ -26,3 +26,7 @@ species)}} </div> {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/genotypes/select-population.html b/uploader/templates/genotypes/select-population.html index 7c81943..acdd063 100644 --- a/uploader/templates/genotypes/select-population.html +++ b/uploader/templates/genotypes/select-population.html @@ -12,20 +12,14 @@ {{flash_all_messages()}} <div class="row"> - <p> - You have indicated that you intend to upload the genotypes for species - '{{species.FullName}}'. We now just require the population for your - experiment/study, and you should be good to go. - </p> -</div> - -<div class="row"> - {{select_population_form(url_for("species.populations.genotypes.select_population", - species_id=species.SpeciesId), - populations)}} + {{select_population_form(url_for("species.populations.genotypes.select_population", species_id=species.SpeciesId), species, populations)}} </div> {%endblock%} {%block sidebarcontents%} {{display_species_card(species)}} {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> +{%endblock%} diff --git a/uploader/templates/index.html b/uploader/templates/index.html index d6f57eb..aa1414e 100644 --- a/uploader/templates/index.html +++ b/uploader/templates/index.html @@ -10,90 +10,98 @@ <div class="row"> {{flash_all_messages()}} <div class="explainer"> - <p>Welcome to the <strong>GeneNetwork Data Quality Control and Upload System</strong>. This system is provided to help in uploading your data onto GeneNetwork where you can do analysis on it.</p> + <p>Welcome to the <strong>GeneNetwork Data Upload and Quality Control + System</strong>.</p> + <p>This tool helps you prepare and upload research data to GeneNetwork for + analysis.</p> - <p>The sections below provide an overview of what features the menu items on - the left provide to you. Please peruse the information to get a good - big-picture understanding of what the system provides you and how to get - the most out of it.</p> + <h2 class="heading">Getting Started</h2> + <p>The sections below explain the features of the system. Review this guide + to learn how to use the system.</p> {%block extrapageinfo%}{%endblock%} - <h2>Species</h2> - - <p>The GeneNetwork service provides datasets and tools for doing genetic - studies — from - <a href="{{gn2server_intro}}" - target="_blank" - title="GeneNetwork introduction — opens in a new tab."> - its introduction</a>: - - <blockquote class="blockquote"> - <p>GeneNetwork is a group of linked data sets and tools used to study - complex networks of genes, molecules, and higher order gene function - and phenotypes. …</p> - </blockquote> - </p> - - <p>With this in mind, it follows that the data in the system is centered - aroud a variety of species. The <strong>species section</strong> will - list the currently available species in the system, and give you the - ability to add new ones, if the one you want to work on does not currently - exist on GeneNetwork</p> - - <h2>Populations</h2> - - <p>Your studies will probably focus on a particular subset of the entire - species you are interested in – your population.</p> - <p>Populations are a way to organise the species data so as to link data to - specific know populations for a particular species, e.g. The BXD - population of mice (Mus musculus)</p> - <p>In older GeneNetwork documentation, you might run into the term - <em>InbredSet</em>. Should you run into it, it is a term that we've - deprecated that essentially just means the population.</p> - - <h2>Samples</h2> - - <p>These are the samples or individuals (sometimes cases) that were involved - in the experiment, and from whom the data was derived.</p> - - <h2>Genotype Data</h2> - - <p>This section will allow you to view and upload the genetic markers for - your species, and the genotype encodings used for your particular - population.</p> - <p>While, technically, genetic markers relate to the species in general, and - not to a particular population, the data (allele information) itself - relates to the particular population it was generated from – - specifically, to the actual individuals used in the experiment.</p> - <p>This is the reason why the genotype data information comes under the - population, and will check for the prior existence of the related - samples/individuals before attempting an upload of your data.</p> - - <h2>Expression Data</h2> + <h3 class="subheading">Species</h3> - <p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - <strong>TODO</strong>: Document this …</p> + <p>GeneNetwork supports genetic studies across multiple species (e.g. mice + [Mus musculus], human [homo sapiens], rats [Rattus norvegicus], etc.) . + Here you can:</p> + <ul> + <li>View all species that are currently supported</li> + <li>Add new species not yet in the system</li> + </ul> + + <h3 class="subheading">Populations</h3> + + <p>A "population" refers to a specific subgroup within a species that you’re + studying (e.g., BXD mice). Here you can:</p> + <ul> + <li>View the populations that exist for a selected species</li> + <li>Add new populations of study for a selected species</li> + </ul> + + <h3 class="subheading">Samples</h3> + + <p>Manage individual specimens or cases used in your experiments. These + include:</p> + + <ul> + <li>Experimental subjects</li> + <li>Data sources (e.g., tissue samples, clinical cases)</li> + <li>Strain means (instead of entering multiple BXD1 individuals, for + example, the mean would be entered for a single BXD1 strain)</li> + </ul> + + + <h3 class="subheading">Genotype Data</h3> + + <p>Upload and review genetic markers and allele encodings for your + population. Key details:</p> + + <ul> + <li>Markers are species-level (e.g., mouse SNP databases).</li> + <li>Allele data is population-specific (tied to your experimental + samples).</li> + </ul> + + <p><strong>Requirement</strong>: Samples must already have been registered + in the system before uploading genotype data.</p> + + <h3 class="subheading">Phenotype Data</h3> + + <p>Phenotypes are the visible traits or features of a living thing. For + example, phenotypes include:</p> + + <ul> + <li>Weight</li> + <li>Height</li> + <li>Color (such as the color of fur or eyes)</li> + </ul> + + <p>This part of the system will allow you to upload and manage the values + for different phenotypes from various samples in your studies.</p> + + <!-- - <h2>Phenotype Data</h2> + <h3 class="subheading">Expression Data</h3> <p class="text-danger"> <span class="glyphicon glyphicon-exclamation-sign"></span> <strong>TODO</strong>: Document this …</p> - <h2>Individual Data</h2> + <h3 class="subheading">Individual Data</h3> <p class="text-danger"> <span class="glyphicon glyphicon-exclamation-sign"></span> <strong>TODO</strong>: Document this …</p> - <h2>RNA-Seq Data</h2> + <h3 class="subheading">RNA-Seq Data</h3> <p class="text-danger"> <span class="glyphicon glyphicon-exclamation-sign"></span> <strong>TODO</strong>: Document this …</p> </div> + --> </div> {%endblock%} diff --git a/uploader/templates/login.html b/uploader/templates/login.html index 1f71416..e76c644 100644 --- a/uploader/templates/login.html +++ b/uploader/templates/login.html @@ -5,7 +5,8 @@ {%block pagetitle%}log in{%endblock%} {%block extrapageinfo%} -<p class="text-dark text-primary"> - You <strong>do need to be logged in</strong> to upload data onto this system. - Please do that by clicking the "Log In" button at the top of the page.</p> +<p class="text-dark"> + You <strong>need to + <a href="{{authserver_authorise_uri()}}" + title="Sign in to the system">sign in</a></strong> to use this system.</p> {%endblock%} diff --git a/uploader/templates/macro-step-indicator.html b/uploader/templates/macro-step-indicator.html new file mode 100644 index 0000000..ac0be77 --- /dev/null +++ b/uploader/templates/macro-step-indicator.html @@ -0,0 +1,15 @@ +{%macro step_indicator(step, width=100)%} +<svg width="{{width}}" height="{{width}}" xmlns="http://www.w3.org/2000/svg"> + <circle cx="{{0.5*width}}" + cy="{{0.5*width}}" + r="{{0.5*width}}" + fill="#E5E5FF" /> + <text x="{{0.5*width}}" + y="{{0.6*width}}" + font-size="{{0.2*width}}" + text-anchor="middle" + fill="#555555"> + Step {{step}} + </text> +</svg> +{%endmacro%} diff --git a/uploader/templates/phenotypes/add-phenotypes-base.html b/uploader/templates/phenotypes/add-phenotypes-base.html index 97b55f2..a2d9484 100644 --- a/uploader/templates/phenotypes/add-phenotypes-base.html +++ b/uploader/templates/phenotypes/add-phenotypes-base.html @@ -48,7 +48,7 @@ These phenotypes are published</label> </div> - <fieldset id="fldset-publication-info" class="hidden"> + <fieldset id="fldset-publication-info" class="visually-hidden"> <legend>Publication Information</legend> <div class="form-group"> <label for="txt-pubmed-id" class="form-label">Pubmed ID</label> @@ -60,7 +60,7 @@ </span> </div> <span id="search-pubmed-id-error" - class="form-text text-muted text-danger hidden"> + class="form-text text-muted text-danger visually-hidden"> </span><br /> <span class="form-text text-muted"> Enter your publication's PubMed ID above and click "Search" to search @@ -189,10 +189,10 @@ pub_details = $("#fldset-publication-info") if(event.target.checked) { // display the publication details - remove_class(pub_details, "hidden"); + remove_class(pub_details, "visually-hidden"); } else { // hide the publication details - add_class(pub_details, "hidden"); + add_class(pub_details, "visually-hidden"); } }); @@ -275,7 +275,7 @@ var fetch_publication_details = (pubmed_id, complete_thunks) => { error_display = $("#search-pubmed-id-error"); error_display.text(""); - add_class(error_display, "hidden"); + add_class(error_display, "visually-hidden"); $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi", { "method": "GET", @@ -294,7 +294,7 @@ data.result[pubmed_id].error) + "'. Please check ID you provided and try " + "again."); - remove_class(error_display, "hidden"); + remove_class(error_display, "visually-hidden"); } else { fetch_publication_abstract( pubmed_id, diff --git a/uploader/templates/phenotypes/bulk-edit-upload.html b/uploader/templates/phenotypes/bulk-edit-upload.html new file mode 100644 index 0000000..d0f38f5 --- /dev/null +++ b/uploader/templates/phenotypes/bulk-edit-upload.html @@ -0,0 +1,62 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="view-dataset"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">View</a> +</li> +{%endblock%} + +{%block contents%} +<div class="row"> + <p>Upload the edited file you downloaded and edited.</p> +</div> + +<div class="row"> + <form id="frm-bulk-edit-upload" + class="form-horizontal" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.edit_upload_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + enctype="multipart/form-data"> + + <div class="form-group row"> + <label for="file-upload-bulk-edit-upload" + class="form-label col-form-label col-sm-2"> + Edited File</label> + <div class="col-sm-10"> + <input id="file-upload-bulk-edit-upload" + name="file-upload-bulk-edit-upload" + class="form-control" + type="file" + accept="text/tab-separated-values" + required="required" /> + </div> + </div> + + <input type="submit" class="btn btn-primary" + value="upload to edit" /> + + </form> +</div> +{%endblock%} + + +{%block javascript%} +{%endblock%} diff --git a/uploader/templates/phenotypes/create-dataset.html b/uploader/templates/phenotypes/create-dataset.html index 93de92f..8e45491 100644 --- a/uploader/templates/phenotypes/create-dataset.html +++ b/uploader/templates/phenotypes/create-dataset.html @@ -74,8 +74,10 @@ {%endif%} required="required" /> <small class="form-text text-muted"> - <p>A longer, descriptive name for the dataset — useful for humans. - </p></small> + <p>A longer, descriptive name for the dataset. The name is meant for use + by humans, and therefore, it should be clear what the dataset contains + from the name.</p> + </small> </div> <div class="form-group"> diff --git a/uploader/templates/phenotypes/index.html b/uploader/templates/phenotypes/index.html index 0c691e6..689c28e 100644 --- a/uploader/templates/phenotypes/index.html +++ b/uploader/templates/phenotypes/index.html @@ -11,16 +11,11 @@ {{flash_all_messages()}} <div class="row"> - <p>This section deals with phenotypes that - <span class="text-warning"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - … what are the characteristics of these phenotypes? …</span></p> - <p>Select the species to begin the process of viewing/uploading data about - your phenotypes</p> + {{select_species_form(url_for("species.populations.phenotypes.index"), species)}} </div> +{%endblock%} -<div class="row"> - {{select_species_form(url_for("species.populations.phenotypes.index"), - species)}} -</div> + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> {%endblock%} diff --git a/uploader/templates/phenotypes/list-datasets.html b/uploader/templates/phenotypes/list-datasets.html index 2eaf43a..2cf2c7f 100644 --- a/uploader/templates/phenotypes/list-datasets.html +++ b/uploader/templates/phenotypes/list-datasets.html @@ -48,9 +48,12 @@ </tbody> </table> {%else%} - <p class="text-warning"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - There is no dataset for this population!</p> + <p>Phenotypes need to go into a dataset. We do not currently have a dataset + for species <strong>'{{species["FullName"]}} ({{species["Name"]}})'</strong> + phenotypes.</p> + + <p>Do, please, create a new dataset by clicking on the "Create Dataset" button + below and following the prompts/instructions.</p> <p><a href="{{url_for('species.populations.phenotypes.create_dataset', species_id=species.SpeciesId, population_id=population.Id)}}" diff --git a/uploader/templates/phenotypes/select-population.html b/uploader/templates/phenotypes/select-population.html index eafd4a7..48c19b1 100644 --- a/uploader/templates/phenotypes/select-population.html +++ b/uploader/templates/phenotypes/select-population.html @@ -11,18 +11,16 @@ {%block contents%} {{flash_all_messages()}} -<div class="row"> - <p>Select the population for your phenotypes to view and manage the phenotype - datasets that relate to it.</p> -</div> <div class="row"> - {{select_population_form(url_for("species.populations.phenotypes.select_population", - species_id=species.SpeciesId), - populations)}} + {{select_population_form(url_for("species.populations.phenotypes.select_population", species_id=species.SpeciesId), species, populations)}} </div> {%endblock%} {%block sidebarcontents%} {{display_species_card(species)}} {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> +{%endblock%} diff --git a/uploader/templates/phenotypes/view-dataset.html b/uploader/templates/phenotypes/view-dataset.html index 4e1be6b..21563d6 100644 --- a/uploader/templates/phenotypes/view-dataset.html +++ b/uploader/templates/phenotypes/view-dataset.html @@ -5,11 +5,6 @@ {%block title%}Phenotypes{%endblock%} -{%block css%} -<link rel="stylesheet" - href="{{url_for('base.datatables', filename='css/jquery.dataTables.css')}}" /> -{%endblock%} - {%block pagetitle%}Phenotypes{%endblock%} {%block lvl4_breadcrumbs%} @@ -61,10 +56,21 @@ <div class="row"> <h2>Phenotype Data</h2> - <table id="tbl-phenotypes-list" class="table"> + + <p>Click on any of the phenotypes in the table below to view and edit that + phenotype's data.</p> + <p>Use the search to filter through all the phenotypes and find specific + phenotypes of interest.</p> +</div> + + +<div class="row"> + + <table id="tbl-phenotypes-list" class="table compact stripe cell-border"> <thead> <tr> - <th>#</th> + <th></th> + <th>Index</th> <th>Record</th> <th>Description</th> </tr> @@ -81,17 +87,29 @@ {%block javascript%} -<script src="{{url_for('base.datatables', - filename='js/jquery.dataTables.js')}}"></script> <script type="text/javascript"> $(function() { - $("#tbl-phenotypes-list").DataTable({ - responsive: true, - data: {{phenotypes | tojson}}, - columns: [ - {data: "sequence_number"}, + var species_id = {{species.SpeciesId}}; + var population_id = {{population.Id}}; + var dataset_id = {{dataset.Id}}; + var dataset_name = "{{dataset.Name}}"; + var data = {{phenotypes | tojson}}; + + var dtPhenotypesList = buildDataTable( + "#tbl-phenotypes-list", + data, + [ { data: function(pheno) { + return `<input type="checkbox" name="selected-phenotypes" ` + + `id="chk-selected-phenotypes-${pheno.InbredSetCode}_${pheno.xref_id}" ` + + `value="${pheno.InbredSetCode}_${pheno.xref_id}" ` + + `class="chk-row-select" />` + } + }, + {data: "sequence_number"}, + { + data: function(pheno, type, set, meta) { var spcs_id = {{species.SpeciesId}}; var pop_id = {{population.Id}}; var dtst_id = {{dataset.Id}}; @@ -104,13 +122,123 @@ `</a>`; } }, - {data: function(pheno) { - return (pheno.Post_publication_description || - pheno.Original_description || - pheno.Pre_publication_description); - }} - ] - }); + { + data: function(pheno) { + return (pheno.Post_publication_description || + pheno.Original_description || + pheno.Pre_publication_description); + } + } + ], + { + select: "multi+shift", + layout: { + top2: { + buttons: [ + { + extend: "selectAll", + className: "btn btn-info", + titleAttr: "Click to select ALL records in the table." + }, + { + extend: "selectNone", + className: "btn btn-info", + titleAttr: "Click to deselect ANY selected record(s) in the table." + }, + { + text: "Bulk Edit (Download Data)", + className: "btn btn-info btn-bulk-edit", + titleAttr: "Click to download data for editing.", + action: (event, dt, node, config) => { + var phenoids = []; + var selected = dt.rows({selected: true, page: "all"}).data(); + for(var idx = 0; idx < selected.length; idx++) { + phenoids.push({ + phenotype_id: selected[idx].Id, + xref_id: selected[idx].xref_id + }); + } + if(phenoids.length == 0) { + alert("No record selected. Nothing to do!"); + return false; + } + + $(".btn-bulk-edit").prop("disabled", true); + $(".btn-bulk-edit").addClass("d-none"); + var spinner = $( + "<div id='bulk-edit-spinner' class='spinner-grow text-info'>"); + spinner_content = $( + "<span class='visually-hidden'>"); + spinner_content.html( + "Downloading data …"); + spinner.append(spinner_content) + $(".btn-bulk-edit").parent().append( + spinner); + + $.ajax( + (`/species/${species_id}/populations/` + + `${population_id}/phenotypes/datasets/` + + `${dataset_id}/edit-download`), + { + method: "POST", + data: JSON.stringify(phenoids), + xhrFields: { + responseType: "blob" + }, + success: (data, textStatus, jqXHR) => { + var link = document.createElement("a"); + uri = window.URL.createObjectURL(data); + link.href = uri; + link.download = `${dataset_name}_data.tsv`; + + document.body.appendChild(link); + link.click(); + window.URL.revokeObjectURL(uri); + link.remove(); + }, + error: (jQXHR, textStatus, errorThrown) => { + console.log("Experienced an error: ", textStatus); + console.log("The ERROR: ", errorThrown); + }, + complete: (jqXHR, textStatus) => { + $("#bulk-edit-spinner").remove(); + $(".btn-bulk-edit").removeClass( + "d-none"); + $(".btn-bulk-edit").prop( + "disabled", false); + }, + contentType: "application/json" + }); + } + }, + { + text: "Bulk Edit (Upload Data)", + className: "btn btn-info btn-bulk-edit", + titleAttr: "Click to upload edited data you got by clicking the `Bulk Edit (Download Data)` button.", + action: (event, dt, node, config) => { + window.location.assign( + `${window.location.protocol}//` + + `${window.location.host}` + + `/species/${species_id}` + + `/populations/${population_id}` + + `/phenotypes/datasets/${dataset_id}` + + `/edit-upload`) + } + } + ] + }, + top1Start: { + pageLength: { + text: "Show _MENU_ of _TOTAL_" + } + }, + topStart: "info", + top1End: null + }, + rowId: function(pheno) { + return `${pheno.InbredSetCode}_${pheno.xref_id}`; + } + }); }); </script> {%endblock%} diff --git a/uploader/templates/platforms/index.html b/uploader/templates/platforms/index.html index 35b6464..555b444 100644 --- a/uploader/templates/platforms/index.html +++ b/uploader/templates/platforms/index.html @@ -19,3 +19,7 @@ {{select_species_form(url_for("species.platforms.index"), species)}} </div> {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/platforms/list-platforms.html b/uploader/templates/platforms/list-platforms.html index 718dd1d..a6bcfdc 100644 --- a/uploader/templates/platforms/list-platforms.html +++ b/uploader/templates/platforms/list-platforms.html @@ -58,7 +58,7 @@ <table class="table"> <thead> <tr> - <th>#</th> + <th></th> <th>Platform Name</th> <th><a href="https://www.ncbi.nlm.nih.gov/geo/browse/?view=platforms&tax={{species.TaxonomyId}}" title="Gene Expression Omnibus: Platforms section" diff --git a/uploader/templates/populations/create-population.html b/uploader/templates/populations/create-population.html index b05ce37..c0c4f45 100644 --- a/uploader/templates/populations/create-population.html +++ b/uploader/templates/populations/create-population.html @@ -37,12 +37,15 @@ <div class="row"> <form method="POST" action="{{url_for('species.populations.create_population', - species_id=species.SpeciesId)}}"> + species_id=species.SpeciesId, + return_to=return_to)}}"> <legend>Create Population</legend> {{flash_all_messages()}} + <input type="hidden" name="return_to" value="{{return_to}}"> + <div {%if errors.population_fullname%} class="form-group has-error" {%else%} @@ -107,9 +110,12 @@ value="{{error_values.population_code or ''}}" class="form-control" /> <small class="form-text text-muted"> - <p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - What is this field is for? Confirm with Arthur and the rest. + <p class="form-text text-muted"> + This is a 3-character code for your population, that is prepended to + the phenotype identifiers. e.g. For the "BXD Family" population, the + code is "BXD" and therefore, the phenotype identifiers for the + population look like the following examples: <em>BXD_10148</em>, + <em>BXD_10180</em>, <em>BXD_10197</em>, etc. </p> </small> </div> diff --git a/uploader/templates/populations/index.html b/uploader/templates/populations/index.html index 4354e02..d2bee77 100644 --- a/uploader/templates/populations/index.html +++ b/uploader/templates/populations/index.html @@ -22,3 +22,7 @@ {{select_species_form(url_for("species.populations.index"), species)}} </div> {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/populations/list-populations.html b/uploader/templates/populations/list-populations.html index 7c7145f..f780e94 100644 --- a/uploader/templates/populations/list-populations.html +++ b/uploader/templates/populations/list-populations.html @@ -51,7 +51,7 @@ <caption>Populations for {{species.FullName}}</caption> <thead> <tr> - <th>#</th> + <th></th> <th>Name</th> <th>Full Name</th> <th>Description</th> diff --git a/uploader/templates/populations/macro-select-population.html b/uploader/templates/populations/macro-select-population.html index af4fd3a..14b0510 100644 --- a/uploader/templates/populations/macro-select-population.html +++ b/uploader/templates/populations/macro-select-population.html @@ -1,30 +1,52 @@ -{%macro select_population_form(form_action, populations)%} -<form method="GET" action="{{form_action}}"> - <legend>Select Population</legend> - - <div class="form-group"> - <label for="select-population" class="form-label">Select Population</label> - <select id="select-population" - name="population_id" - class="form-control" - required="required"> - <option value="">Select Population</option> - {%for family in populations%} - <optgroup {%if family[0][1] is not none%} - label="{{family[0][1]}}" - {%else%} - label="Undefined" - {%endif%}> - {%for population in family[1]%} - <option value="{{population.Id}}">{{population.FullName}}</option> - {%endfor%} - </optgroup> - {%endfor%} - </select> +{%from "macro-step-indicator.html" import step_indicator%} + +{%macro select_population_form(form_action, species, populations)%} +<form method="GET" action="{{form_action}}" class="form-horizontal"> + + <h2>{{step_indicator("2")}} What population do you want to work with?</h2> + + {%if populations | length != 0%} + + <p class="form-text">Search for, and select the population from the table + below and click "Continue"</p> + + <div class="radio"> + <label class="control-label" for="rdo-cant-find-population"> + <input type="radio" id="rdo-cant-find-population" + name="population_id" value="CREATE-POPULATION" /> + I cannot find the population I want — create it! + </label> + </div> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" value="continue" class="btn btn-primary" /> + </div> + + <div style="margin-top:3em;"> + <table id="tbl-select-population" class="table compact stripe" + data-populations-list='{{populations | tojson}}'> + <thead> + <tr> + <th></th> + <th>Population</th> + </tr> + </thead> + + <tbody></tbody> + </table> </div> - <div class="form-group"> - <input type="submit" value="Select" class="btn btn-primary" /> + {%else%} + <p class="form-text"> + There are no populations currently defined for {{species['FullName']}} + ({{species['SpeciesName']}}).<br /> + Click "Continue" to create the first!</p> + <input type="hidden" name="population_id" value="CREATE-POPULATION" /> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" value="continue" class="btn btn-primary" /> </div> + {%endif%} + </form> {%endmacro%} diff --git a/uploader/templates/samples/index.html b/uploader/templates/samples/index.html index ee4a63e..ee98734 100644 --- a/uploader/templates/samples/index.html +++ b/uploader/templates/samples/index.html @@ -17,3 +17,7 @@ {{select_species_form(url_for("species.populations.samples.index"), species)}} </div> {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/samples/list-samples.html b/uploader/templates/samples/list-samples.html index 13e5cec..185e784 100644 --- a/uploader/templates/samples/list-samples.html +++ b/uploader/templates/samples/list-samples.html @@ -73,7 +73,7 @@ <table class="table"> <thead> <tr> - <th>#</th> + <th></th> <th>Name</th> <th>Auxilliary Name</th> <th>Symbol</th> diff --git a/uploader/templates/samples/select-population.html b/uploader/templates/samples/select-population.html index f437780..1cc7573 100644 --- a/uploader/templates/samples/select-population.html +++ b/uploader/templates/samples/select-population.html @@ -12,28 +12,15 @@ {{flash_all_messages()}} <div class="row"> - <p>You have selected "{{species.FullName}}" as the species that your data relates to.</p> - <p>Next, we need information regarding the population your data relates to. Do please select the population from the existing ones below</p> -</div> - -<div class="row"> {{select_population_form( - url_for("species.populations.samples.select_population", species_id=species.SpeciesId), - populations)}} -</div> - -<div class="row"> - <p> - If you cannot find the population your data relates to in the drop-down - above, you might want to - <a href="{{url_for('species.populations.create_population', - species_id=species.SpeciesId)}}" - title="Create a new population for species '{{species.FullName}},"> - add a new population to GeneNetwork</a> - instead. + url_for("species.populations.samples.select_population", species_id=species.SpeciesId), species, populations)}} </div> {%endblock%} {%block sidebarcontents%} {{display_species_card(species)}} {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> +{%endblock%} diff --git a/uploader/templates/species/create-species.html b/uploader/templates/species/create-species.html index 0d0bedf..138dbaa 100644 --- a/uploader/templates/species/create-species.html +++ b/uploader/templates/species/create-species.html @@ -19,72 +19,88 @@ <div class="row"> <form id="frm-create-species" method="POST" - action="{{url_for('species.create_species')}}"> + action="{{url_for('species.create_species', return_to=return_to)}}" + class="form-horizontal"> <legend>Create Species</legend> {{flash_all_messages()}} + <input type="hidden" name="return_to" value="{{return_to}}"> + <div class="form-group"> - <label for="txt-taxonomy-id" class="form-label"> + <label for="txt-taxonomy-id" class="control-label col-sm-2"> Taxonomy ID</label> - <div class="input-group"> - <input id="txt-taxonomy-id" - name="species_taxonomy_id" - type="text" - class="form-control" /> - <span class="input-group-btn"> - <button id="btn-search-taxonid" class="btn btn-info">Search</button> - </span> + <div class="col-sm-10"> + <div class="input-group"> + <input id="txt-taxonomy-id" + name="species_taxonomy_id" + type="text" + class="form-control" /> + <span class="input-group-btn"> + <button id="btn-search-taxonid" class="btn btn-info">Search</button> + </span> + </div> + <small class="form-text text-small text-muted"> + Use + <a href="https://www.ncbi.nlm.nih.gov/Taxonomy/taxonomyhome.html/" + title="NCBI's Taxonomy Browser homepage" + target="_blank"> + NCBI's Taxonomy Browser homepage</a> to search for the species you + want. If the species exists on NCBI, they will have a Taxonomy ID. Copy + that Taxonomy ID to this field, and click "Search" to auto-fill the + details.<br /> + This field is optional.</small> </div> - <small class="form-text text-small text-muted">Provide the taxonomy ID for - your species that can be used to link to external sites like NCBI. Enter - the taxonomy ID and click "Search" to auto-fill the form with data. - <br /> - While it is recommended to provide a value for this field, doing so is - optional. - </small> </div> <div class="form-group"> - <label for="txt-species-name" class="form-label">Common Name</label> - <input id="txt-species-name" - name="common_name" - type="text" - class="form-control" - required="required" /> - <small class="form-text text-muted">Provide the common, possibly - non-scientific name for the species here, e.g. Human, Mouse, etc.</small> + <label for="txt-species-name" class="control-label col-sm-2">Common Name</label> + <div class="col-sm-10"> + <input id="txt-species-name" + name="common_name" + type="text" + class="form-control" + required="required" /> + <small class="form-text text-muted">This is the day-to-day term used by + laymen, e.g. Mouse (instead of Mus musculus), round worm (instead of + Ascaris lumbricoides), etc.<br /> + For species without this, just enter the scientific name. + </small> + </div> </div> <div class="form-group"> - <label for="txt-species-scientific" class="form-label"> + <label for="txt-species-scientific" class="control-label col-sm-2"> Scientific Name</label> - <input id="txt-species-scientific" - name="scientific_name" - type="text" - class="form-control" - required="required" /> - <small class="form-text text-muted">Provide the scientific name for the - species you are creating, e.g. Homo sapiens, Mus musculus, etc.</small> + <div class="col-sm-10"> + <input id="txt-species-scientific" + name="scientific_name" + type="text" + class="form-control" + required="required" /> + <small class="form-text text-muted">This is the scientific name for the + species e.g. Homo sapiens, Mus musculus, etc.</small> + </div> </div> <div class="form-group"> - <label for="select-species-family" class="form-label">Family</label> - <select id="select-species-family" - name="species_family" - required="required" - class="form-control"> - <option value="">Please select a grouping</option> - {%for family in families%} - <option value="{{family}}">{{family}}</option> - {%endfor%} - </select> - <small class="form-text text-muted"> - This is a generic grouping for the species that determines under which - grouping the species appears in the GeneNetwork menus</small> + <label for="select-species-family" class="control-label col-sm-2">Family</label> + <div class="col-sm-10"> + <select id="select-species-family" + name="species_family" + required="required" + class="form-control"> + <option value="ungrouped">I do not know what to pick</option> + {%for family in families%} + <option value="{{family}}">{{family}}</option> + {%endfor%} + </select> + <small class="form-text text-muted"> + This is a rough grouping of the species.</small> + </div> </div> - <div class="form-group"> + <div class="col-sm-offset-2 col-sm-10"> <input type="submit" value="create new species" class="btn btn-primary" /> @@ -113,7 +129,7 @@ } msg = ( "Request to '${uri}' failed with message '${textStatus}'. " - + "Please try again later, or fill the details manually."); + + "Please try again later, or fill the details manually."); alert(msg); console.error(msg, data, textStatus); return false; diff --git a/uploader/templates/species/list-species.html b/uploader/templates/species/list-species.html index 85c9d40..64084b0 100644 --- a/uploader/templates/species/list-species.html +++ b/uploader/templates/species/list-species.html @@ -29,7 +29,7 @@ <caption>Available Species</caption> <thead> <tr> - <th>#</td> + <th></td> <th title="A common, layman's name for the species.">Common Name</th> <th title="The scientific name for the species">Organism Name</th> <th title="An identifier for the species in the NCBI taxonomy database"> diff --git a/uploader/templates/species/macro-select-species.html b/uploader/templates/species/macro-select-species.html index dd086c0..3714ae4 100644 --- a/uploader/templates/species/macro-select-species.html +++ b/uploader/templates/species/macro-select-species.html @@ -1,36 +1,59 @@ +{%from "macro-step-indicator.html" import step_indicator%} + {%macro select_species_form(form_action, species)%} -{%if species | length > 0%} -<form method="GET" action="{{form_action}}"> - <div class="form-group"> - <label for="select-species" class="form-label">Species</label> - <select id="select-species" - name="species_id" - class="form-control" - required="required"> - <option value="">Select Species</option> - {%for group in species%} - {{group}} - <optgroup {%if group[0][1] is not none%} - label="{{group[0][1].capitalize()}}" - {%else%} - label="Undefined" - {%endif%}> - {%for aspecies in group[1]%} - <option value="{{aspecies.SpeciesId}}">{{aspecies.MenuName}}</option> - {%endfor%} - </optgroup> - {%endfor%} - </select> +<form method="GET" action="{{form_action}}" class="form-horizontal"> + + <h2>{{step_indicator("1")}} What species do you want to work with?</h2> + + {%if species | length != 0%} + + <p class="form-text">Search for, and select the species from the table below + and click "Continue"</p> + + <div class="radio"> + <label for="rdo-cant-find-species" + style="font-weight: 1;"> + <input id="rdo-cant-find-species" type="radio" name="species_id" + value="CREATE-SPECIES" /> + I could not find the species I want (create it). + </label> </div> - <div class="form-group"> - <input type="submit" value="Select" class="btn btn-primary" /> + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" + class="btn btn-primary" + value="continue" /> </div> + + <div style="margin-top:3em;"> + <table id="tbl-select-species" class="table compact stripe" + data-species-list='{{species | tojson}}'> + <div class=""> + <thead> + <tr> + <th></th> + <th>Species Name</th> + </tr> + </thead> + + <tbody></tbody> + </table> + </div> + + {%else%} + + <label class="control-label" for="rdo-cant-find-species"> + <input id="rdo-cant-find-species" type="radio" name="species_id" + value="CREATE-SPECIES" /> + There are no species to select from. Create the first one.</label> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" + class="btn btn-primary col-sm-offset-1" + value="continue" /> + </div> + + {%endif%} + </form> -{%else%} -<p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-mark"></span> - We could not find species to select from! -</p> -{%endif%} {%endmacro%} |