diff options
75 files changed, 2193 insertions, 714 deletions
diff --git a/.guix-channel b/.guix-channel index 54206b2..f1a8fa6 100644 --- a/.guix-channel +++ b/.guix-channel @@ -35,11 +35,12 @@ (channel (name guix-bioinformatics) (url "https://git.genenetwork.org/guix-bioinformatics") - (commit "903465c85c9b2ae28480b236c3364da873ca8f51")) + (commit "9b0955f14ec725990abb1f6af3b9f171e4943f77")) (channel (name guix-past) (url "https://codeberg.org/guix-science/guix-past") (branch "master") + (commit "473c942b509ab3ead35159d27dfbf2031a36cd4d") (introduction (channel-introduction (version 0) @@ -50,6 +51,7 @@ (name guix-rust-past-crates) (url "https://codeberg.org/guix/guix-rust-past-crates.git") (branch "trunk") + (commit "b8b7ffbd1cec9f56f93fae4da3a74163bbc9c570") (introduction (channel-introduction (version 0) diff --git a/README.org b/README.org index ca77653..efa837b 100644 --- a/README.org +++ b/README.org @@ -219,7 +219,7 @@ To check for correct type usage in the application, run: Run unit tests with: #+BEGIN_SRC shell $ export UPLOADER_CONF=</path/to/configuration/file.py> - $ pytest -m unit_test + $ pytest -m unit_test -n auto #+END_SRC To run ALL tests (not just unit tests): diff --git a/qc_app/default_settings.py b/qc_app/default_settings.py index 7a9da0f..7bb0bf8 100644 --- a/qc_app/default_settings.py +++ b/qc_app/default_settings.py @@ -7,7 +7,7 @@ import os LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") SECRET_KEY = b"<Please! Please! Please! Change This!>" -UPLOAD_FOLDER = "/tmp/qc_app_files" +UPLOADS_DIRECTORY = "/tmp/qc_app_files" REDIS_URL = "redis://" JOBS_TTL_SECONDS = 1209600 # 14 days GNQC_REDIS_PREFIX="GNQC" diff --git a/scripts/cli/options.py b/scripts/cli/options.py index 70d2a27..58d3df4 100644 --- a/scripts/cli/options.py +++ b/scripts/cli/options.py @@ -44,3 +44,13 @@ def add_population_id(parser: ArgumentParser) -> ArgumentParser: type=int, help="The ID for the population to operate on.") return parser + + +def add_dataset_id(parser: ArgumentParser) -> ArgumentParser: + """Add dataset-id as a mandatory argument.""" + parser = add_population_id(parser) + parser.add_argument("dataset_id", + metavar="DATASET-ID", + type=int, + help="The ID for the dataset to operate on.") + return parser diff --git a/scripts/insert_samples.py b/scripts/insert_samples.py index fc029f9..96ae8e2 100644 --- a/scripts/insert_samples.py +++ b/scripts/insert_samples.py @@ -6,10 +6,10 @@ import argparse import traceback import MySQLdb as mdb -from redis import Redis + from gn_libs.mysqldb import database_connection -from uploader.check_connections import check_db, check_redis +from uploader.check_connections import check_db from uploader.species.models import species_by_id from uploader.population.models import population_by_id from uploader.samples.models import ( @@ -35,7 +35,6 @@ class SeparatorAction(argparse.Action): setattr(namespace, self.dest, (chr(9) if values == "\\t" else values)) def insert_samples(conn: mdb.Connection,# pylint: disable=[too-many-arguments, too-many-positional-arguments] - rconn: Redis,# pylint: disable=[unused-argument] speciesid: int, populationid: int, samplesfile: pathlib.Path, @@ -119,11 +118,6 @@ if __name__ == "__main__": help=("The character used to delimit (surround?) the value in " "each column.")) - # == Script-specific extras == - parser.add_argument("--redisuri", - help="URL to initialise connection to redis", - default="redis:///") - args = parser.parse_args() return args @@ -132,17 +126,13 @@ if __name__ == "__main__": status_code = 1 # Exit with an Exception args = cli_args() check_db(args.databaseuri) - check_redis(args.redisuri) if not args.samplesfile.exists(): logging.error("File not found: '%s'.", args.samplesfile) return 2 - with (Redis.from_url(args.redisuri, decode_responses=True) as rconn, - database_connection(args.databaseuri) as dbconn): - + with database_connection(args.databaseuri) as dbconn: try: status_code = insert_samples(dbconn, - rconn, args.speciesid, args.populationid, args.samplesfile, diff --git a/scripts/load_phenotypes_to_db.py b/scripts/load_phenotypes_to_db.py index e303bb3..31eb715 100644 --- a/scripts/load_phenotypes_to_db.py +++ b/scripts/load_phenotypes_to_db.py @@ -5,9 +5,9 @@ import json import time import logging import argparse -import datetime from pathlib import Path from zipfile import ZipFile +from datetime import datetime from typing import Any, Iterable from urllib.parse import urljoin from functools import reduce, partial @@ -198,13 +198,16 @@ save_phenotypes_n = partial(save_numeric_data, def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments,too-many-arguments] - authserver, - token, + auth_details, + resource_details, species, population, dataset, xrefdata): """Grant the user access to their data.""" + logger.info("Updating authorisation for the data.") + logger.debug("Resource details for the authorisation: %s", resource_details) + authserver, token = auth_details _tries = 0 _delay = 1 headers = { @@ -215,14 +218,14 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments return urljoin(authserver, endpoint) def __fetch_user_details__(): - logger.debug("… Fetching user details") + logger.info("… Fetching user details") return mrequests.get( authserveruri("/auth/user/"), headers=headers ) def __link_data__(user): - logger.debug("… linking uploaded data to user's group") + logger.info("… linking uploaded data to user's group") return mrequests.post( authserveruri("/auth/data/link/phenotype"), headers=headers, @@ -245,7 +248,7 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments }).then(lambda ld_results: (user, ld_results)) def __fetch_phenotype_category_details__(user, linkeddata): - logger.debug("… fetching phenotype category details") + logger.info("… fetching phenotype category details") return mrequests.get( authserveruri("/auth/resource/categories"), headers=headers @@ -258,20 +261,18 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments ) def __create_resource__(user, linkeddata, category): - logger.debug("… creating authorisation resource object") - now = datetime.datetime.now().isoformat() + logger.info("… creating authorisation resource object") return mrequests.post( authserveruri("/auth/resource/create"), headers=headers, json={ + **resource_details, "resource_category": category["resource_category_id"], - "resource_name": (f"{user['email']}—{dataset['Name']}—{now}—" - f"{len(xrefdata)} phenotypes"), "public": "off" }).then(lambda cr_results: (user, linkeddata, cr_results)) def __attach_data_to_resource__(user, linkeddata, resource): - logger.debug("… attaching data to authorisation resource object") + logger.info("… attaching data to authorisation resource object") return mrequests.post( authserveruri("/auth/resource/data/link"), headers=headers, @@ -288,8 +289,8 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments # This is hacky. If the auth already exists, something went wrong # somewhere. # This needs investigation to recover correctly. - logger.info( - "The authorisation for the data was already set up.") + logger.error( + "Error: The authorisation for the data was already set up.") return 0 logger.error("ERROR: Updating the authorisation for the data failed.") logger.debug( @@ -461,6 +462,25 @@ if __name__ == "__main__": logging.getLogger("uploader.phenotypes.models").setLevel(log_level) + def __parse_resource_details__(meta) -> dict: + """Parse out details regarding the wrapper resource from the metadata.""" + _key_mappings_ = { + # allow both 'data_*' and 'data*' for the metadata. + "data_description": "description", + "datadescription": "description" + } + return { + "resource_name": meta.get( + "dataname", + meta.get("data_name", + "Unnamed phenotypes - " + datetime.now().isoformat())), + "resource_metadata": { + rkey: meta[mkey] + for mkey, rkey in _key_mappings_.items() if mkey in meta + } + } + + def main(): """Entry-point for this script.""" args = parse_args() @@ -516,8 +536,10 @@ if __name__ == "__main__": # Update authorisations (break this down) — maybe loop until it works? logger.info("Updating authorisation.") _job_metadata = job["metadata"] - return update_auth(_job_metadata["authserver"], - _job_metadata["token"], + + return update_auth((_job_metadata["authserver"], + _job_metadata["token"]), + __parse_resource_details__(_job_metadata), *db_results) diff --git a/scripts/phenotypes/__init__.py b/scripts/phenotypes/__init__.py new file mode 100644 index 0000000..73ad839 --- /dev/null +++ b/scripts/phenotypes/__init__.py @@ -0,0 +1 @@ +"Scripts for dealing with phenotypes." diff --git a/scripts/phenotypes/delete_phenotypes.py b/scripts/phenotypes/delete_phenotypes.py new file mode 100644 index 0000000..461f3ec --- /dev/null +++ b/scripts/phenotypes/delete_phenotypes.py @@ -0,0 +1,173 @@ +"""Delete phenotypes.""" +import sys +import logging +from pathlib import Path +from typing import Optional +from urllib.parse import urljoin +from argparse import Namespace, ArgumentParser + +import requests +from MySQLdb.cursors import DictCursor, BaseCursor + +from gn_libs.mysqldb import database_connection + +from uploader.phenotypes.models import delete_phenotypes +from scripts.cli.logging import setup_logging +from scripts.cli.options import (add_logging, + add_mariadb_uri, + add_population_id) + +logger = logging.getLogger(__name__) + +def read_xref_ids_file(filepath: Optional[Path]) -> tuple[int, ...]: + """Read the phenotypes' cross-reference IDS from file.""" + if filepath is None: + return tuple() + + logger.debug("Using file '%s' to retrieve XREF IDs for deletion.", + filepath.name) + _ids: tuple[int, ...] = tuple() + with filepath.open(mode="r") as infile: + for line in infile.readlines(): + try: + _ids += (int(line.strip()),) + except TypeError: + pass + + return _ids + + +def fetch_all_xref_ids( + cursor: BaseCursor, population_id: int) -> tuple[int, ...]: + """Fetch all cross-reference IDs.""" + cursor.execute("SELECT Id FROM PublishXRef WHERE InbredSetId=%s", + (population_id,)) + return tuple(int(row["Id"]) for row in cursor.fetchall()) + + +def update_auth( + auth_details: tuple[str, str], + species_id: int, + population_id: int, + dataset_id: int, + xref_ids: tuple[int, ...] = tuple() +): + """Update the authorisation server: remove items to delete.""" + authserver, token = auth_details + resp = requests.post( + urljoin(authserver, + (f"/auth/data/phenotypes/{species_id}/{population_id}" + f"/{dataset_id}/delete")), + timeout=(9.13, 20), + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + }, + json={"xref_ids": xref_ids}) + resp.raise_for_status() + + +def delete_the_phenotypes( + cursor: BaseCursor, + population_id: int, + xref_ids: tuple[int, ...] = tuple()) -> int: + """Process and delete the phenotypes.""" + delete_phenotypes(cursor, population_id, xref_ids) + + return 0 + +if __name__ == "__main__": + def parse_args() -> Namespace: + """Parse CLI arguments.""" + parser = add_logging( + add_population_id( + add_mariadb_uri( + ArgumentParser( + prog="delete-phenotypes", + description=( + "Script to delete phenotypes from the database."))))) + parser.add_argument( + "dataset_id", + metavar="DATASET-ID", + type=int, + help="The dataset identifier for phenotypes to delete.") + parser.add_argument( + "auth_server_uri", + metavar="AUTH-SERVER-URI", + type=str, + help="URI to the authorisation server.") + parser.add_argument( + "auth_token", + metavar="AUTH-TOKEN", + type=str, + help=("Token to use to update the authorisation system with the " + "deletions done.")) + parser.add_argument( + "--xref_ids_file", + metavar="XREF-IDS-FILE", + type=Path, + help=("Path to a file with phenotypes cross-reference IDs to " + "delete.")) + parser.add_argument( + "--delete-all", + action="store_true", + help=("If no 'XREF-IDS-FILE' is provided, this flag determines " + "whether or not all the phenotypes for the given population " + "will be deleted.")) + return parser.parse_args() + + + def main(): + """The `delete-phenotypes` script's entry point.""" + args = parse_args() + setup_logging(logger, args.log_level.upper(), tuple()) + with (database_connection(args.db_uri) as conn, + conn.cursor(cursorclass=DictCursor) as cursor): + xref_ids = read_xref_ids_file(args.xref_ids_file) + try: + assert not (len(xref_ids) > 0 and args.delete_all) + xref_ids = (fetch_all_xref_ids(cursor, args.population_id) + if args.delete_all else xref_ids) + logger.debug("Will delete %s phenotypes and related data", + len(xref_ids)) + if len(xref_ids) == 0: + print("No cross-reference IDs were provided. Aborting.") + return 0 + + print("Updating authorisations: ", end="") + update_auth((args.auth_server_uri, args.auth_token), + args.species_id, + args.population_id, + args.dataset_id, + xref_ids) + print("OK.") + print("Deleting the data: ", end="") + delete_phenotypes(cursor, args.population_id, xref_ids=xref_ids) + print("OK.") + if args.xref_ids_file is not None: + print("Deleting temporary file: ", end="") + args.xref_ids_file.unlink() + print("OK.") + + return 0 + except AssertionError: + logger.error( + "'DELETE-ALL' and 'XREF-IDS' are mutually exclusive. " + "If you specify the list of XREF-IDS (in a file) to delete " + "and also specify to 'DELETE-ALL' phenotypes in the " + "population, we have no way of knowing what it is you want.") + return 1 + except requests.exceptions.HTTPError as _exc: + resp = _exc.response + resp_data = resp.json() + logger.debug("%s: %s", + resp_data["error"], + resp_data["error_description"], + exc_info=True) + return 1 + except Exception as _exc:# pylint: disable=[broad-exception-caught] + logger.debug("Failed while attempting to delete phenotypes.", + exc_info=True) + return 1 + + sys.exit(main()) diff --git a/scripts/rqtl2/phenotypes_qc.py b/scripts/rqtl2/phenotypes_qc.py index 72d6c83..084c876 100644 --- a/scripts/rqtl2/phenotypes_qc.py +++ b/scripts/rqtl2/phenotypes_qc.py @@ -198,7 +198,7 @@ def qc_phenocovar_file( "-", "-", (f"File {filepath.name} is missing the {heading} heading " - "in the header line."))),) + "in the header row/line."))),) def collect_errors(errors_and_linecount, line): _errs, _lc = errors_and_linecount @@ -312,8 +312,9 @@ def qc_pheno_file(# pylint: disable=[too-many-locals, too-many-arguments, too-ma "header row", "-", ", ".join(_absent), - ("The following phenotype names do not exist in any of the " - f"provided phenocovar files: ({', '.join(_absent)})"))),) + ("The following trait names/identifiers do not exist in any of " + "the provided descriptions/covariates files: " + f"({', '.join(_absent)})"))),) def collect_errors(errors_and_linecount, line): _errs, _lc = errors_and_linecount diff --git a/scripts/run_qtlreaper.py b/scripts/run_qtlreaper.py index 7d58402..2269ea6 100644 --- a/scripts/run_qtlreaper.py +++ b/scripts/run_qtlreaper.py @@ -1,10 +1,12 @@ """Script to run rust-qtlreaper and update database with results.""" +import os import sys import csv import time import secrets import logging import subprocess +import multiprocessing from pathlib import Path from functools import reduce from typing import Union, Iterator @@ -146,30 +148,55 @@ def dispatch(args: Namespace) -> int: _qtlreaper_main_output = args.working_dir.joinpath( f"main-output-{secrets.token_urlsafe(15)}.tsv")#type: ignore[attr-defined] + _qtlreaper_permu_output = args.working_dir.joinpath( + f"permu-output-{secrets.token_urlsafe(15)}.tsv") logger.debug("Main output filename: %s", _qtlreaper_main_output) with subprocess.Popen( ("qtlreaper", "--n_permutations", "1000", "--geno", _genofile, "--traits", _traitsfile, - "--main_output", _qtlreaper_main_output)) as _qtlreaper: + "--main_output", _qtlreaper_main_output, + "--permu_output", _qtlreaper_permu_output, + "--threads", str(int(1+(multiprocessing.cpu_count()/2)))), + env=({**os.environ, "RUST_BACKTRACE": "full"} + if logger.getEffectiveLevel() == logging.DEBUG + else dict(os.environ))) as _qtlreaper: while _qtlreaper.poll() is None: logger.debug("QTLReaper process running…") time.sleep(1) - results = tuple(#type: ignore[var-annotated] - max(qtls, key=lambda qtl: qtl["LRS"]) - for qtls in - reduce(__qtls_by_trait__, - parse_tsv_file(_qtlreaper_main_output), - {}).values()) - save_qtl_values_to_db(conn, results) + results = ( + tuple(#type: ignore[var-annotated] + max(qtls, key=lambda qtl: qtl["LRS"]) + for qtls in + reduce(__qtls_by_trait__, + parse_tsv_file(_qtlreaper_main_output), + {}).values()) + if _qtlreaper_main_output.exists() + else tuple()) logger.debug("Cleaning up temporary files.") - _traitsfile.unlink() - _qtlreaper_main_output.unlink() + + # short-circuits to delete file if exists + if _traitsfile.exists(): + _traitsfile.unlink() + logger.info("Deleted generated traits' file for QTLReaper.") + + if _qtlreaper_main_output.exists(): + _qtlreaper_main_output.unlink() + logger.info("Deleted QTLReaper's main output file.") + + if _qtlreaper_permu_output.exists(): + _qtlreaper_permu_output.unlink() + logger.info("Deleted QTLReaper's permutations file.") + + if _qtlreaper.returncode != 0: + return _qtlreaper.returncode + + save_qtl_values_to_db(conn, results) logger.info("Successfully computed p values for %s traits.", len(_traitsdata)) return 0 except FileNotFoundError as fnf: - logger.error(", ".join(fnf.args), exc_info=False) + logger.error(", ".join(str(arg) for arg in fnf.args), exc_info=False) except AssertionError as aserr: logger.error(", ".join(aserr.args), exc_info=False) except Exception as _exc:# pylint: disable=[broad-exception-caught] diff --git a/tests/conftest.py b/tests/conftest.py index a716c52..2009aab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -183,7 +183,7 @@ def redis_conn_with_completed_job_some_errors(redis_url, redis_ttl, jobs_prefix, def uploads_dir(client): # pylint: disable=[redefined-outer-name] """Returns the configured, uploads directory, creating it if it does not exist.""" - the_dir = client.application.config["UPLOAD_FOLDER"] + the_dir = client.application.config["UPLOADS_DIRECTORY"] if not os.path.exists(the_dir): os.mkdir(the_dir) diff --git a/tests/test_instance_dir/config.py b/tests/test_instance_dir/config.py index 2ee569b..f04b3df 100644 --- a/tests/test_instance_dir/config.py +++ b/tests/test_instance_dir/config.py @@ -6,6 +6,6 @@ import os LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") SECRET_KEY = b"<Please! Please! Please! Change This!>" -UPLOAD_FOLDER = "/tmp/qc_app_files" +UPLOADS_DIRECTORY = "/tmp/qc_app_files" REDIS_URL = "redis://" JOBS_TTL_SECONDS = 600 # 10 minutes diff --git a/tests/uploader/test_parse.py b/tests/uploader/test_parse.py index 20c75b7..56e1b41 100644 --- a/tests/uploader/test_parse.py +++ b/tests/uploader/test_parse.py @@ -50,7 +50,7 @@ def test_parse_with_existing_uploaded_file( assert the_job["command"] == " ".join([ sys.executable, "-m", "scripts.validate_file", db_url, redis_url, jobs_prefix, job_id, "--redisexpiry", str(redis_ttl), str(speciesid), - filetype, f"{client.application.config['UPLOAD_FOLDER']}/{filename}"]) + filetype, f"{client.application.config['UPLOADS_DIRECTORY']}/{filename}"]) @pytest.mark.parametrize( "filename,uri,error_msgs", diff --git a/uploader/__init__.py b/uploader/__init__.py index 0ba1f81..afaa78d 100644 --- a/uploader/__init__.py +++ b/uploader/__init__.py @@ -73,6 +73,28 @@ def setup_modules_logging(app_logger, modules): _logger.setLevel(loglevel) +def __setup_scratch_directory__(app: Flask) -> Flask: + app.config["SCRATCH_DIRECTORY"] = Path( + app.config["SCRATCH_DIRECTORY"]).absolute() + return app + +def __setup_upload_directory__(app: Flask) -> Flask: + if app.config.get("UPLOADS_DIRECTORY", "").strip() == "": + app.config["UPLOADS_DIRECTORY"] = app.config[ + "SCRATCH_DIRECTORY"].joinpath("uploads") + else: + app.config["UPLOADS_DIRECTORY"] = Path( + app.config["UPLOADS_DIRECTORY"].strip()).absolute() + + return app + + +def update_unspecified_defaults(app: Flask) -> Flask: + """Setup the defaults for necessary configurations that do not have values + specified for them.""" + return __setup_upload_directory__(__setup_scratch_directory__(app)) + + def create_app(config: Optional[dict] = None): """The application factory. @@ -100,6 +122,7 @@ def create_app(config: Optional[dict] = None): # Silently ignore secrets if the file does not exist. app.config.from_pyfile(secretsfile) app.config.update(config) # Override everything with passed in config + update_unspecified_defaults(app) ### END: Application configuration app.config["SESSION_CACHELIB"] = FileSystemCache( @@ -108,12 +131,8 @@ def create_app(config: Optional[dict] = None): default_timeout=int(app.config["SESSION_FILESYSTEM_CACHE_TIMEOUT"])) setup_logging(app) - setup_modules_logging(app.logger, ( - "uploader.base_routes", - "uploader.flask_extensions", - "uploader.publications.models", - "uploader.publications.datatables", - "uploader.phenotypes.models")) + setup_modules_logging( + app.logger, tuple(app.config.get("LOGGABLE_MODULES", []))) # setup jinja2 symbols app.add_template_global(user_logged_in) diff --git a/uploader/background_jobs.py b/uploader/background_jobs.py index 2c55272..a71dd44 100644 --- a/uploader/background_jobs.py +++ b/uploader/background_jobs.py @@ -1,35 +1,51 @@ """Generic views and utilities to handle background jobs.""" import uuid +import datetime import importlib from typing import Callable from functools import partial from werkzeug.wrappers.response import Response from flask import ( + flash, + request, redirect, Blueprint, - render_template, current_app as app) from gn_libs import jobs from gn_libs import sqlite3 from gn_libs.jobs.jobs import JobNotFound -from uploader.flask_extensions import url_for +from uploader import session from uploader.authorisation import require_login +from uploader.flask_extensions import url_for, render_template background_jobs_bp = Blueprint("background-jobs", __name__) HandlerType = Callable[[dict], Response] -def __default_error_handler__(job: dict) -> Response: - return redirect(url_for("background-jobs.job_error", job_id=job["job_id"])) +def make_datetime_formatter(dtformat: str = "%A, %d %B %Y at %H:%M %Z") -> Callable[[str], str]: + """Make a datetime formatter with the provided `dtformat`""" + def __formatter__(val: str) -> str: + dt = datetime.datetime.fromisoformat(val) + return dt.strftime(dtformat.strip()) + + return __formatter__ + +__default_datetime_formatter__ = make_datetime_formatter() + + +def __default_handler__(_job): + return render_template("background-jobs/job-summary.html", + job=_job, + display_datetime=__default_datetime_formatter__) def register_handlers( job_type: str, success_handler: HandlerType, # pylint: disable=[redefined-outer-name] - error_handler: HandlerType = __default_error_handler__ + error_handler: HandlerType = __default_handler__ # pylint: disable=[redefined-outer-name] ) -> str: """Register success and error handlers for each job type.""" @@ -61,7 +77,7 @@ def register_job_handlers(job: dict): try: _error_handler = __load_handler__(metadata["error_handler"]) except Exception as _exc:# pylint: disable=[broad-exception-caught] - _error_handler = __default_error_handler__ + _error_handler = __default_handler__ register_handlers( metadata["job-type"], _success_handler, _error_handler) @@ -77,11 +93,7 @@ def handler(job: dict, handler_type: str) -> HandlerType: if bool(_handler): return _handler(job) - def __default_success_handler__(_job): - return render_template( - "background-jobs/default-success-page.html", job=_job) - - return __default_success_handler__ + return __default_handler__(job) error_handler = partial(handler, handler_type="error") @@ -98,13 +110,15 @@ def job_status(job_id: uuid.UUID): status = job["metadata"]["status"] register_job_handlers(job) - if status == "error": + if status in ("error", "stopped"): return error_handler(job) if status == "completed": return success_handler(job) - return render_template("jobs/job-status.html", job=job) + return render_template("background-jobs/job-status.html", + job=job, + display_datetime=__default_datetime_formatter__) except JobNotFound as _jnf: return render_template("jobs/job-not-found.html", job_id=job_id) @@ -119,3 +133,93 @@ def job_error(job_id: uuid.UUID): return render_template("jobs/job-error.html", job=job) except JobNotFound as _jnf: return render_template("jobs/job-not-found.html", job_id=job_id) + + +@background_jobs_bp.route("/list") +@require_login +def list_jobs(): + """List background jobs.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + return render_template( + "background-jobs/list-jobs.html", + jobs=jobs.jobs_by_external_id( + conn, session.user_details()["user_id"]), + display_datetime=__default_datetime_formatter__) + + +@background_jobs_bp.route("/summary/<uuid:job_id>") +@require_login +def job_summary(job_id: uuid.UUID): + """Provide a summary for completed jobs.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + try: + job = jobs.job(conn, job_id, fulldetails=True) + status = job["metadata"]["status"] + + if status in ("completed", "error", "stopped"): + return render_template("background-jobs/job-summary.html", + job=job, + display_datetime=__default_datetime_formatter__) + return redirect(url_for( + "background-jobs.job_status", job_id=job["job_id"])) + except JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) + + +@background_jobs_bp.route("/delete/<uuid:job_id>", methods=["GET", "POST"]) +@require_login +def delete_single(job_id: uuid.UUID): + """Delete a single job.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + try: + job = jobs.job(conn, job_id, fulldetails=True) + status = job["metadata"]["status"] + if status not in ("completed", "error", "stopped"): + flash("We cannot delete a running job.", "alert alert-danger") + return redirect(url_for( + "background-jobs.job_summary", job_id=job_id)) + + if request.method == "GET": + return render_template("background-jobs/delete-job.html", + job=job, + display_datetime=__default_datetime_formatter__) + + if request.form["btn-confirm-delete"] == "delete": + jobs.delete_job(conn, job_id) + flash("Job was deleted successfully.", "alert alert-success") + return redirect(url_for("background-jobs.list_jobs")) + flash("Delete cancelled.", "alert alert-info") + return redirect(url_for( + "background-jobs.job_summary", job_id=job_id)) + except JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) + + +@background_jobs_bp.route("/stop/<uuid:job_id>", methods=["GET", "POST"]) +@require_login +def stop_job(job_id: uuid.UUID): + """Stop a running job.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + try: + job = jobs.job(conn, job_id, fulldetails=True) + status = job["metadata"]["status"] + if status != "running": + flash("Cannot stop a job that is not running.", "alert alert-danger") + return redirect(url_for( + "background-jobs.job_summary", job_id=job_id)) + + if request.method == "GET": + return render_template("background-jobs/stop-job.html", + job=job, + display_datetime=__default_datetime_formatter__) + + if request.form["btn-confirm-stop"] == "stop": + jobs.kill_job(conn, job_id) + flash("Job was stopped successfully.", "alert alert-success") + return redirect(url_for( + "background-jobs.job_summary", job_id=job_id)) + flash("Stop cancelled.", "alert alert-info") + return redirect(url_for( + "background-jobs.job_summary", job_id=job_id)) + except JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) diff --git a/uploader/configutils.py b/uploader/configutils.py new file mode 100644 index 0000000..c5db50b --- /dev/null +++ b/uploader/configutils.py @@ -0,0 +1,13 @@ +"""Functions to fetch settings.""" +from pathlib import Path + +def fetch_setting(app, setting): + """Fetch a specified configuration `setting` from the `app` object.""" + return app.config[setting] + +def uploads_dir(app) -> Path: + """Fetch the uploads directory""" + _dir = Path(fetch_setting(app, "UPLOADS_DIRECTORY")).absolute() + assert _dir.exists() and _dir.is_dir(), ( + f"'{_dir}' needs to be an existing directory.") + return _dir diff --git a/uploader/default_settings.py b/uploader/default_settings.py index 52cdad5..04e1c0a 100644 --- a/uploader/default_settings.py +++ b/uploader/default_settings.py @@ -5,8 +5,14 @@ actual configuration file used for the production and staging systems. LOG_LEVEL = "WARNING" SECRET_KEY = b"<Please! Please! Please! Change This!>" -UPLOAD_FOLDER = "/tmp/qc_app_files" -TEMPORARY_DIRECTORY = "/tmp/gn-uploader-tmpdir" + +# Scratch directory and uploads: +# *** The scratch directory *** +# We avoid `/tmp` entirely for the scratch directory to avoid shared global +# mutable state with other users/applications/processes. +SCRATCH_DIRECTORY = "~/tmp/gn-uploader-scratchdir" +UPLOADS_DIRECTORY = ""# If not set, will be under scratch directory. + REDIS_URL = "redis://" JOBS_TTL_SECONDS = 1209600 # 14 days GNQC_REDIS_PREFIX="gn-uploader" @@ -33,3 +39,6 @@ JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before d ## --- Feature flags --- FEATURE_FLAGS_HTTP: list[str] = [] + +## --- Modules for which to log output --- +LOGGABLE_MODULES: list[str] = [] diff --git a/uploader/errors.py b/uploader/errors.py index 3e7c893..2ac48b8 100644 --- a/uploader/errors.py +++ b/uploader/errors.py @@ -3,7 +3,8 @@ import traceback from werkzeug.exceptions import HTTPException import MySQLdb as mdb -from flask import Flask, request, render_template, current_app as app +from flask import Flask, request, current_app as app +from uploader.flask_extensions import render_template def handle_general_exception(exc: Exception): """Handle generic exceptions.""" diff --git a/uploader/expression_data/dbinsert.py b/uploader/expression_data/dbinsert.py index 6d8ce80..7040698 100644 --- a/uploader/expression_data/dbinsert.py +++ b/uploader/expression_data/dbinsert.py @@ -94,7 +94,7 @@ def select_platform(): job = jobs.job(rconn, jobs.jobsnamespace(), job_id) if job: filename = job["filename"] - filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + filepath = f"{app.config['UPLOADS_DIRECTORY']}/{filename}" if os.path.exists(filepath): default_species = 1 gchips = genechips() @@ -367,7 +367,7 @@ def insert_data(): assert form.get("datasetid"), "dataset" filename = form["filename"] - filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + filepath = f"{app.config['UPLOADS_DIRECTORY']}/{filename}" redisurl = app.config["REDIS_URL"] if os.path.exists(filepath): with Redis.from_url(redisurl, decode_responses=True) as rconn: @@ -377,7 +377,7 @@ def insert_data(): form["species"], form["genechipid"], form["datasetid"], app.config["SQL_URI"], redisurl, app.config["JOBS_TTL_SECONDS"]), - redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors") + redisurl, f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return redirect(url_for("dbinsert.insert_status", job_id=job["jobid"])) return render_error(f"File '{filename}' no longer exists.") diff --git a/uploader/expression_data/views.py b/uploader/expression_data/views.py index 0b318b7..0e9b072 100644 --- a/uploader/expression_data/views.py +++ b/uploader/expression_data/views.py @@ -162,7 +162,7 @@ def upload_file(species_id: int, population_id: int): species=species, population=population) - upload_dir = app.config["UPLOAD_FOLDER"] + upload_dir = app.config["UPLOADS_DIRECTORY"] request_errors = errors(request) if request_errors: for error in request_errors: @@ -225,7 +225,7 @@ def parse_file(species_id: int, population_id: int): _errors = True if filename: - filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename) + filepath = os.path.join(app.config["UPLOADS_DIRECTORY"], filename) if not os.path.exists(filepath): flash("Selected file does not exist (any longer)", "alert-danger") _errors = True @@ -241,7 +241,7 @@ def parse_file(species_id: int, population_id: int): species_id, filepath, filetype,# type: ignore[arg-type] app.config["JOBS_TTL_SECONDS"]), redisurl, - f"{app.config['UPLOAD_FOLDER']}/job_errors") + f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return redirect(url_for("species.populations.expression-data.parse_status", species_id=species_id, @@ -263,7 +263,7 @@ def parse_status(species_id: int, population_id: int, job_id: str): return render_template("no_such_job.html", job_id=job_id), 400 error_filename = jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + job_id, f"{app.config['UPLOADS_DIRECTORY']}/job_errors") if os.path.exists(error_filename): stat = os.stat(error_filename) if stat.st_size > 0: @@ -345,7 +345,7 @@ def fail(species_id: int, population_id: int, job_id: str): if job: error_filename = jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + job_id, f"{app.config['UPLOADS_DIRECTORY']}/job_errors") if os.path.exists(error_filename): stat = os.stat(error_filename) if stat.st_size > 0: diff --git a/uploader/files/chunks.py b/uploader/files/chunks.py index c4360b5..f63f32f 100644 --- a/uploader/files/chunks.py +++ b/uploader/files/chunks.py @@ -5,6 +5,8 @@ from typing import Iterator from flask import current_app as app from werkzeug.utils import secure_filename +from uploader.configutils import uploads_dir + def chunked_binary_read(filepath: Path, chunksize: int = 2048) -> Iterator: """Read a file in binary mode in chunks.""" @@ -29,4 +31,4 @@ def chunks_directory(uniqueidentifier: str) -> Path: """Compute the directory where chunks are temporarily stored.""" if uniqueidentifier == "": raise ValueError("Unique identifier cannot be empty!") - return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}") + return Path(uploads_dir(app), f"tempdir_{uniqueidentifier}") diff --git a/uploader/files/functions.py b/uploader/files/functions.py index 7b9f06b..68f4e16 100644 --- a/uploader/files/functions.py +++ b/uploader/files/functions.py @@ -8,6 +8,8 @@ from flask import current_app from werkzeug.utils import secure_filename from werkzeug.datastructures import FileStorage +from uploader.configutils import uploads_dir + from .chunks import chunked_binary_read def save_file(fileobj: FileStorage, upload_dir: Path, hashed: bool = True) -> Path: @@ -30,7 +32,7 @@ def save_file(fileobj: FileStorage, upload_dir: Path, hashed: bool = True) -> Pa def fullpath(filename: str): """Get a file's full path. This makes use of `flask.current_app`.""" - return Path(current_app.config["UPLOAD_FOLDER"], filename).absolute() + return Path(uploads_dir(current_app), filename).absolute() def sha256_digest_over_file(filepath: Path) -> str: diff --git a/uploader/files/views.py b/uploader/files/views.py index 29059c7..ea0e827 100644 --- a/uploader/files/views.py +++ b/uploader/files/views.py @@ -6,13 +6,15 @@ from pathlib import Path from flask import request, jsonify, Blueprint, current_app as app +from uploader.configutils import uploads_dir + from .chunks import chunk_name, chunks_directory files = Blueprint("files", __name__) def target_file(fileid: str) -> Path: """Compute the full path for the target file.""" - return Path(app.config["UPLOAD_FOLDER"], fileid) + return Path(uploads_dir(app), fileid) @files.route("/upload/resumable", methods=["GET"]) diff --git a/uploader/flask_extensions.py b/uploader/flask_extensions.py index 30fbad7..0fc774a 100644 --- a/uploader/flask_extensions.py +++ b/uploader/flask_extensions.py @@ -2,19 +2,17 @@ import logging from typing import Any, Optional -from flask import (request, current_app as app, url_for as flask_url_for) +from flask import ( + request, + current_app as app, + url_for as flask_url_for, + render_template as flask_render_template) logger = logging.getLogger(__name__) -def url_for( - endpoint: str, - _anchor: Optional[str] = None, - _method: Optional[str] = None, - _scheme: Optional[str] = None, - _external: Optional[bool] = None, - **values: Any) -> str: - """Extension to flask's `url_for` function.""" +def fetch_flags(): + """Fetch get arguments that are defined as feature flags.""" flags = {} for flag in app.config["FEATURE_FLAGS_HTTP"]: flag_value = (request.args.get(flag) or request.form.get(flag) or "").strip() @@ -22,12 +20,33 @@ def url_for( flags[flag] = flag_value continue continue + logger.debug("HTTP FEATURE FLAGS: %s", flags) + return flags - logger.debug("HTTP FEATURE FLAGS: %s, other variables: %s", flags, values) + +def url_for( + endpoint: str, + _anchor: Optional[str] = None, + _method: Optional[str] = None, + _scheme: Optional[str] = None, + _external: Optional[bool] = None, + **values: Any) -> str: + """Extension to flask's `url_for` function.""" + logger.debug("other variables: %s", values) return flask_url_for(endpoint=endpoint, _anchor=_anchor, _method=_method, _scheme=_scheme, _external=_external, **values, - **flags) + **fetch_flags()) + + +def render_template(template_name_or_list, **context: Any) -> str: + """Extend flask's `render_template` function""" + return flask_render_template( + template_name_or_list, + **{ + **context, + **fetch_flags() # override any flag values + }) diff --git a/uploader/genotypes/models.py b/uploader/genotypes/models.py index 4c3e634..34d2cfe 100644 --- a/uploader/genotypes/models.py +++ b/uploader/genotypes/models.py @@ -31,16 +31,28 @@ def genotype_markers( species_id: int, offset: int = 0, limit: Optional[int] = None -) -> tuple[dict, ...]: +) -> tuple[tuple[dict, ...], int]: """Retrieve markers from the database.""" - _query = "SELECT * FROM Geno WHERE SpeciesId=%s" - if bool(limit) and limit > 0:# type: ignore[operator] - _query = _query + f" LIMIT {limit} OFFSET {offset}" + _query_template = ( + "SELECT %%COLS%% FROM Geno AS gno " + "WHERE gno.SpeciesId=%s " + "%%LIMIT%%") with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute(_query, (species_id,)) + cursor.execute( + _query_template.replace("%%LIMIT%%", "").replace( + "%%COLS%%", "COUNT(gno.Id) AS total_records"), + (species_id,)) + _total_records = cursor.fetchone()["total_records"] + cursor.execute( + _query_template.replace("%%COLS%%", "gno.*").replace( + "%%LIMIT%%", + (f"LIMIT {int(limit)} OFFSET {int(offset)}" + if bool(limit) and limit > 0 + else "")), + (species_id,)) debug_query(cursor, app.logger) - return tuple(dict(row) for row in cursor.fetchall()) + return tuple(dict(row) for row in cursor.fetchall()), _total_records def genotype_dataset( diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py index d991614..f27671c 100644 --- a/uploader/genotypes/views.py +++ b/uploader/genotypes/views.py @@ -1,8 +1,12 @@ """Views for the genotypes.""" +import logging + from MySQLdb.cursors import DictCursor +from pymonad.either import Left, Right, Either from gn_libs.mysqldb import database_connection from flask import (flash, request, + jsonify, redirect, Blueprint, render_template, @@ -16,8 +20,8 @@ from uploader.route_utils import generic_select_population from uploader.datautils import safe_int, enumerate_sequence from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler -from uploader.request_checks import with_species, with_population from uploader.population.models import population_by_species_and_id +from uploader.request_checks import with_species, with_dataset, with_population from .models import (genotype_markers, genotype_dataset, @@ -25,56 +29,17 @@ from .models import (genotype_markers, genotype_markers_count, genocode_by_population) +logger = logging.getLogger(__name__) genotypesbp = Blueprint("genotypes", __name__) render_template = make_template_renderer("genotypes") -@genotypesbp.route("populations/genotypes", methods=["GET"]) -@require_login -def index(): - """Direct entry-point for genotypes.""" - with database_connection(app.config["SQL_URI"]) as conn: - if not bool(request.args.get("species_id")): - return render_template("genotypes/index.html", - species=all_species(conn), - activelink="genotypes") - - species_id = request.args.get("species_id") - if species_id == "CREATE-SPECIES": - return redirect(url_for( - "species.create_species", - return_to="species.populations.genotypes.select_population")) - - species = species_by_id(conn, request.args.get("species_id")) - if not bool(species): - flash(f"Could not find species with ID '{request.args.get('species_id')}'!", - "alert-danger") - return redirect(url_for("species.populations.genotypes.index")) - return redirect(url_for("species.populations.genotypes.select_population", - species_id=species["SpeciesId"])) - - -@genotypesbp.route("/<int:species_id>/populations/genotypes/select-population", - methods=["GET"]) -@require_login -@with_species(redirect_uri="species.populations.genotypes.index") -def select_population(species: dict, species_id: int):# pylint: disable=[unused-argument] - """Select the population under which the genotypes go.""" - return generic_select_population( - species, - "genotypes/select-population.html", - request.args.get("population_id") or "", - "species.populations.genotypes.select_population", - "species.populations.genotypes.list_genotypes", - "genotypes", - "Invalid population selected!") - @genotypesbp.route( "/<int:species_id>/populations/<int:population_id>/genotypes", methods=["GET"]) @require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") +@with_population(species_redirect_uri="species.list_species", + redirect_uri="species.populations.list_species_populations") def list_genotypes(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """List genotype details for species and population.""" with database_connection(app.config["SQL_URI"]) as conn: @@ -92,34 +57,31 @@ def list_genotypes(species: dict, population: dict, **kwargs):# pylint: disable= @genotypesbp.route( - "/<int:species_id>/populations/<int:population_id>/genotypes/list-markers", + "/<int:species_id>/populations/<int:population_id>/genotypes/<int:dataset_id>/list-markers", methods=["GET"]) @require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") -def list_markers( - species: dict, - population: dict, - **kwargs -):# pylint: disable=[unused-argument] - """List a species' genetic markers.""" +@with_species(redirect_uri="species.populations.genotypes.list_genotypes") +def list_markers(species: dict, **_kwargs): + """List the markers that exist for this species.""" + args = request.args + offset = int(args.get("start") or 0) with database_connection(app.config["SQL_URI"]) as conn: - start_from = max(safe_int(request.args.get("start_from") or 0), 0) - count = safe_int(request.args.get("count") or 20) - return render_template("genotypes/list-markers.html", - species=species, - population=population, - total_markers=genotype_markers_count( - conn, species["SpeciesId"]), - start_from=start_from, - count=count, - markers=enumerate_sequence( - genotype_markers(conn, - species["SpeciesId"], - offset=start_from, - limit=count), - start=start_from+1), - activelink="list-markers") + markers, total_records = genotype_markers( + conn, + species["SpeciesId"], + offset=offset, + limit=int(args.get("length") or 0)) + return jsonify({ + **({"draw": int(args.get("draw"))} + if bool(args.get("draw") or False) + else {}), + "recordsTotal": total_records, + "recordsFiltered": len(markers), + "markers": tuple({**marker, "index": idx} + for idx, marker in + enumerate(markers, start=offset+1)) + }) + @genotypesbp.route( "/<int:species_id>/populations/<int:population_id>/genotypes/datasets/" @@ -132,14 +94,14 @@ def view_dataset(species_id: int, population_id: int, dataset_id: int): species = species_by_id(conn, species_id) if not bool(species): flash("Invalid species provided!", "alert-danger") - return redirect(url_for("species.populations.genotypes.index")) + return redirect(url_for("species.list_species")) population = population_by_species_and_id( conn, species_id, population_id) if not bool(population): flash("Invalid population selected!", "alert-danger") return redirect(url_for( - "species.populations.genotypes.select_population", + "species.populations.list_species_populations", species_id=species_id)) dataset = genotype_dataset(conn, species_id, population_id, dataset_id) @@ -162,25 +124,32 @@ def view_dataset(species_id: int, population_id: int, dataset_id: int): "create", methods=["GET", "POST"]) @require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") +@with_population(species_redirect_uri="species.list_species", + redirect_uri="species.populations.list_species_populations") def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """Create a genotype dataset.""" + if request.method == "GET": + return render_template("genotypes/create-dataset.html", + species=species, + population=population, + activelink="create-dataset") + with (database_connection(app.config["SQL_URI"]) as conn, conn.cursor(cursorclass=DictCursor) as cursor): - if request.method == "GET": - return render_template("genotypes/create-dataset.html", - species=species, - population=population, - activelink="create-dataset") - - form = request.form - new_dataset = save_new_dataset( - cursor, - population["Id"], - form["geno-dataset-name"], - form["geno-dataset-fullname"], - form["geno-dataset-shortname"]) + + def __save_dataset__() -> Either: + form = request.form + try: + return Right(save_new_dataset( + cursor, + population["Id"], + form["geno-dataset-name"], + form["geno-dataset-fullname"], + form["geno-dataset-shortname"])) + except Exception: + msg = "Error adding new Genotype dataset to database." + logger.error(msg, exc_info=True) + return Left(Exception(msg)) def __success__(_success): flash("Successfully created genotype dataset.", "alert-success") @@ -189,18 +158,20 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable= species_id=species["SpeciesId"], population_id=population["Id"])) - return oauth2_post( - "auth/resource/genotypes/create", - json={ - **dict(request.form), - "species_id": species["SpeciesId"], - "population_id": population["Id"], - "dataset_id": new_dataset["Id"], - "dataset_name": form["geno-dataset-name"], - "dataset_fullname": form["geno-dataset-fullname"], - "dataset_shortname": form["geno-dataset-shortname"], - "public": "on" - } + return __save_dataset__().then( + lambda new_dataset: oauth2_post( + "auth/resource/genotypes/create", + json={ + **dict(request.form), + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": new_dataset["Id"], + "dataset_name": new_dataset["Name"], + "dataset_fullname": new_dataset["FullName"], + "dataset_shortname": new_dataset["ShortName"], + "public": "on" + } + ) ).either( make_either_error_handler( "There was an error creating the genotype dataset."), diff --git a/uploader/oauth2/client.py b/uploader/oauth2/client.py index 4e81afd..e37816d 100644 --- a/uploader/oauth2/client.py +++ b/uploader/oauth2/client.py @@ -4,7 +4,7 @@ import time import uuid import random from datetime import datetime, timedelta -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin, urlparse, urlencode import requests from flask import request, current_app as app @@ -18,6 +18,7 @@ from authlib.integrations.requests_client import OAuth2Session from uploader import session import uploader.monadic_requests as mrequests +from uploader.flask_extensions import fetch_flags SCOPE = ("profile group role resource register-client user masquerade " "introspect migrate-data") @@ -176,11 +177,13 @@ def authserver_authorise_uri(): """Build up the authorisation URI.""" req_baseurl = urlparse(request.base_url, scheme=request.scheme) host_uri = f"{req_baseurl.scheme}://{req_baseurl.netloc}/" - return urljoin( - authserver_uri(), - "auth/authorise?response_type=code" - f"&client_id={oauth2_clientid()}" - f"&redirect_uri={urljoin(host_uri, 'oauth2/code')}") + args = { + "response_type": "code", + "client_id": oauth2_clientid(), + "redirect_uri": ( + f"{urljoin(host_uri, 'oauth2/code')}?{urlencode(fetch_flags())}") + } + return f"{urljoin(authserver_uri(), 'auth/authorise')}?{urlencode(args)}" def __no_token__(_err) -> Left: diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py index a22497c..3d656d2 100644 --- a/uploader/phenotypes/models.py +++ b/uploader/phenotypes/models.py @@ -1,4 +1,6 @@ """Database and utility functions for phenotypes.""" +import time +import random import logging import tempfile from pathlib import Path @@ -6,8 +8,8 @@ from functools import reduce from datetime import datetime from typing import Union, Optional, Iterable -import MySQLdb as mdb -from MySQLdb.cursors import Cursor, DictCursor +from MySQLdb.connections import Connection +from MySQLdb.cursors import Cursor, DictCursor, BaseCursor from gn_libs.mysqldb import debug_query @@ -27,7 +29,7 @@ __PHENO_DATA_TABLES__ = { def datasets_by_population( - conn: mdb.Connection, + conn: Connection, species_id: int, population_id: int ) -> tuple[dict, ...]: @@ -42,7 +44,7 @@ def datasets_by_population( return tuple(dict(row) for row in cursor.fetchall()) -def dataset_by_id(conn: mdb.Connection, +def dataset_by_id(conn: Connection, species_id: int, population_id: int, dataset_id: int) -> dict: @@ -57,7 +59,7 @@ def dataset_by_id(conn: mdb.Connection, return dict(cursor.fetchone()) -def phenotypes_count(conn: mdb.Connection, +def phenotypes_count(conn: Connection, population_id: int, dataset_id: int) -> int: """Count the number of phenotypes in the dataset.""" @@ -85,27 +87,46 @@ def phenotype_publication_data(conn, phenotype_id) -> Optional[dict]: return dict(res) -def dataset_phenotypes(conn: mdb.Connection, - population_id: int, - dataset_id: int, - offset: int = 0, - limit: Optional[int] = None) -> tuple[dict, ...]: +def dataset_phenotypes(# pylint: disable=[too-many-arguments, too-many-positional-arguments] + conn: Connection, + population_id: int, + dataset_id: int, + offset: int = 0, + limit: Optional[int] = None, + xref_ids: tuple[int, ...] = tuple() +) -> tuple[dict, ...]: """Fetch the actual phenotypes.""" - _query = ( - "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, ist.InbredSetCode " + _narrow_by_ids = ( + f" AND pxr.Id IN ({', '.join(['%s'] * len(xref_ids))})" + if len(xref_ids) > 0 else "") + _narrow_by_limit = ( + f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + _pub_query = ( + "SELECT pub.* " + "FROM PublishXRef AS pxr " + "INNER JOIN Publication AS pub ON pxr.PublicationId=pub.Id " + "WHERE pxr.InbredSetId=%s") + _narrow_by_ids + _pheno_query = (( + "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, pxr.PublicationId, " + "ist.InbredSetCode " "FROM Phenotype AS pheno " "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " "INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId " "INNER JOIN InbredSet AS ist ON pf.InbredSetId=ist.Id " - "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + ( - f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + + _narrow_by_ids + + _narrow_by_limit) with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute(_query, (population_id, dataset_id)) + cursor.execute(_pub_query, (population_id,) + xref_ids) debug_query(cursor, logger) - return tuple(dict(row) for row in cursor.fetchall()) + _pubs = {row["Id"]: dict(row) for row in cursor.fetchall()} + cursor.execute(_pheno_query, (population_id, dataset_id) + xref_ids) + debug_query(cursor, logger) + return tuple({**dict(row), "publication": _pubs[row["PublicationId"]]} + for row in cursor.fetchall()) -def __phenotype_se__(cursor: Cursor, xref_id, dataids_and_strainids): +def __phenotype_se__(cursor: BaseCursor, xref_id, dataids_and_strainids): """Fetch standard-error values (if they exist) for a phenotype.""" paramstr = ", ".join(["(%s, %s)"] * len(dataids_and_strainids)) flat = tuple(item for sublist in dataids_and_strainids for item in sublist) @@ -187,7 +208,7 @@ def __merge_pheno_data_and_se__(data, sedata) -> dict: def phenotype_by_id( - conn: mdb.Connection, + conn: Connection, species_id: int, population_id: int, dataset_id: int, @@ -225,7 +246,7 @@ def phenotype_by_id( return None -def phenotypes_data(conn: mdb.Connection, +def phenotypes_data(conn: Connection, population_id: int, dataset_id: int, offset: int = 0, @@ -249,7 +270,7 @@ def phenotypes_data(conn: mdb.Connection, def phenotypes_vector_data(# pylint: disable=[too-many-arguments, too-many-positional-arguments] - conn: mdb.Connection, + conn: Connection, species_id: int, population_id: int, xref_ids: tuple[int, ...] = tuple(), @@ -301,7 +322,7 @@ def phenotypes_vector_data(# pylint: disable=[too-many-arguments, too-many-posit return reduce(__organise__, cursor.fetchall(), {}) -def save_new_dataset(cursor: Cursor, +def save_new_dataset(cursor: BaseCursor, population_id: int, dataset_name: str, dataset_fullname: str, @@ -346,7 +367,7 @@ def __pre_process_phenotype_data__(row): def create_new_phenotypes(# pylint: disable=[too-many-locals] - conn: mdb.Connection, + conn: Connection, population_id: int, publication_id: int, phenotypes: Iterable[dict] @@ -474,7 +495,7 @@ def create_new_phenotypes(# pylint: disable=[too-many-locals] def save_phenotypes_data( - conn: mdb.Connection, + conn: Connection, table: str, data: Iterable[dict] ) -> int: @@ -504,7 +525,7 @@ def save_phenotypes_data( def quick_save_phenotypes_data( - conn: mdb.Connection, + conn: Connection, table: str, dataitems: Iterable[dict], tmpdir: Path @@ -534,3 +555,134 @@ def quick_save_phenotypes_data( ")") debug_query(cursor, logger) return _count + + +def __sleep_random__(): + """Sleep a random amount of time chosen from 0.05s to 1s in increments of 0.05""" + time.sleep(random.choice(tuple(i / 20.0 for i in range(1, 21)))) + + +def delete_phenotypes_data( + cursor: BaseCursor, + data_ids: tuple[int, ...] +) -> tuple[int, int, int]: + """Delete numeric data for phenotypes with the given data IDs.""" + if len(data_ids) == 0: + return (0, 0, 0) + + # Loop to handle big deletes i.e. ≥ 10000 rows + _dcount, _secount, _ncount = (0, 0, 0)# Count total rows deleted + while True: + _paramstr = ", ".join(["%s"] * len(data_ids)) + cursor.execute( + "DELETE FROM PublishData " + f"WHERE Id IN ({_paramstr}) " + "ORDER BY Id ASC, StrainId ASC "# Make deletions deterministic + "LIMIT 1000", + data_ids) + _dcount_curr = cursor.rowcount + _dcount += _dcount_curr + + cursor.execute( + "DELETE FROM PublishSE " + f"WHERE DataId IN ({_paramstr}) " + "ORDER BY DataId ASC, StrainId ASC "# Make deletions deterministic + "LIMIT 1000", + data_ids) + _secount_curr = cursor.rowcount + _secount += _secount_curr + + cursor.execute( + "DELETE FROM NStrain " + f"WHERE DataId IN ({_paramstr}) " + "ORDER BY DataId ASC, StrainId ASC "# Make deletions deterministic + "LIMIT 1000", + data_ids) + _ncount_curr = cursor.rowcount + _ncount += _ncount_curr + __sleep_random__() + + if all((_dcount_curr == 0, _secount_curr == 0, _ncount_curr == 0)): + # end loop if there are no more rows to delete. + break + + return (_dcount, _secount, _ncount) + + +def __linked_ids__( + cursor: BaseCursor, + population_id: int, + xref_ids: tuple[int, ...] +) -> tuple[tuple[int, int, int], ...]: + """Retrieve `DataId` values from `PublishXRef` table.""" + _paramstr = ", ".join(["%s"] * len(xref_ids)) + cursor.execute("SELECT PhenotypeId, PublicationId, DataId " + "FROM PublishXRef " + f"WHERE InbredSetId=%s AND Id IN ({_paramstr})", + (population_id,) + xref_ids) + return tuple( + (int(row["PhenotypeId"]), int(row["PublicationId"]), int(row["DataId"])) + for row in cursor.fetchall()) + + +def delete_phenotypes( + conn_or_cursor: Union[Connection, Cursor], + population_id: int, + xref_ids: tuple[int, ...] +) -> tuple[int, int, int, int]: + """Delete phenotypes and all their data.""" + def __delete_phenos__(cursor: BaseCursor, pheno_ids: tuple[int, ...]) -> int: + """Delete data from the `Phenotype` table.""" + _paramstr = ", ".join(["%s"] * len(pheno_ids)) + + _pcount = 0 + while True: + cursor.execute( + "DELETE FROM Phenotype " + f"WHERE Id IN ({_paramstr}) " + "ORDER BY Id " + "LIMIT 1000", + pheno_ids) + _pcount_curr = cursor.rowcount + _pcount += _pcount_curr + __sleep_random__() + if _pcount_curr == 0: + break + + return cursor.rowcount + + def __delete_xrefs__(cursor: BaseCursor) -> int: + _paramstr = ", ".join(["%s"] * len(xref_ids)) + + _xcount = 0 + while True: + cursor.execute( + "DELETE FROM PublishXRef " + f"WHERE InbredSetId=%s AND Id IN ({_paramstr}) " + "ORDER BY Id " + "LIMIT 10000", + (population_id,) + xref_ids) + _xcount_curr = cursor.rowcount + _xcount += _xcount_curr + __sleep_random__() + if _xcount_curr == 0: + break + + return _xcount + + def __with_cursor__(cursor): + _phenoids, _pubids, _dataids = reduce( + lambda acc, curr: (acc[0] + (curr[0],), + acc[1] + (curr[1],), + acc[2] + (curr[2],)), + __linked_ids__(cursor, population_id, xref_ids), + (tuple(), tuple(), tuple())) + __delete_phenos__(cursor, _phenoids) + return (__delete_xrefs__(cursor),) + delete_phenotypes_data( + cursor, _dataids) + + if isinstance(conn_or_cursor, BaseCursor): + return __with_cursor__(conn_or_cursor) + + with conn_or_cursor.cursor(cursorclass=DictCursor) as cursor: + return __with_cursor__(cursor) diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py index c4c6170..c03f3f5 100644 --- a/uploader/phenotypes/views.py +++ b/uploader/phenotypes/views.py @@ -1,4 +1,6 @@ """Views handling ('classical') phenotypes."""# pylint: disable=[too-many-lines] +import io +import csv import sys import uuid import json @@ -21,12 +23,14 @@ from gn_libs import jobs as gnlibs_jobs from gn_libs.jobs.jobs import JobNotFound from gn_libs.mysqldb import database_connection +from werkzeug.datastructures import Headers from flask import (flash, request, jsonify, redirect, Blueprint, - current_app as app) + current_app as app, + Response as FlaskResponse) from r_qtl import r_qtl2_qc as rqc from r_qtl import exceptions as rqe @@ -34,6 +38,7 @@ from r_qtl import exceptions as rqe from uploader import jobs from uploader import session from uploader.files import save_file +from uploader.configutils import uploads_dir from uploader.flask_extensions import url_for from uploader.ui import make_template_renderer from uploader.oauth2.client import oauth2_post @@ -312,6 +317,11 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable= dataset_shortname = ( form["dataset-shortname"] or form["dataset-name"]).strip() _pheno_dataset = save_new_dataset( + # It's not necessary to update the authorisation server to register + # new phenotype resource here, since each phenotype trait can, in + # theory, have its own access control allowing/disallowing access to + # it. In practice, however, we tend to gather multiple traits into a + # single resource for access control. cursor, population["Id"], form["dataset-name"].strip(), @@ -329,7 +339,7 @@ def process_phenotypes_rqtl2_bundle(error_uri): try: ## Handle huge files here... phenobundle = save_file(request.files["phenotypes-bundle"], - Path(app.config["UPLOAD_FOLDER"])) + uploads_dir(app)) rqc.validate_bundle(phenobundle) return phenobundle except AssertionError as _aerr: @@ -352,7 +362,7 @@ def process_phenotypes_individual_files(error_uri): "comment.char": form["file-comment-character"], "na.strings": form["file-na"].split(" "), } - bundlepath = Path(app.config["UPLOAD_FOLDER"], + bundlepath = Path(uploads_dir(app), f"{str(uuid.uuid4()).replace('-', '')}.zip") with ZipFile(bundlepath,mode="w") as zfile: for rqtlkey, formkey, _type in ( @@ -370,7 +380,7 @@ def process_phenotypes_individual_files(error_uri): # Chunked upload of large files was used filedata = json.loads(form[formkey]) zfile.write( - Path(app.config["UPLOAD_FOLDER"], filedata["uploaded-file"]), + Path(uploads_dir(app), filedata["uploaded-file"]), arcname=filedata["original-name"]) cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filedata["original-name"]] else: @@ -382,9 +392,9 @@ def process_phenotypes_individual_files(error_uri): return error_uri filepath = save_file( - _sentfile, Path(app.config["UPLOAD_FOLDER"]), hashed=False) + _sentfile, uploads_dir(app), hashed=False) zfile.write( - Path(app.config["UPLOAD_FOLDER"], filepath), + Path(uploads_dir(app), filepath), arcname=filepath.name) cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filepath.name] @@ -464,7 +474,7 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p **({"publicationid": request.form["publication-id"]} if request.form.get("publication-id") else {})})}), _redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") + f"{uploads_dir(app)}/job_errors") app.logger.debug("JOB DETAILS: %s", _job) jobstatusuri = url_for("species.populations.phenotypes.job_status", @@ -521,6 +531,65 @@ def job_status( @phenotypesbp.route( "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/job/<uuid:job_id>/download-errors", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def download_errors( + species: dict, + population: dict, + dataset: dict, + job_id: uuid.UUID, + **kwargs):# pylint: disable=[unused-argument] + """Download the list of errors as a CSV file.""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id)) + _prefix_ = jobs.jobsnamespace() + _jobid_ = job['jobid'] + def __generate_chunks__(): + _errors_ = ( + json.loads(error) + for key in rconn.keys( + f"{_prefix_}:{str(_jobid_)}:*:errors:*") + for error in rconn.lrange(key, 0, -1)) + _chunk_no_ = 0 + _all_errors_printed_ = False + while not _all_errors_printed_: + _chunk_ = [] + try: + for _ in range(0, 1000): + _chunk_.append(next(_errors_)) + except StopIteration: + _all_errors_printed_ = True + if len(_chunk_) <= 0: + raise + + _out_ = io.StringIO() + _writer_ = csv.DictWriter(_out_, fieldnames=tuple(_chunk_[0].keys())) + if _chunk_no_ == 0: + _writer_.writeheader() + _writer_.writerows(_chunk_) + _chunk_no_ += 1 + yield _out_.getvalue() + if _all_errors_printed_: + return + + headers = Headers() + headers.set("Content-Disposition", + "attachment", + filename=f"{job['job-type']}_{job['jobid']}.csv") + return FlaskResponse( + __generate_chunks__(), mimetype="text/csv", headers=headers) + except jobs.JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" "/<int:dataset_id>/job/<uuid:job_id>/review", methods=["GET"]) @require_login @@ -598,6 +667,8 @@ def review_job_data( conn, int(_job_metadata["publicationid"])) if _job_metadata.get("publicationid") else None), + user=session.user_details(), + timestamp=datetime.datetime.now().isoformat(), activelink="add-phenotypes") @@ -611,6 +682,12 @@ def load_phenotypes_success_handler(job): job_id=job["job_id"])) +def proceed_to_job_status(job): + """A generic 'job success' handler for asynchronous phenotype jobs.""" + app.logger.debug("The new job: %s", job) + return redirect(url_for("background-jobs.job_status", job_id=job["job_id"])) + + @phenotypesbp.route( "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" "/<int:dataset_id>/load-data-to-database", @@ -653,11 +730,6 @@ def load_data_to_database( def __handle_error__(resp): return render_template("http-error.html", *resp.json()) - def __handle_success__(load_job): - app.logger.debug("The phenotypes loading job: %s", load_job) - return redirect(url_for( - "background-jobs.job_status", job_id=load_job["job_id"])) - return request_token( token_uri=urljoin(oauth2client.authserver_uri(), "auth/token"), @@ -676,18 +748,25 @@ def load_data_to_database( "publication_id": _meta["publicationid"], "authserver": oauth2client.authserver_uri(), "token": token["access_token"], + "dataname": request.form["data_name"].strip(), "success_handler": ( "uploader.phenotypes.views" - ".load_phenotypes_success_handler") - }) + ".load_phenotypes_success_handler"), + **{ + key: request.form[key] + for key in ("data_description",) + if key in request.form.keys() + } + }, + external_id=session.logged_in_user_id()) ).then( lambda job: gnlibs_jobs.launch_job( job, _jobs_db, - Path(f"{app.config['UPLOAD_FOLDER']}/job_errors"), + Path(f"{uploads_dir(app)}/job_errors"), worker_manager="gn_libs.jobs.launcher", loglevel=_loglevel) - ).either(__handle_error__, __handle_success__) + ).either(__handle_error__, proceed_to_job_status) def update_phenotype_metadata(conn, metadata: dict): @@ -1059,9 +1138,10 @@ def recompute_means(# pylint: disable=[unused-argument] "success_handler": ( "uploader.phenotypes.views." "recompute_phenotype_means_success_handler") - }), + }, + external_id=session.logged_in_user_id()), _jobs_db, - Path(f"{app.config['UPLOAD_FOLDER']}/job_errors"), + Path(f"{uploads_dir(app)}/job_errors"), worker_manager="gn_libs.jobs.launcher", loglevel=_loglevel) return redirect(url_for("background-jobs.job_status", @@ -1103,7 +1183,7 @@ def rerun_qtlreaper(# pylint: disable=[unused-argument] _job_id = uuid.uuid4() _loglevel = logging.getLevelName(app.logger.getEffectiveLevel()).lower() - _workingdir = Path(app.config["TEMPORARY_DIRECTORY"]).joinpath("qtlreaper") + _workingdir = Path(app.config["SCRATCH_DIRECTORY"]).joinpath("qtlreaper") _workingdir.mkdir(exist_ok=True) command = [ sys.executable, @@ -1138,9 +1218,10 @@ def rerun_qtlreaper(# pylint: disable=[unused-argument] "success_handler": ( "uploader.phenotypes.views." "rerun_qtlreaper_success_handler") - }), + }, + external_id=session.logged_in_user_id()), _jobs_db, - Path(f"{app.config['UPLOAD_FOLDER']}/job_errors"), + Path(f"{uploads_dir(app)}/job_errors"), worker_manager="gn_libs.jobs.launcher", loglevel=_loglevel) return redirect(url_for("background-jobs.job_status", @@ -1152,3 +1233,119 @@ def rerun_qtlreaper(# pylint: disable=[unused-argument] def rerun_qtlreaper_success_handler(job): """Handle success (re)running QTLReaper script.""" return return_to_dataset_view_handler(job, "QTLReaper ran successfully!") + + +def delete_phenotypes_success_handler(job): + """Handle success running the 'delete-phenotypes' script.""" + return return_to_dataset_view_handler( + job, "Phenotypes deleted successfully.") + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/delete", + methods=["GET", "POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def delete_phenotypes(# pylint: disable=[unused-argument, too-many-locals] + species: dict, + population: dict, + dataset: dict, + **kwargs +): + """Delete the specified phenotype data.""" + _dataset_page = redirect(url_for( + "species.populations.phenotypes.view_dataset", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"])) + + def __handle_error__(resp): + flash( + "Error retrieving authorisation token. Phenotype deletion " + "failed. Please try again later.", + "alert alert-danger") + return _dataset_page + + _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + with (database_connection(app.config["SQL_URI"]) as conn, + sqlite3.connection(_jobs_db) as jobsconn): + form = request.form + xref_ids = tuple(int(item) for item in set(form.getlist("xref_ids"))) + + match form.get("action"): + case "cancel": + return redirect(url_for( + "species.populations.phenotypes.view_dataset", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"])) + case "delete": + _loglevel = logging.getLevelName( + app.logger.getEffectiveLevel()).lower() + if form.get("confirm_delete_all_phenotypes", "") == "on": + _cmd = ["--delete-all"] + else: + # setup phenotypes xref_ids file + _xref_ids_file = Path( + app.config["SCRATCH_DIRECTORY"], + f"delete-phenotypes-{uuid.uuid4()}.txt") + with _xref_ids_file.open(mode="w", encoding="utf8") as ptr: + ptr.write("\n".join(str(_id) for _id in xref_ids)) + + _cmd = ["--xref_ids_file", str(_xref_ids_file)] + + _job_id = uuid.uuid4() + return request_token( + token_uri=urljoin( + oauth2client.authserver_uri(), "auth/token"), + user_id=session.user_details()["user_id"] + ).then( + lambda token: gnlibs_jobs.initialise_job( + jobsconn, + _job_id, + [ + sys.executable, + "-u", + "-m", + "scripts.phenotypes.delete_phenotypes", + "--log-level", _loglevel, + app.config["SQL_URI"], + str(species["SpeciesId"]), + str(population["Id"]), + str(dataset["Id"]), + app.config["AUTH_SERVER_URL"], + token["access_token"]] + _cmd, + "delete-phenotypes", + extra_meta={ + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": dataset["Id"], + "success_handler": ( + "uploader.phenotypes.views." + "delete_phenotypes_success_handler") + }, + external_id=session.logged_in_user_id()) + ).then( + lambda _job: gnlibs_jobs.launch_job( + _job, + _jobs_db, + Path(f"{uploads_dir(app)}/job_errors"), + worker_manager="gn_libs.jobs.launcher", + loglevel=_loglevel) + ).either(__handle_error__, proceed_to_job_status) + case _: + _phenos: tuple[dict, ...] = tuple() + if len(xref_ids) > 0: + _phenos = dataset_phenotypes( + conn, population["Id"], dataset["Id"], xref_ids=xref_ids) + + return render_template( + "phenotypes/confirm-delete-phenotypes.html", + species=species, + population=population, + dataset=dataset, + phenotypes=_phenos) diff --git a/uploader/population/rqtl2.py b/uploader/population/rqtl2.py index 97d4854..bb5066e 100644 --- a/uploader/population/rqtl2.py +++ b/uploader/population/rqtl2.py @@ -134,7 +134,7 @@ def upload_rqtl2_bundle(species_id: int, population_id: int): try: app.logger.debug("Files in the form: %s", request.files) the_file = save_file(request.files["rqtl2_bundle_file"], - Path(app.config["UPLOAD_FOLDER"])) + Path(app.config["UPLOADS_DIRECTORY"])) except AssertionError: app.logger.debug(traceback.format_exc()) flash("Please provide a valid R/qtl2 zip bundle.", @@ -185,7 +185,7 @@ def trigger_rqtl2_bundle_qc( "rqtl2-bundle-file": str(rqtl2bundle.absolute()), "original-filename": originalfilename})}), redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") + f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return jobid @@ -895,7 +895,7 @@ def confirm_bundle_details(species_id: int, population_id: int): }) }), redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") + f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return redirect(url_for("expression-data.rqtl2.rqtl2_processing_status", jobid=jobid)) diff --git a/uploader/publications/models.py b/uploader/publications/models.py index dcfa02b..d913144 100644 --- a/uploader/publications/models.py +++ b/uploader/publications/models.py @@ -101,6 +101,20 @@ def fetch_publication_by_id(conn: Connection, publication_id: int) -> dict: return dict(_res) if _res else {} +def fetch_publications_by_ids( + conn: Connection, publications_ids: tuple[int, ...] +) -> tuple[dict, ...]: + """Fetch publications with the given IDs.""" + if len(publications_ids) == 0: + return tuple() + + with conn.cursor(cursorclass=DictCursor) as cursor: + paramstr = ", ".join(["%s"] * len(publications_ids)) + cursor.execute(f"SELECT * FROM Publication WHERE Id IN ({paramstr})", + tuple(publications_ids)) + return tuple(dict(row) for row in cursor.fetchall()) + + def fetch_publication_phenotypes( conn: Connection, publication_id: int) -> Iterable[dict]: """Fetch all phenotypes linked to this publication.""" diff --git a/uploader/publications/views.py b/uploader/publications/views.py index d9eb294..89e9f5d 100644 --- a/uploader/publications/views.py +++ b/uploader/publications/views.py @@ -1,5 +1,6 @@ """Endpoints for publications""" import json +import datetime from gn_libs.mysqldb import database_connection from flask import ( @@ -89,9 +90,12 @@ def create_publication(): } if request.method == "GET": + now = datetime.datetime.now() return render_template( "publications/create-publication.html", - get_args=_get_args) + get_args=_get_args, + current_year=now.year, + current_month=now.strftime("%B")) form = request.form authors = form.get("publication-authors").encode("utf8") if authors is None or authors == "": diff --git a/uploader/request_checks.py b/uploader/request_checks.py index f1d8027..84935f9 100644 --- a/uploader/request_checks.py +++ b/uploader/request_checks.py @@ -2,14 +2,20 @@ These are useful for reusability, and hence maintainability of the code. """ +import logging + +from typing import Callable from functools import wraps -from gn_libs.mysqldb import database_connection +from gn_libs.mysqldb import Connection, database_connection from flask import flash, url_for, redirect, current_app as app from uploader.species.models import species_by_id from uploader.population.models import population_by_species_and_id +logger = logging.getLogger(__name__) + + def with_species(redirect_uri: str): """Ensure the species actually exists.""" def __decorator__(function): @@ -28,7 +34,7 @@ def with_species(redirect_uri: str): "alert-danger") return redirect(url_for(redirect_uri)) except ValueError as _verr: - app.logger.debug( + logger.debug( "Exception converting value to integer: %s", kwargs.get("species_id"), exc_info=True) @@ -63,7 +69,7 @@ def with_population(species_redirect_uri: str, redirect_uri: str): "alert-danger") return select_population_uri except ValueError as _verr: - app.logger.debug( + logger.debug( "Exception converting value to integer: %s", kwargs.get("population_id"), exc_info=True) @@ -73,3 +79,45 @@ def with_population(species_redirect_uri: str, redirect_uri: str): return function(**{**kwargs, "population": population}) return __with_population__ return __decorator__ + + +def with_dataset( + species_redirect_uri: str, + population_redirect_uri: str, + redirect_uri: str, + dataset_by_id: Callable[ + [Connection, int, int, int], + dict] +): + """Ensure the dataset actually exists.""" + def __decorator__(func): + @wraps(func) + @with_population(species_redirect_uri, population_redirect_uri) + def __with_dataset__(**kwargs): + try: + _spcid = int(kwargs["species_id"]) + _popid = int(kwargs["population_id"]) + _dsetid = int(kwargs.get("dataset_id")) + select_dataset_uri = redirect(url_for( + redirect_uri, species_id=_spcid, population_id=_popid)) + if not bool(_dsetid): + flash("You need to select a valid 'dataset_id' value.", + "alert-danger") + return select_dataset_uri + with database_connection(app.config["SQL_URI"]) as conn: + dataset = dataset_by_id(conn, _spcid, _popid, _dsetid) + if not bool(dataset): + flash("You must select a valid dataset.", + "alert-danger") + return select_dataset_uri + except ValueError as _verr: + logger.debug( + "Exception converting 'dataset_id' to integer: %s", + kwargs.get("dataset_id"), + exc_info=True) + flash("Expected 'dataset_id' value to be an integer." + "alert-danger") + return select_dataset_uri + return func(**{**kwargs, "dataset": dataset}) + return __with_dataset__ + return __decorator__ diff --git a/uploader/samples/views.py b/uploader/samples/views.py index f8baf7e..2a09f8e 100644 --- a/uploader/samples/views.py +++ b/uploader/samples/views.py @@ -1,17 +1,19 @@ """Code regarding samples""" -import os import sys import uuid +import logging from pathlib import Path -from redis import Redis from flask import (flash, request, redirect, Blueprint, current_app as app) -from uploader import jobs +from gn_libs import jobs +from gn_libs import sqlite3 + +from uploader import session from uploader.files import save_file from uploader.flask_extensions import url_for from uploader.ui import make_template_renderer @@ -23,8 +25,7 @@ from uploader.datautils import safe_int, enumerate_sequence from uploader.species.models import all_species, species_by_id from uploader.request_checks import with_species, with_population from uploader.db_utils import (with_db_connection, - database_connection, - with_redis_connection) + database_connection) from .models import samples_by_species_and_population @@ -96,22 +97,6 @@ def list_samples(species: dict, population: dict, **kwargs):# pylint: disable=[u activelink="list-samples") -def build_sample_upload_job(# pylint: disable=[too-many-arguments, too-many-positional-arguments] - speciesid: int, - populationid: int, - samplesfile: Path, - separator: str, - firstlineheading: bool, - quotechar: str): - """Define the async command to run the actual samples data upload.""" - return [ - sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"], - str(speciesid), str(populationid), str(samplesfile.absolute()), - separator, f"--redisuri={app.config['REDIS_URL']}", - f"--quotechar={quotechar}" - ] + (["--firstlineheading"] if firstlineheading else []) - - @samplesbp.route("<int:species_id>/populations/<int:population_id>/upload-samples", methods=["GET", "POST"]) @require_login @@ -153,7 +138,7 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma try: samples_file = save_file(request.files["samples_file"], - Path(app.config["UPLOAD_FOLDER"])) + Path(app.config["UPLOADS_DIRECTORY"])) except AssertionError: flash("You need to provide a file with the samples data.", "alert-error") @@ -170,102 +155,50 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma quotechar = (request.form.get("field_delimiter", '"') or '"') - redisuri = app.config["REDIS_URL"] - with Redis.from_url(redisuri, decode_responses=True) as rconn: - #T0DO: Add a QC step here — what do we check? - # 1. Does any sample in the uploaded file exist within the database? - # If yes, what is/are its/their species and population? - # 2. If yes 1. above, provide error with notes on which species and - # populations already own the samples. - the_job = jobs.launch_job( + _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + with sqlite3.connection(_jobs_db) as conn: + job = jobs.launch_job( jobs.initialise_job( - rconn, - jobs.jobsnamespace(), + conn, str(uuid.uuid4()), - build_sample_upload_job( - species["SpeciesId"], - population["InbredSetId"], - samples_file, + [ + sys.executable, "-m", "scripts.insert_samples", + app.config["SQL_URI"], + str(species["SpeciesId"]), + str(population["InbredSetId"]), + str(samples_file.absolute()), separator, - firstlineheading, - quotechar), + f"--quotechar={quotechar}" + ] + (["--firstlineheading"] if firstlineheading else []), "samples_upload", - app.config["JOBS_TTL_SECONDS"], - {"job_name": f"Samples Upload: {samples_file.name}"}), - redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") - return redirect(url_for( - "species.populations.samples.upload_status", - species_id=species_id, - population_id=population_id, - job_id=the_job["jobid"])) - - -@samplesbp.route("<int:species_id>/populations/<int:population_id>/" - "upload-samples/status/<uuid:job_id>", - methods=["GET"]) -@require_login -@with_population(species_redirect_uri="species.populations.samples.index", - redirect_uri="species.populations.samples.select_population") -def upload_status(species: dict, population: dict, job_id: uuid.UUID, **kwargs):# pylint: disable=[unused-argument] - """Check on the status of a samples upload job.""" - job = with_redis_connection(lambda rconn: jobs.job( - rconn, jobs.jobsnamespace(), job_id)) - if job: - status = job["status"] - if status == "success": - return render_template("samples/upload-success.html", - job=job, - species=species, - population=population,) - - if status == "error": - return redirect(url_for( - "species.populations.samples.upload_failure", - species_id=species["SpeciesId"], - population_id=population["Id"], - job_id=job_id)) - - error_filename = Path(jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) - if error_filename.exists(): - stat = os.stat(error_filename) - if stat.st_size > 0: - return redirect(url_for( - "samples.upload_failure", job_id=job_id)) - - return render_template("samples/upload-progress.html", - species=species, - population=population, - job=job) # maybe also handle this? - - return render_template("no_such_job.html", - job_id=job_id, - species=species, - population=population), 400 - - -@samplesbp.route("<int:species_id>/populations/<int:population_id>/" - "upload-samples/failure/<uuid:job_id>", - methods=["GET"]) -@require_login -@with_population(species_redirect_uri="species.populations.samples.index", - redirect_uri="species.populations.samples.select_population") -def upload_failure(species: dict, population: dict, job_id: uuid.UUID, **kwargs):# pylint: disable=[unused-argument] - """Display the errors of the samples upload failure.""" - job = with_redis_connection(lambda rconn: jobs.job( - rconn, jobs.jobsnamespace(), job_id)) - if not bool(job): - return render_template("no_such_job.html", job_id=job_id), 400 - - error_filename = Path(jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) - if error_filename.exists(): - stat = os.stat(error_filename) - if stat.st_size > 0: - return render_template("worker_failure.html", job_id=job_id) - - return render_template("samples/upload-failure.html", - species=species, - population=population, - job=job) + extra_meta={ + "job_name": f"Samples Upload: {samples_file.name}", + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "success_handler": ( + "uploader.samples.views.samples_upload_success_handler") + }, + external_id=session.logged_in_user_id()), + _jobs_db, + Path(f"{app.config['UPLOADS_DIRECTORY']}/job_errors").absolute(), + loglevel=logging.getLevelName( + app.logger.getEffectiveLevel()).lower()) + return redirect( + url_for("background-jobs.job_status", job_id=job["job_id"])) + + +def samples_upload_success_handler(job): + """Handler for background jobs: Successful upload of samples""" + return return_to_samples_list_view_handler( + job, "Samples uploaded successfully.") + + +def return_to_samples_list_view_handler(job, msg): + """Handler for background jobs: Return to list_samples page.""" + flash(msg, "alert alert-success") + return redirect(url_for( + "species.populations.samples." + "list_samples", + species_id=job["metadata"]["species_id"], + population_id=job["metadata"]["population_id"], + job_id=job["job_id"])) diff --git a/uploader/session.py b/uploader/session.py index 5af5827..9872ceb 100644 --- a/uploader/session.py +++ b/uploader/session.py @@ -1,12 +1,15 @@ """Deal with user sessions""" +import logging from uuid import UUID, uuid4 from datetime import datetime from typing import Any, Optional, TypedDict +from flask import session from authlib.jose import KeySet -from flask import request, session from pymonad.either import Left, Right, Either +logger = logging.getLogger(__name__) + class UserDetails(TypedDict): """Session information relating specifically to the user.""" @@ -22,8 +25,6 @@ class SessionInfo(TypedDict): session_id: UUID user: UserDetails anon_id: UUID - user_agent: str - ip_addr: str masquerade: Optional[UserDetails] auth_server_jwks: Optional[dict[str, Any]] @@ -66,9 +67,6 @@ def session_info() -> SessionInfo: "logged_in": False }, "anon_id": anon_id, - "user_agent": request.headers.get("User-Agent"), - "ip_addr": request.environ.get("HTTP_X_FORWARDED_FOR", - request.remote_addr), "masquerading": None })) @@ -91,6 +89,17 @@ def user_details() -> UserDetails: """Retrieve user details.""" return session_info()["user"] + +def logged_in_user_id() -> Optional[UUID]: + """Get user id for logged in user. If user has not logged in, return None.""" + return user_token().then( + lambda _tok: user_details() + ).then( + lambda _user: Either(_user["user_id"], + (None, _user["email"] != "anon@ymous.user")) + ).either(lambda _err: None, lambda uid: uid) + + def user_token() -> Either: """Retrieve the user token.""" return session_info()["user"]["token"] diff --git a/uploader/static/css/layout-common.css b/uploader/static/css/layout-common.css index 36a5735..9c9d034 100644 --- a/uploader/static/css/layout-common.css +++ b/uploader/static/css/layout-common.css @@ -1,3 +1,21 @@ * { box-sizing: border-box; } + +body { + display: grid; + grid-gap: 1em; +} + +#header { + margin: -0.7em; /* Fill entire length of screen */ + /* Define layout for the children elements */ + display: grid; +} + +#header #header-nav { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; + display: flex; +} diff --git a/uploader/static/css/layout-large.css b/uploader/static/css/layout-large.css index 8abd2dd..2d53627 100644 --- a/uploader/static/css/layout-large.css +++ b/uploader/static/css/layout-large.css @@ -1,8 +1,6 @@ @media screen and (min-width: 20.1in) { body { - display: grid; grid-template-columns: 7fr 3fr; - grid-gap: 1em; } #header { @@ -12,7 +10,7 @@ /* Define layout for the children elements */ display: grid; - grid-template-columns: 8fr 2fr; + grid-template-columns: 1fr 9fr; } #header #header-text { @@ -45,6 +43,8 @@ grid-column-start: 1; grid-column-end: 3; padding: 0 3px; + + margin: -0.3em -0.7em 0 -0.7em; } #main #main-content { diff --git a/uploader/static/css/layout-medium.css b/uploader/static/css/layout-medium.css index 2cca711..50ceeb4 100644 --- a/uploader/static/css/layout-medium.css +++ b/uploader/static/css/layout-medium.css @@ -1,8 +1,6 @@ @media screen and (width > 8in) and (max-width: 20in) { body { - display: grid; grid-template-columns: 65fr 35fr; - grid-gap: 1em; } #header { @@ -12,7 +10,7 @@ /* Define layout for the children elements */ display: grid; - grid-template-columns: 8fr 2fr; + grid-template-columns: 2fr 8fr; } #header #header-text { @@ -51,7 +49,6 @@ /* Place it in the parent element */ grid-column-start: 1; grid-column-end: 2; - grid-gap: 5px; /* Define layout for the children elements */ max-width: 100%; diff --git a/uploader/static/css/layout-small.css b/uploader/static/css/layout-small.css index 80a3759..2e47217 100644 --- a/uploader/static/css/layout-small.css +++ b/uploader/static/css/layout-small.css @@ -2,7 +2,7 @@ body { display: grid; grid-template-columns: 1fr; - grid-template-rows: 1fr 2fr 7fr; + grid-template-rows: 1fr 90fr; grid-gap: 1em; } @@ -31,6 +31,11 @@ grid-column-end: 2; } + #header #header-nav ul { + display: grid; + grid-template-columns: 1fr; + } + #main { /* Place it in the parent element */ grid-column-start: 1; @@ -38,7 +43,7 @@ display: grid; /* Define layout for the children elements */ - grid-template-rows: 1.5em 80% 20%; + grid-template-rows: 1fr 80fr 20fr; grid-template-columns: 1fr; } diff --git a/uploader/static/css/theme.css b/uploader/static/css/theme.css index cd3d103..6f5cb0c 100644 --- a/uploader/static/css/theme.css +++ b/uploader/static/css/theme.css @@ -8,24 +8,27 @@ body { #header { background-color: #336699; color: #FFFFFF; - border-radius: 3px; min-height: 30px; + border-bottom: solid black 1px; } #header #header-nav .nav li a { /* Content styling */ color: #FFFFFF; - background: #4477AA; - border: solid 5px #336699; - border-radius: 5px; font-size: 0.7em; text-align: center; padding: 1px 7px; + text-decoration: none; } #main #breadcrumbs { - border-radius:3px; text-align: center; + background-color: #D5D5D5; + padding: 0 1em 0 1em; +} + +#main #breadcrumbs .breadcrumb { + padding-top: 0.5em; } #main #main-content { @@ -78,13 +81,22 @@ table.dataTable tbody tr.selected td { background-color: #ffee99 !important; } -.form-group { +#frm-add-phenotypes .form-group { margin-bottom: 2em; padding-bottom: 0.2em; - border-bottom: solid gray 1px; + border-bottom: solid #A9A9A9 1px; } .breadcrumb-item { text-transform: Capitalize; } + +.breadcrumb-item a { + text-decoration: none; +} + +.table thead tr th { + text-align: center; + vertical-align: middle; +} diff --git a/uploader/static/images/frontpage_banner.png b/uploader/static/images/frontpage_banner.png new file mode 100644 index 0000000..d25e1c9 --- /dev/null +++ b/uploader/static/images/frontpage_banner.png Binary files differdiff --git a/uploader/static/js/datatables.js b/uploader/static/js/datatables.js index 82fd696..bfcda2a 100644 --- a/uploader/static/js/datatables.js +++ b/uploader/static/js/datatables.js @@ -11,13 +11,36 @@ var addTableLength = (menuList, lengthToAdd, dataLength) => { var defaultLengthMenu = (data) => { menuList = [] - var lengths = [10, 25, 50, 100, 1000, data.length]; + var lengths = [10, 25, 50, 100, 1000]; + if(data.length > 1000) { + lengths.push(data.length) + } lengths.forEach((len) => { menuList = addTableLength(menuList, len, data.length); }); return menuList; }; +var setRowCheckableProperty = (node, state) => { + /** + * Set a row's (`node`) checkbox's or radio button's checked state to the + * boolean value `state`. + **/ + if(typeof(state) == "boolean") { + var pseudoclass = state == false ? ":checked" : ":not(:checked)"; + var checkable = ( + $(node).find(`input[type="checkbox"]${pseudoclass}`)[0] + || + $(node).find(`input[type="radio"]${pseudoclass}`)[0]); + $(checkable).prop("checked", state); + } else { + throw new Error("`state` *MUST* be a boolean value.") + } +}; + +var setRowChecked = (node) => {setRowCheckableProperty(node, true);}; +var setRowUnchecked = (node) => {setRowCheckableProperty(node, false);}; + var buildDataTable = (tableId, data = [], columns = [], userSettings = {}) => { var defaultSettings = { responsive: true, @@ -35,35 +58,40 @@ var buildDataTable = (tableId, data = [], columns = [], userSettings = {}) => { lengthMenu: "", info: "" }, - data: data, - columns: columns, - drawCallback: (settings) => { - $(this[0]).find("tbody tr").each((idx, row) => { - var arow = $(row); - var checkboxOrRadio = arow.find(".chk-row-select"); - if (checkboxOrRadio) { - if (arow.hasClass("selected")) { - checkboxOrRadio.prop("checked", true); - } else { - checkboxOrRadio.prop("checked", false); - } - } + drawCallback: function (settings) { + var api = this.api(); + api.rows({selected: true}).nodes().each((node, index) => { + setRowChecked(node); + }); + api.rows({selected: false}).nodes().each((node, index) => { + setRowUnchecked(node); }); } } var theDataTable = $(tableId).DataTable({ ...defaultSettings, - ...userSettings + ...userSettings, + ...(data.length == 0 ? {} : {data: data}), + ...(columns.length == 0 ? {} : {columns: columns}) }); - theDataTable.on("select", (event, datatable, type, cell, originalEvent) => { - datatable.rows({selected: true}).nodes().each((node, index) => { - $(node).find(".chk-row-select").prop("checked", true) - }); + theDataTable.on("select", (event, datatable, type, indexes) => { + datatable + .rows(indexes) + .nodes() + .each((node, index) => { + setRowChecked(node); + }); }); - theDataTable.on("deselect", (event, datatable, type, cell, originalEvent) => { - datatable.rows({selected: false}).nodes().each((node, index) => { - $(node).find(".chk-row-select").prop("checked", false) - }); + theDataTable.on("deselect", (event, datatable, type, indexes) => { + datatable + .rows(indexes) + .nodes() + .each(function(node, index) { + setRowUnchecked(node); + }); }); + + theDataTable.selectAll = () => {theDataTable.rows().select()}; + theDataTable.deselectAll = () => {theDataTable.rows().deselect()}; return theDataTable; }; diff --git a/uploader/static/js/utils.js b/uploader/static/js/utils.js index 1b31661..62d3662 100644 --- a/uploader/static/js/utils.js +++ b/uploader/static/js/utils.js @@ -28,7 +28,8 @@ var remove_class = (element, classvalue) => { var add_class = (element, classvalue) => { remove_class(element, classvalue); - element.attr("class", (element.attr("class") || "") + " " + classvalue); + element.attr("class", + ((element.attr("class") || "") + " " + classvalue).trim()); }; $(".not-implemented").click((event) => { diff --git a/uploader/templates/background-jobs/base.html b/uploader/templates/background-jobs/base.html new file mode 100644 index 0000000..7201207 --- /dev/null +++ b/uploader/templates/background-jobs/base.html @@ -0,0 +1,10 @@ +{%extends "base.html"%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('background-jobs.list_jobs')}}"> + background jobs + </a> +</li> +{%endblock%} diff --git a/uploader/templates/background-jobs/default-success-page.html b/uploader/templates/background-jobs/default-success-page.html deleted file mode 100644 index 5732456..0000000 --- a/uploader/templates/background-jobs/default-success-page.html +++ /dev/null @@ -1,17 +0,0 @@ -{%extends "phenotypes/base.html"%} -{%from "flash_messages.html" import flash_all_messages%} - -{%block title%}Background Jobs: Success{%endblock%} - -{%block pagetitle%}Background Jobs: Success{%endblock%} - -{%block contents%} -{{flash_all_messages()}} - -<div class="row"> - <p>Job <strong>{{job.job_id}}</strong>, - {%if job.get("metadata", {}).get("job-type")%} - of type '<em>{{job.metadata["job-type"]}}</em> - {%endif%}' completed successfully.</p> -</div> -{%endblock%} diff --git a/uploader/templates/background-jobs/delete-job.html b/uploader/templates/background-jobs/delete-job.html new file mode 100644 index 0000000..242c775 --- /dev/null +++ b/uploader/templates/background-jobs/delete-job.html @@ -0,0 +1,61 @@ +{%extends "background-jobs/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "background-jobs/macro-display-job-details.html" import display_job_details%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('background-jobs.job_summary', job_id=job.job_id)}}"> + summary + </a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2 class="heading">background jobs: delete?</h2> + + <p class="text-danger">Are you sure you want to delete the job below?</p> + + {{display_job_details(job, display_datetime)}} +</div> + +<div class="row"> + <form id="frm-delete-job" + method="POST" + action="{{url_for('background-jobs.delete_single', job_id=job.job_id)}}"> + <div class="row"> + <div class="col"> + <input type="submit" + class="btn btn-info" + value="cancel" + name="btn-confirm-delete" /> + </div> + <div class="col"> + <input type="submit" + class="btn btn-danger" + value="delete" + name="btn-confirm-delete" /> + </div> + </div> + </form> +</div> +{%endblock%} + + +{%block sidebarcontents%} +<div class="row"> + <h6 class="subheading">What is this?</h6> +</div> +<div class="row"> + <p>Confirm whether or not you want to delete job + <strong>{{job.job_id}}</strong>.</p> +</div> +{{super()}} +{%endblock%} diff --git a/uploader/templates/background-jobs/job-status.html b/uploader/templates/background-jobs/job-status.html new file mode 100644 index 0000000..2e75c6d --- /dev/null +++ b/uploader/templates/background-jobs/job-status.html @@ -0,0 +1,45 @@ +{%extends "background-jobs/base.html"%} +{%from "background-jobs/macro-display-job-details.html" import display_job_details%} + +{%from "flash_messages.html" import flash_all_messages%} + +{%block extrameta%} +<meta http-equiv="refresh" content="5" /> +{%endblock%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2 class="heading">job status</h2> + + {{display_job_details(job, display_datetime)}} +</div> + +<div class="row"> + <div class="col"> + <a href="{{url_for('background-jobs.stop_job', job_id=job.job_id)}}" + title="Stop/Kill this job." + class="btn btn-danger">stop job</a> + </div> +</div> + +<div class="row"> + <h3 class="subheading">STDOUT</h3> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stdout"]}}</pre> + </div> +</div> + +<div class="row"> + <h3 class="subheading">STDERR</h3> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stderr"]}}</pre> + </div> +</div> + +{%endblock%} diff --git a/uploader/templates/background-jobs/job-summary.html b/uploader/templates/background-jobs/job-summary.html new file mode 100644 index 0000000..ef9ef6c --- /dev/null +++ b/uploader/templates/background-jobs/job-summary.html @@ -0,0 +1,75 @@ +{%extends "background-jobs/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "background-jobs/macro-display-job-details.html" import display_job_details%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('background-jobs.job_summary', job_id=job.job_id)}}"> + summary + </a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2 class="heading">background jobs: summary</h2> + + {{display_job_details(job, display_datetime)}} +</div> + +<div class="row"> + {%if view_under_construction%} + <div class="col"> + <a href="#" + class="btn btn-info not-implemented" + title="Update the expiry date and time for this job.">update expiry</a> + </div> + + {%if job.metadata.status in ("stopped",)%} + <div class="col"> + <a href="#" + class="btn btn-warning not-implemented" + title="Create a new copy of this job, and run the copy.">Run Copy</a> + </div> + {%endif%} + {%endif%} + + <div class="col"> + <a href="{{url_for('background-jobs.delete_single', job_id=job.job_id)}}" + class="btn btn-danger" + title="Delete this job.">delete</a> + </div> +</div> + +<div class="row"> + <h3 class="subheading">Script Errors and Logging</h3> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stderr"]}}</pre> + </div> +</div> + +<div class="row"> + <h3 class="subheading">Script Output</h3> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stdout"]}}</pre> + </div> +</div> +{%endblock%} + + +{%block sidebarcontents%} +<div class="row"> + <h6 class="subheading">What is this?</h6> +</div> +<div class="row"> + <p>This page shows the results of running job '{{job.job_id}}'.</p> +</div> +{{super()}} +{%endblock%} diff --git a/uploader/templates/background-jobs/list-jobs.html b/uploader/templates/background-jobs/list-jobs.html new file mode 100644 index 0000000..c16b850 --- /dev/null +++ b/uploader/templates/background-jobs/list-jobs.html @@ -0,0 +1,79 @@ +{%extends "background-jobs/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"><h2 class="heading">Background Jobs</h2></div> + +<div class="row"> + <div class="table-responsive"> + <table class="table"> + <thead> + <tr class="table-primary"> + <th>Type</th> + <th>Created</th> + <th title="Date and time past which the job's details will be deleted from the system."> + Expires</th> + <th>Status</th> + <th>Actions</th> + </tr> + </thead> + + <tbody> + {%for job in jobs%} + <tr> + <td>{{job.metadata["job-type"]}}</td> + <td>{{display_datetime(job.created)}}</td> + <td title="Date and time past which the job's details will be deleted from the system."> + {{display_datetime(job.expires)}} + </td> + <td {%if job.metadata.status == "completed"%} + class="fw-bold text-capitalize text-success" + {%elif job.metadata.status == "error"%} + class="fw-bold text-capitalize text-danger" + {%elif job.metadata.status == "stopped"%} + class="fw-bold text-capitalize text-warning" + {%else%} + class="fw-bold text-capitalize text-info" + {%endif%}> + <div> + {{job.metadata.status}} + </div> + </td> + <td> + <a href="{{url_for('background-jobs.job_summary', job_id=job.job_id)}}" + class="btn btn-info" + title="View more detailed information about this job."> + view summary</a> + </td> + </tr> + {%else%} + <tr> + <td colspan="5"> + You do not have any jobs you have run in the background.</td> + </tr> + {%endfor%} + </tbody> + </table> + </div> +</div> +{%endblock%} + + +{%block sidebarcontents%} +<div class="row"> + <h6 class="subheading">What is this?</h6> +</div> +<div class="row"> + <p>The table lists the jobs that are running in the background, that you + started.</p> + <p>You can use the tools provided on this page to manage the jobs, and to view + each job's details.</p> +</div> +{{super()}} +{%endblock%} diff --git a/uploader/templates/background-jobs/macro-display-job-details.html b/uploader/templates/background-jobs/macro-display-job-details.html new file mode 100644 index 0000000..82e33c0 --- /dev/null +++ b/uploader/templates/background-jobs/macro-display-job-details.html @@ -0,0 +1,29 @@ +{%macro display_job_details(job, display_datetime)%} +<table class="table"> + <thead> + </thead> + + <tbody> + <tr> + <th class="table-primary">Job ID</th> + <td>{{job.job_id}}</td> + </tr> + <tr> + <th class="table-primary">Type</th> + <td>{{job.metadata["job-type"]}}</td> + </tr> + <tr> + <th class="table-primary">Created</th> + <td>{{display_datetime(job.created)}}</td> + </tr> + <tr> + <th class="table-primary">Expires</th> + <td>{{display_datetime(job.expires)}}</td> + </tr> + <tr> + <th class="table-primary">Status</th> + <td>{{job.metadata.status}}</td> + </tr> + </tbody> +</table> +{%endmacro%} diff --git a/uploader/templates/background-jobs/stop-job.html b/uploader/templates/background-jobs/stop-job.html new file mode 100644 index 0000000..fc190ac --- /dev/null +++ b/uploader/templates/background-jobs/stop-job.html @@ -0,0 +1,61 @@ +{%extends "background-jobs/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "background-jobs/macro-display-job-details.html" import display_job_details%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('background-jobs.job_summary', job_id=job.job_id)}}"> + summary + </a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2 class="heading">background jobs: stop?</h2> + + <p class="text-danger">Are you sure you want to stop the job below?</p> + + {{display_job_details(job, display_datetime)}} +</div> + +<div class="row"> + <form id="frm-stop-job" + method="POST" + action="{{url_for('background-jobs.stop_job', job_id=job.job_id)}}"> + <div class="row"> + <div class="col"> + <input type="submit" + class="btn btn-info" + value="cancel" + name="btn-confirm-stop" /> + </div> + <div class="col"> + <input type="submit" + class="btn btn-danger" + value="stop" + name="btn-confirm-stop" /> + </div> + </div> + </form> +</div> +{%endblock%} + + +{%block sidebarcontents%} +<div class="row"> + <h6 class="subheading">What is this?</h6> +</div> +<div class="row"> + <p>Confirm whether or not you want to stop job + <strong>{{job.job_id}}</strong>.</p> +</div> +{{super()}} +{%endblock%} diff --git a/uploader/templates/base.html b/uploader/templates/base.html index 719a646..ae4ecef 100644 --- a/uploader/templates/base.html +++ b/uploader/templates/base.html @@ -30,14 +30,32 @@ <header id="header"> <span id="header-text">GeneNetwork</span> <nav id="header-nav"> - <ul class="nav justify-content-end"> + <ul class="nav"> + {%if user_logged_in()%} + <li> + <a href="{{url_for('background-jobs.list_jobs')}}" + title="User's background jobs."> + <!-- https://icons.getbootstrap.com/icons/back/ --> + <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-back" viewBox="0 0 16 16"> + <path d="M0 2a2 2 0 0 1 2-2h8a2 2 0 0 1 2 2v2h2a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2v-2H2a2 2 0 0 1-2-2zm2-1a1 1 0 0 0-1 1v8a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1V2a1 1 0 0 0-1-1z"/> + </svg> + Background jobs + </a> + </li> + <li> - {%if user_logged_in()%} <a href="{{url_for('oauth2.logout')}}" title="Log out of the system"> + <!-- https://icons.getbootstrap.com/icons/file-person/ --> + <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-file-person" viewBox="0 0 16 16"> + <path d="M12 1a1 1 0 0 1 1 1v10.755S12 11 8 11s-5 1.755-5 1.755V2a1 1 0 0 1 1-1zM4 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V2a2 2 0 0 0-2-2z"/> + <path d="M8 10a3 3 0 1 0 0-6 3 3 0 0 0 0 6"/> + </svg> <span class="glyphicon glyphicon-user"></span> - {{user_email()}} Sign Out</a> - {%else%} + Sign Out ({{user_email()}})</a> + </li> + {%else%} + <li> <a href="{{authserver_authorise_uri()}}" title="Log in to the system">Sign In</a> {%endif%} diff --git a/uploader/templates/genotypes/create-dataset.html b/uploader/templates/genotypes/create-dataset.html index 10331c1..ff174fb 100644 --- a/uploader/templates/genotypes/create-dataset.html +++ b/uploader/templates/genotypes/create-dataset.html @@ -35,13 +35,15 @@ id="txt-geno-dataset-name" name="geno-dataset-name" required="required" - class="form-control" /> + class="form-control" + value="{{population.Name}}Geno" + readonly="readonly" /> <small class="form-text text-muted"> <p>This is a short representative, but constrained name for the genotype dataset.<br /> - The field will only accept letters ('A-Za-z'), numbers (0-9), hyphens - and underscores. Any other character will cause the name to be - rejected.</p></small> + It is used internally by the Genenetwork system. Do not change this + value.</p> + </small> </div> <div class="form-group"> @@ -50,7 +52,8 @@ id="txt-geno-dataset-fullname" name="geno-dataset-fullname" required="required" - class="form-control" /> + class="form-control" + value="{{population.Name}} Genotypes" /> <small class="form-text text-muted"> <p>This is a longer, more descriptive name for your dataset.</p></small> </div> @@ -61,7 +64,8 @@ <input type="text" id="txt-geno-dataset-shortname" name="geno-dataset-shortname" - class="form-control" /> + class="form-control" + value="{{population.Name}}Geno" /> <small class="form-text text-muted"> <p>A short name for your dataset. If you leave this field blank, the short name will be set to the same value as the diff --git a/uploader/templates/genotypes/index.html b/uploader/templates/genotypes/index.html deleted file mode 100644 index b50ebc5..0000000 --- a/uploader/templates/genotypes/index.html +++ /dev/null @@ -1,32 +0,0 @@ -{%extends "genotypes/base.html"%} -{%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-select-species.html" import select_species_form%} - -{%block title%}Genotypes{%endblock%} - -{%block pagetitle%}Genotypes{%endblock%} - - -{%block contents%} -{{flash_all_messages()}} - -<div class="row"> - <p> - This section allows you to upload genotype information for your experiments, - in the case that you have not previously done so. - </p> - <p> - We'll need to link the genotypes to the species and population, so do please - go ahead and select those in the next two steps. - </p> -</div> - -<div class="row"> - {{select_species_form(url_for("species.populations.genotypes.index"), - species)}} -</div> -{%endblock%} - -{%block javascript%} -<script type="text/javascript" src="/static/js/species.js"></script> -{%endblock%} diff --git a/uploader/templates/genotypes/list-genotypes.html b/uploader/templates/genotypes/list-genotypes.html index a2b98c8..131576f 100644 --- a/uploader/templates/genotypes/list-genotypes.html +++ b/uploader/templates/genotypes/list-genotypes.html @@ -9,19 +9,6 @@ {{flash_all_messages()}} <div class="row"> - <h2>Genetic Markers</h2> - <p>There are a total of {{total_markers}} currently registered genetic markers - for the "{{species.FullName}}" species. You can click - <a href="{{url_for('species.populations.genotypes.list_markers', - species_id=species.SpeciesId, - population_id=population.Id)}}" - title="View genetic markers for species '{{species.FullName}}"> - this link to view the genetic markers - </a>. - </p> -</div> - -<div class="row"> <h2>Genotype Encoding</h2> <p> The genotype encoding used for the "{{population.FullName}}" population from @@ -56,59 +43,101 @@ </table> {%if genocode | length < 1%} - <a href="#add-genotype-encoding" - title="Add a genotype encoding system for this population" - class="btn btn-primary not-implemented"> - add genotype encoding + <div class="col"> + <a href="#add-genotype-encoding" + title="Add a genotype encoding system for this population" + class="btn btn-primary not-implemented"> + define genotype encoding </a> + </div> {%endif%} </div> -<div class="row text-danger"> - <h3>Some Important Concepts to Consider/Remember</h3> - <ul> - <li>Reference vs. Non-reference alleles</li> - <li>In <em>GenoCode</em> table, items are ordered by <strong>InbredSet</strong></li> - </ul> - <h3>Possible references</h3> - <ul> - <li>https://mr-dictionary.mrcieu.ac.uk/term/genotype/</li> - <li>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7363099/</li> - </ul> +<div class="row"> + <h2>Genotype Dataset</h2> +</div> + +{%if dataset is not none%} + +<div class="row"> + <h3>Dataset Details</h3> + <table class="table"> + <thead> + <tr> + <th>Name</th> + <th>Full Name</th> + </tr> + </thead> + + <tbody> + <tr> + <td>{{dataset.Name}}</td> + <td><a href="{{url_for('species.populations.genotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + title="View details regarding and manage dataset '{{dataset.FullName}}'" + target="_blank"> + {{dataset.FullName}}</a></td> + </tr> + </tbody> + </table> + + <p> + To see more information regarding this dataset (e.g. which markers have + sample allele data, the allele data itself, etc) click on the "Full Name" + link above.</p> +</div> + +<div class="row"> + <h3>Genotype Markers</h3> + + <div class="row"> + <p> + The table below lists all of the markers that exist for species + {{species.SpeciesName}} ({{species.FullName}}), regardless of whether + (or not) we have corresponding sample allele data for a particular marker. + </p> + <table id="tbl-genetic-markers" class="table compact stripe cell-border"> + <thead> + <tr> + <th title="">#</th> + <th title="">Index</th> + <th title="">Marker Name</th> + <th title="Chromosome">Chr</th> + <th title="Physical location of the marker in megabasepairs"> + Location (Mb)</th> + <th title="">Source</th> + <th title="">Source2</th> + </thead> + + <tbody> + {%for marker in markers%} + <tr> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + <td></td> + </tr> + {%endfor%} + </tbody> + </table> </div> +{%else%} + <div class="row"> - <h2>Genotype Datasets</h2> - - <p>The genotype data is organised under various genotype datasets. You can - click on the link for the relevant dataset to view a little more information - about it.</p> - - {%if dataset is not none%} - <table class="table"> - <thead> - <tr> - <th>Name</th> - <th>Full Name</th> - </tr> - </thead> - - <tbody> - <tr> - <td>{{dataset.Name}}</td> - <td><a href="{{url_for('species.populations.genotypes.view_dataset', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id)}}" - title="View details regarding and manage dataset '{{dataset.FullName}}'"> - {{dataset.FullName}}</a></td> - </tr> - </tbody> - </table> - {%else%} + <p> + Your genotype data will need to be under a dataset. Unfortunately there is + currently no dataset defined for this population. + </p> + <p class="text-warning"> <span class="glyphicon glyphicon-exclamation-sign"></span> - There is no genotype dataset defined for this population. + Click the button below to define the genotype dataset for this population. </p> <p> <a href="{{url_for('species.populations.genotypes.create_dataset', @@ -117,16 +146,81 @@ title="Create a new genotype dataset for the '{{population.FullName}}' population for the '{{species.FullName}}' species." class="btn btn-primary"> create new genotype dataset</a></p> - {%endif%} </div> -<div class="row text-warning"> - <p> - <span class="glyphicon glyphicon-exclamation-sign"></span> - <strong>NOTE</strong>: Currently the GN2 (and related) system(s) expect a - single genotype dataset. If there is more than one, the system apparently - fails in unpredictable ways. - </p> - <p>Fix this to allow multiple datasets, each with a different assembly from - all the rest.</p> + +{%endif%} + +<div class="row"> + <h2>Notes</h2> + <div class="row text-danger"> + <h3>Genetic Markers: Some Important Concepts to Consider/Remember</h3> + <ul> + <li>Reference vs. Non-reference alleles</li> + <li>In <em>GenoCode</em> table, items are ordered by <strong>InbredSet</strong></li> + </ul> + <h3>Possible references</h3> + <ul> + <li>https://mr-dictionary.mrcieu.ac.uk/term/genotype/</li> + <li>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7363099/</li> + </ul> + </div> + + <div class="row text-warning"> + <h3>Genotype Dataset</h3> + <p> + <span class="glyphicon glyphicon-exclamation-sign"></span> + <strong>NOTE</strong>: Currently the GN2 (and related) system(s) expect a + single genotype dataset per population. If there is more than one, the + system apparently fails in unpredictable ways. + </p> + </div> </div> + +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + + $(function() { + var dtGeneticMarkers = buildDataTable( + "#tbl-genetic-markers", + [], + [ + { + data: function(marker) { + return `<input type="checkbox" name="selected-markers" ` + + `id="chk-selected-markers-${marker.Id}-${marker.GenoFreezeId}" ` + + `value="${marker.Id}_${marker.GenoFreezeId}" ` + + `class="chk-row-select" />`; + } + }, + {data: 'index'}, + {data: "Name", searchable: true}, + {data: "Chr", searchable: true}, + {data: "Mb", searchable: true}, + {data: "Source", searchable: true}, + {data: "Source2", searchable: true} + ], + { + ajax: { + url: "{{url_for('species.populations.genotypes.list_markers', species_id=species.SpeciesId, population_id=population.Id, dataset_id=dataset.Id)}}", + dataSrc: "markers" + }, + paging: true, + scroller: true, + scrollY: "50vh", + scrollCollapse: true, + layout: { + top: "info", + topStart: null, + topEnd: null, + bottom: null, + bottomStart: null, + bottomEnd: null + } + }); + }); + +</script> {%endblock%} diff --git a/uploader/templates/genotypes/list-markers.html b/uploader/templates/genotypes/list-markers.html index 5f3dd6f..22189c7 100644 --- a/uploader/templates/genotypes/list-markers.html +++ b/uploader/templates/genotypes/list-markers.html @@ -57,7 +57,7 @@ <table class="table"> <thead> <tr> - <th title="">#</th> + <th title="">Index</th> <th title="">Marker Name</th> <th title="Chromosome">Chr</th> <th title="Physical location of the marker in megabasepairs"> diff --git a/uploader/templates/genotypes/select-population.html b/uploader/templates/genotypes/select-population.html deleted file mode 100644 index acdd063..0000000 --- a/uploader/templates/genotypes/select-population.html +++ /dev/null @@ -1,25 +0,0 @@ -{%extends "genotypes/base.html"%} -{%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-display-species-card.html" import display_species_card%} -{%from "populations/macro-select-population.html" import select_population_form%} - -{%block title%}Genotypes{%endblock%} - -{%block pagetitle%}Genotypes{%endblock%} - - -{%block contents%} -{{flash_all_messages()}} - -<div class="row"> - {{select_population_form(url_for("species.populations.genotypes.select_population", species_id=species.SpeciesId), species, populations)}} -</div> -{%endblock%} - -{%block sidebarcontents%} -{{display_species_card(species)}} -{%endblock%} - -{%block javascript%} -<script type="text/javascript" src="/static/js/populations.js"></script> -{%endblock%} diff --git a/uploader/templates/genotypes/view-dataset.html b/uploader/templates/genotypes/view-dataset.html index 1c4eccf..d95a8e3 100644 --- a/uploader/templates/genotypes/view-dataset.html +++ b/uploader/templates/genotypes/view-dataset.html @@ -46,8 +46,9 @@ <div class="row"> <h2>Genotype Data</h2> - <p class="text-danger"> - Provide link to enable uploading of genotype data here.</p> + <div class="col" style="margin-bottom: 3px;"> + <a href="#" class="btn btn-primary not-implemented">upload genotypes</a> + </div> </div> {%endblock%} diff --git a/uploader/templates/index.html b/uploader/templates/index.html index e426732..6e9c777 100644 --- a/uploader/templates/index.html +++ b/uploader/templates/index.html @@ -20,7 +20,7 @@ <div class="col"> <a href="{{url_for('species.create_species', return_to='base.index')}}" class="btn btn-outline-primary" - title="Create a new species.">Create a new Species</a> + title="Add a new species to Genenetwork.">add a new Species</a> </div> </div> {%endmacro%} @@ -106,8 +106,8 @@ id="publications-content" role="tabpanel" aria-labelledby="publications-content-tab"> - <p>View, edit and delete existing publications, and add new - publications by clicking on the button below.</p> + <p>You can view, edit, and delete existing publications, as well as add + new ones, by clicking the button below.</p> <a href="{{url_for('publications.index')}}" title="Manage publications." @@ -116,47 +116,55 @@ </div> </div> - {%else%} +{%else%} - <div class="row"> - <p>The Genenetwork Uploader (<em>gn-uploader</em>) enables upload of new data - into the Genenetwork System. It provides Quality Control over data, and - guidance in case you data does not meet the standards for acceptance.</p> - <p> +<div class="row"> + <img src="/static/images/frontpage_banner.png" + alt="Banner image showing the process flow a user would follow." /> +</div> + +<div class="row"> + <p>The GeneNetwork Uploader (gn-uploader) lets you easily add new data to the + GeneNetwork System. It automatically checks your data for quality and walks + you through fixing any issues before submission.</p> +</div> + +<div class="row"> + <div class="col"> <a href="{{authserver_authorise_uri()}}" title="Sign in to the system" class="btn btn-primary">Sign in</a> - to get started.</p> </div> - {%endif%} +</div> +{%endif%} - {%endblock%} +{%endblock%} - {%block sidebarcontents%} - {%if view_under_construction%} - <div class="row"> - <p>The data in Genenetwork is related to one species or another. Use the form - provided to select from existing species, or click on the - "Create a New Species" button if you cannot find the species you want to - work with.</p> - </div> - <div class="row"> - <form id="frm-quick-navigation"> - <legend>Quick Navigation</legend> - <div class="form-group"> - <label for="fqn-species-id">Species</label> - <select name="species_id"> - <option value="">Select species</option> - </select> - </div> - </form> - </div> - {%endif%} - {%endblock%} +{%block sidebarcontents%} +{%if view_under_construction%} +<div class="row"> + <p>The data in Genenetwork is related to one species or another. Use the form + provided to select from existing species, or click on the + "Create a New Species" button if you cannot find the species you want to + work with.</p> +</div> +<div class="row"> + <form id="frm-quick-navigation"> + <legend>Quick Navigation</legend> + <div class="form-group"> + <label for="fqn-species-id">Species</label> + <select name="species_id"> + <option value="">Select species</option> + </select> + </div> + </form> +</div> +{%endif%} +{%endblock%} - {%block javascript%} - <script type="text/javascript" src="/static/js/species.js"></script> - {%endblock%} +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/phenotypes/add-phenotypes-base.html b/uploader/templates/phenotypes/add-phenotypes-base.html index 690c7e1..3207129 100644 --- a/uploader/templates/phenotypes/add-phenotypes-base.html +++ b/uploader/templates/phenotypes/add-phenotypes-base.html @@ -29,8 +29,7 @@ {%block frm_add_phenotypes_elements%}{%endblock%} - <fieldset id="fldset-publication-info"> - <legend>Publication Information</legend> + <h4>Publication Information</h4> <input type="hidden" name="publication-id" id="txt-publication-id" /> <span class="form-text text-muted"> Select a publication for your data. <br /> @@ -44,7 +43,7 @@ <table id="tbl-select-publication" class="table compact stripe"> <thead> <tr> - <th>#</th> + <th>Index</th> <th>PubMed ID</th> <th>Title</th> <th>Authors</th> @@ -53,7 +52,6 @@ <tbody></tbody> </table> - </fieldset> <div class="form-group"> <input type="submit" @@ -86,7 +84,8 @@ if(pub.PubMed_ID) { return `<a href="https://pubmed.ncbi.nlm.nih.gov/` + `${pub.PubMed_ID}/" target="_blank" ` + - `title="Link to publication on NCBI.">` + + `title="Link to publication on NCBI. This will ` + + `open in a new tab.">` + `${pub.PubMed_ID}</a>`; } return ""; @@ -99,10 +98,7 @@ if(pub.Title) { title = pub.Title } - return `<a href="/publications/view/${pub.Id}" ` + - `target="_blank" ` + - `title="Link to view publication details">` + - `${title}</a>`; + return title; } }, { diff --git a/uploader/templates/phenotypes/add-phenotypes-raw-files.html b/uploader/templates/phenotypes/add-phenotypes-raw-files.html index a02fae7..b1322b2 100644 --- a/uploader/templates/phenotypes/add-phenotypes-raw-files.html +++ b/uploader/templates/phenotypes/add-phenotypes-raw-files.html @@ -21,8 +21,7 @@ {%endblock%} {%block frm_add_phenotypes_elements%} -<fieldset id="fldset-file-metadata"> - <legend>File(s) Metadata</legend> + <h4>File(s) Metadata</h4> <div class="form-group"> <label for="txt-file-separator" class="form-label">File Separator</label> <div class="input-group"> @@ -89,12 +88,9 @@ <a href="#docs-file-na" title="Documentation for no-value fields"> documentation for more information</a>.</span> </div> -</fieldset> -<fieldset id="fldset-files"> <legend>Data File(s)</legend> - <fieldset id="fldset-descriptions-file"> <div class="form-group"> <div class="form-check"> <input id="chk-phenotype-descriptions-transposed" @@ -145,10 +141,8 @@ {{display_preview_table( "tbl-preview-pheno-desc", "phenotype descriptions")}} </div> - </fieldset> - - <fieldset id="fldset-data-file"> + <div class="form-group"> <div class="form-check"> <input id="chk-phenotype-data-transposed" @@ -196,11 +190,9 @@ on the expected format for the file provided here.</p>')}} {{display_preview_table("tbl-preview-pheno-data", "phenotype data")}} </div> - </fieldset> {%if population.Family in families_with_se_and_n%} - <fieldset id="fldset-se-file"> <div class="form-group"> <div class="form-check"> <input id="chk-phenotype-se-transposed" @@ -247,10 +239,8 @@ {{display_preview_table("tbl-preview-pheno-se", "standard errors")}} </div> - </fieldset> - <fieldset id="fldset-n-file"> <div class="form-group"> <div class="form-check"> <input id="chk-phenotype-n-transposed" @@ -297,8 +287,6 @@ {{display_preview_table("tbl-preview-pheno-n", "number of samples/individuals")}} </div> - </fieldset> -</fieldset> {%endif%} {%endblock%} @@ -477,7 +465,7 @@ .map((field) => { var value = field.trim(); if(navalues.includes(value)) { - return "⋘NUL⋙"; + return "[NO-VALUE]"; } return value; }) diff --git a/uploader/templates/phenotypes/base.html b/uploader/templates/phenotypes/base.html index fe7ccd3..5959422 100644 --- a/uploader/templates/phenotypes/base.html +++ b/uploader/templates/phenotypes/base.html @@ -3,6 +3,7 @@ {%block breadcrumbs%} {{super()}} +{%if dataset%} <li class="breadcrumb-item"> <a href="{{url_for('species.populations.phenotypes.view_dataset', species_id=species['SpeciesId'], @@ -11,6 +12,7 @@ {{dataset["Name"]}} </a> </li> +{%endif%} {%endblock%} {%block contents%} diff --git a/uploader/templates/phenotypes/confirm-delete-phenotypes.html b/uploader/templates/phenotypes/confirm-delete-phenotypes.html new file mode 100644 index 0000000..3cf6e65 --- /dev/null +++ b/uploader/templates/phenotypes/confirm-delete-phenotypes.html @@ -0,0 +1,196 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Delete Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="view-dataset"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">View</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"><h2>Delete Phenotypes</h2></div> + +{%if phenotypes | length > 0%} +<div class="row"> + <p>You have requested to delete the following phenotypes:</p> +</div> + +<div class="row"> + <div class="col"> + <a id="btn-select-all-phenotypes" + href="#" + class="btn btn-info" + title="Select all phenotypes">select all</a> + </div> + <div class="col"> + <a id="btn-deselect-all-phenotypes" + href="#" + class="btn btn-warning" + title="Deselect all phenotypes">deselect all</a> + </div> +</div> + +<div class="row"> + <table id="tbl-delete-phenotypes" class="table"> + <thead> + <tr> + <th>Index</th> + <th>Record ID</th> + <th>Description</th> + </tr> + </thead> + <tbody> + {%for phenotype in phenotypes%} + <tr> + <td> + <input id="chk-xref-id-{{phenotype.xref_id}}" + name="xref_ids" + type="checkbox" + value="{{phenotype.xref_id}}" + class="chk-row-select" /> + </td> + <td>{{phenotype.xref_id}}</td> + <td>{{phenotype.Post_publication_description or + phenotype.Pre_publication_description or + phenotype.original_description}}</td> + </tr> + {%endfor%} + </tbody> + </table> +</div> + +<div class="row"> + <form id="frm-delete-phenotypes-selected" + method="POST" + action="{{url_for('species.populations.phenotypes.delete_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}"> + <div class="row"> + <div class="col"> + <input class="btn btn-info" + type="submit" + title="Cancel delete and return to dataset page." + name="action" + value="cancel" /></div> + <div class="col"> + <input id="btn-delete-phenotypes-selected" + class="btn btn-danger" + type="submit" + title="Delete the selected phenotypes from this dataset." + name="action" + value="delete" /> + </div> + </div> + </form> +</div> +{%else%} +<div class="row"> + <p>You did not select any phenotypes to delete. Delete everything?</p> +</div> + +<div class="row"> + <form id="frm-delete-phenotypes-all" + method="POST" + action="{{url_for('species.populations.phenotypes.delete_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}"> + <div class="form-check"> + <input class="form-check-input" + type="checkbox" + name="confirm_delete_all_phenotypes" + id="chk-confirm-delete-all-phenotypes" /> + <label class="form-check-label" + for="chk-confirm-delete-all-phenotypes"> + delete all phenotypes?</label> + </div> + + <div class="row"> + <div class="col"> + <input class="btn btn-info" + type="submit" + title="Cancel delete and return to dataset page." + name="action" + value="cancel" /></div> + <div class="col"> + <input class="btn btn-danger" + type="submit" + title="Delete all phenotypes in this dataset." + name="action" + value="delete" /> + </div> + </div> + </form> +</div> +{%endif%} + +{%endblock%} + +{%block javascript%} +<script type="text/javascript"> + $(function() { + var dt = buildDataTable( + "#tbl-delete-phenotypes", + data=[], + columns=[], + userSettings={ + responsive: true, + select: { + style: "os", + info: false + }, + initComplete: function(setting, json) { + var api = this.api(); + api.rows().select(); + api.rows({selected: true}).nodes().each((node, index) => { + setRowChecked(node); + }); + } + }); + + $("#btn-select-all-phenotypes").on("click", function(event) { + dt.selectAll(); + }); + + $("#btn-deselect-all-phenotypes").on("click", function(event) { + dt.deselectAll(); + }); + + $("#btn-delete-phenotypes-selected").on("click", function(event) { + event.preventDefault(); + form = $("#frm-delete-phenotypes-selected"); + form.find(".dynamically-added-element").remove(); + dt.rows({selected: true}).nodes().each(function(node, index) { + var xref_id = $(node) + .find('input[type="checkbox"]:checked') + .val(); + var chk = $('<input type="checkbox">'); + chk.attr("class", "dynamically-added-element"); + chk.attr("value", xref_id); + chk.attr("name", "xref_ids"); + chk.attr("style", "display: none"); + chk.prop("checked", true); + form.append(chk); + }); + form.append( + $('<input type="hidden" name="action" value="delete" />')); + form.submit(); + }) + }); +</script> +{%endblock%} + diff --git a/uploader/templates/phenotypes/create-dataset.html b/uploader/templates/phenotypes/create-dataset.html index 19a2b34..9963953 100644 --- a/uploader/templates/phenotypes/create-dataset.html +++ b/uploader/templates/phenotypes/create-dataset.html @@ -48,7 +48,8 @@ {%else%} class="form-control" {%endif%} - required="required" /> + required="required" + readonly="readonly" /> <small class="form-text text-muted"> <p>A short representative name for the dataset.</p> <p>Recommended: Use the population name and append "Publish" at the end. @@ -66,7 +67,7 @@ <input id="txt-dataset-fullname" name="dataset-fullname" type="text" - value="{{original_formdata.get('dataset-fullname', '')}}" + value="{{original_formdata.get('dataset-fullname', '') or population.Name + ' Phenotypes'}}" {%if errors["dataset-fullname"] is defined%} class="form-control danger" {%else%} diff --git a/uploader/templates/phenotypes/edit-phenotype.html b/uploader/templates/phenotypes/edit-phenotype.html index 115d6af..1b3ee9d 100644 --- a/uploader/templates/phenotypes/edit-phenotype.html +++ b/uploader/templates/phenotypes/edit-phenotype.html @@ -142,7 +142,7 @@ <table class="table table-striped table-responsive table-form-table"> <thead style="position: sticky; top: 0;"> <tr> - <th>#</th> + <th>Index</th> <th>Sample</th> <th>Value</th> {%if population.Family in families_with_se_and_n%} diff --git a/uploader/templates/phenotypes/job-status.html b/uploader/templates/phenotypes/job-status.html index 0bbe8e0..951907f 100644 --- a/uploader/templates/phenotypes/job-status.html +++ b/uploader/templates/phenotypes/job-status.html @@ -52,10 +52,10 @@ <p> {%if errors | length == 0%} <a href="{{url_for('species.populations.phenotypes.review_job_data', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id, - job_id=job_id)}}" + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + job_id=job_id)}}" class="btn btn-primary" title="Continue to process data">Continue</a> {%else%} @@ -70,13 +70,28 @@ </div> <h3 class="subheading">upload errors</h3> +{%if errors | length == 0 %} <div class="row" style="max-height: 20em; overflow: scroll;"> - {%if errors | length == 0 %} <p class="text-info"> <span class="glyphicon glyphicon-info-sign"></span> No errors found so far </p> - {%else%} +</div> +{%else%} +{%if errors | length > 0%} +<div class="row"> + <div class="col"> + <a href="{{url_for('species.populations.phenotypes.download_errors', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + job_id=job_id)}}" + class="btn btn-info" + title="Download the errors as a CSV file.">download errors CSV</a> + </div> +</div> +{%endif%} +<div class="row" style="max-height: 20em; overflow: scroll;"> <table class="table table-responsive"> <thead style="position: sticky; top: 0; background: white;"> <tr> @@ -111,7 +126,8 @@ {%endfor%} </tbody> </table> - {%endif%} +</div> +{%endif%} </div> <div class="row"> diff --git a/uploader/templates/phenotypes/macro-display-preview-table.html b/uploader/templates/phenotypes/macro-display-preview-table.html index 5a4c422..6dffe9f 100644 --- a/uploader/templates/phenotypes/macro-display-preview-table.html +++ b/uploader/templates/phenotypes/macro-display-preview-table.html @@ -1,19 +1,11 @@ {%macro display_preview_table(tableid, filetype)%} -<div class="card"> - <div class="card-body"> - <h5 class="card-title">{{filetype | title}}: File Preview</h5> - <div class="card-text" style="overflow: scroll;"> - <table id="{{tableid}}" class="table table-condensed table-responsive"> - <thead> - <tr> - </tr> - <tbody> - <tr> - <td class="data-row-template text-info"></td> - </tr> - </tbody> - </table> - </div> - </div> +<div class="table-responsive" + style="max-width:39.2em;border-radius:5px;border: solid 1px;overflow-x: scroll;"> + <h5>{{filetype | title}}: File Preview</h5> + <table id="{{tableid}}" class="table"> + <thead><tr></tr></thead> + + <tbody></tbody> + </table> </div> {%endmacro%} diff --git a/uploader/templates/phenotypes/review-job-data.html b/uploader/templates/phenotypes/review-job-data.html index c8355b2..0e8f119 100644 --- a/uploader/templates/phenotypes/review-job-data.html +++ b/uploader/templates/phenotypes/review-job-data.html @@ -70,6 +70,9 @@ {%endif%} {%endfor%} </ul> +</div> + +<div class="row"> <form id="frm-review-phenotype-data" method="POST" @@ -78,10 +81,38 @@ population_id=population.Id, dataset_id=dataset.Id)}}"> <input type="hidden" name="data-qc-job-id" value="{{job.jobid}}" /> - <input type="submit" - value="continue" - class="btn btn-primary" /> + <div class="form-group"> + <label for="txt-data-name">data name</label> + <input type="text" + id="txt-data-name" + class="form-control" + name="data_name" + title="A short, descriptive name for this data." + placeholder="{{user.email}} - {{dataset.Name}} - {{timestamp}}" + value="{{user.email}} - {{dataset.Name}} - {{timestamp}}" + required="required"> + <span class="form-text text-muted"> + This is a short, descriptive name for the data. It is useful to humans, + enabling them identify what traits each data "resource" wraps around. + </span> + </div> + + {%if view_under_construction%} + <div class="form-group"> + <label for="txt-data-description">data description</label> + <textarea id="txt-data-description" + class="form-control" + name="data_description" + title="A longer description for this data." + rows="5"></textarea> + <span class="form-text text-muted"> + </span> + </div> + {%endif%} + + <button type="submit" class="btn btn-primary">continue</button> </form> + </div> {%else%} <div class="row"> diff --git a/uploader/templates/phenotypes/view-dataset.html b/uploader/templates/phenotypes/view-dataset.html index 1fd15b2..fc84757 100644 --- a/uploader/templates/phenotypes/view-dataset.html +++ b/uploader/templates/phenotypes/view-dataset.html @@ -23,25 +23,12 @@ {{flash_all_messages()}} <div class="row"> - <p>The basic dataset details are:</p> - - <table class="table"> - <thead> - <tr> - <th>Name</th> - <th>Full Name</th> - <th>Short Name</th> - </tr> - </thead> + <h2>Phenotype Data</h2> - <tbody> - <tr> - <td>{{dataset.Name}}</td> - <td>{{dataset.FullName}}</td> - <td>{{dataset.ShortName}}</td> - </tr> - </tbody> - </table> + <p>Click on any of the phenotypes in the table below to view and edit that + phenotype's data.</p> + <p>Use the search to filter through all the phenotypes and find specific + phenotypes of interest.</p> </div> <div class="row"> @@ -67,7 +54,7 @@ <input type="submit" title="Compute/Recompute the means for all phenotypes." class="btn btn-info" - value="(rec/c)ompute means" + value="compute means" id="submit-frm-recompute-phenotype-means" /> </form> </div> @@ -85,24 +72,29 @@ <input type="submit" title="Run/Rerun QTLReaper." class="btn btn-info" - value="(re)run QTLReaper" + value="run QTLReaper" id="submit-frm-rerun-qtlreaper" /> </form> </div> -</div> - -<div class="row"> - <h2>Phenotype Data</h2> - <p>Click on any of the phenotypes in the table below to view and edit that - phenotype's data.</p> - <p>Use the search to filter through all the phenotypes and find specific - phenotypes of interest.</p> + <div class="col"> + <form id="frm-delete-phenotypes" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.delete_phenotypes', + species_id=species['SpeciesId'], + population_id=population['Id'], + dataset_id=dataset['Id'])}}"> + <input type="submit" + class="btn btn-danger" + id="btn-delete-phenotypes" + title="Delete phenotypes from this dataset. If no phenotypes are selected in the table, this will delete ALL the phenotypes." + value="delete phenotypes" /> + </form> + </div> </div> - -<div class="row"> - +<div class="row" style="margin-top: 0.5em;"> <table id="tbl-phenotypes-list" class="table compact stripe cell-border"> <thead> <tr> @@ -156,14 +148,36 @@ return `<a href="${url.toString()}" target="_blank">` + `${pheno.InbredSetCode}_${pheno.xref_id}` + `</a>`; - } + }, + title: "Record", + visible: true, + searchable: true }, { data: function(pheno) { return (pheno.Post_publication_description || pheno.Original_description || pheno.Pre_publication_description); - } + }, + title: "Description", + visible: true, + searchable: true + }, + { + data: function(pheno) { + return pheno.publication.Title; + }, + title: "Publication Title", + visible: false, + searchable: true + }, + { + data: function(pheno) { + return pheno.publication.Authors; + }, + title: "Authors", + visible: false, + searchable: true } ], { @@ -202,6 +216,33 @@ }); form.submit(); }); + + $("#btn-delete-phenotypes").on( + "click", + function(event) { + // Collect selected phenotypes for deletion, if any. + event.preventDefault(); + form = $("#frm-delete-phenotypes"); + form.find(".dynamically-added-element").remove(); + $("#tbl-phenotypes-list") + .DataTable() + .rows({selected: true}). + nodes().each(function(node, index) { + var parts = $(node) + .find(".chk-row-select") + .val() + .split("_"); + var xref_id = parts[parts.length - 1].trim(); + var chk = $('<input type="checkbox">'); + chk.attr("class", "dynamically-added-element"); + chk.attr("value", xref_id); + chk.attr("name", "xref_ids"); + chk.attr("style", "display: none"); + chk.prop("checked", true); + form.append(chk); + }); + form.submit(); + }); }); </script> {%endblock%} diff --git a/uploader/templates/phenotypes/view-phenotype.html b/uploader/templates/phenotypes/view-phenotype.html index a69b024..a59949e 100644 --- a/uploader/templates/phenotypes/view-phenotype.html +++ b/uploader/templates/phenotypes/view-phenotype.html @@ -103,7 +103,7 @@ or "group:resource:delete-resource" in privileges%} <table class="table"> <thead> <tr> - <th>#</th> + <th>Index</th> <th>Sample</th> <th>Value</th> {%if has_se%} diff --git a/uploader/templates/populations/view-population.html b/uploader/templates/populations/view-population.html index ac89bc7..29add29 100644 --- a/uploader/templates/populations/view-population.html +++ b/uploader/templates/populations/view-population.html @@ -95,14 +95,13 @@ id="genotypes-content" role="tabpanel" aria-labelledby="genotypes-content-tab"> - <p>This allows you to upload the data that concerns your genotypes.</p> - <p>Any samples/individuals/cases/strains that do not already exist in the - system will be added. This does not delete any existing data.</p> + <p>Click the button to view and manage genetic data for individuals in + this population.</p> <a href="{{url_for('species.populations.genotypes.list_genotypes', species_id=species.SpeciesId, population_id=population.Id)}}" title="Upload genotype information for the '{{population.FullName}}' population of the '{{species.FullName}}' species." - class="btn btn-primary">upload genotypes</a> + class="btn btn-primary">manage genotypes</a> </div> <div class="tab-pane fade" id="expression-data-content" role="tabpanel" aria-labelledby="expression-data-content-tab"> <p>Upload expression data (mRNA data) for this population.</p> diff --git a/uploader/templates/publications/create-publication.html b/uploader/templates/publications/create-publication.html index fb0127d..da5889e 100644 --- a/uploader/templates/publications/create-publication.html +++ b/uploader/templates/publications/create-publication.html @@ -91,22 +91,22 @@ class="col-sm-2 col-form-label"> Month</label> <div class="col-sm-4"> - <select class="form-control" + <select class="form-select" id="select-publication-month" name="publication-month"> <option value="">Select a month</option> - <option value="january">January</option> - <option value="february">February</option> - <option value="march">March</option> - <option value="april">April</option> - <option value="may">May</option> - <option value="june">June</option> - <option value="july">July</option> - <option value="august">August</option> - <option value="september">September</option> - <option value="october">October</option> - <option value="november">November</option> - <option value="december">December</option> + <option {%if current_month | lower == "january"%}selected="selected"{%endif%}value="january">January</option> + <option {%if current_month | lower == "february"%}selected="selected"{%endif%}value="february">February</option> + <option {%if current_month | lower == "march"%}selected="selected"{%endif%}value="march">March</option> + <option {%if current_month | lower == "april"%}selected="selected"{%endif%}value="april">April</option> + <option {%if current_month | lower == "may"%}selected="selected"{%endif%}value="may">May</option> + <option {%if current_month | lower == "june"%}selected="selected"{%endif%}value="june">June</option> + <option {%if current_month | lower == "july"%}selected="selected"{%endif%}value="july">July</option> + <option {%if current_month | lower == "august"%}selected="selected"{%endif%}value="august">August</option> + <option {%if current_month | lower == "september"%}selected="selected"{%endif%}value="september">September</option> + <option {%if current_month | lower == "october"%}selected="selected"{%endif%}value="october">October</option> + <option {%if current_month | lower == "november"%}selected="selected"{%endif%}value="november">November</option> + <option {%if current_month | lower == "december"%}selected="selected"{%endif%}value="december">December</option> </select> <span class="form-text text-muted">Month of publication</span> </div> @@ -119,7 +119,10 @@ id="txt-publication-year" name="publication-year" class="form-control" - min="1960" /> + min="1960" + max="{{current_year}}" + value="{{current_year or ''}}" + required="required" /> <span class="form-text text-muted">Year of publication</span> </div> </div> diff --git a/uploader/templates/publications/index.html b/uploader/templates/publications/index.html index 54d3fc0..eb2e81b 100644 --- a/uploader/templates/publications/index.html +++ b/uploader/templates/publications/index.html @@ -17,7 +17,7 @@ </div> <div class="row"> - <p>Click on title to view more details and to edit details for that + <p>Click on the title to view more details or to edit the information for that publication.</p> </div> @@ -25,7 +25,7 @@ <table id="tbl-list-publications" class="table compact stripe"> <thead> <tr> - <th>#</th> + <th>Index</th> <th>PubMed ID</th> <th>Title</th> <th>Authors</th> diff --git a/uploader/templates/species/base.html b/uploader/templates/species/base.html index a7c1a8f..3be79f0 100644 --- a/uploader/templates/species/base.html +++ b/uploader/templates/species/base.html @@ -2,9 +2,11 @@ {%block breadcrumbs%} {{super()}} +{%if species%} <li class="breadcrumb-item"> <a href="{{url_for('species.view_species', species_id=species['SpeciesId'])}}"> {{species["Name"]|title}} </a> </li> +{%endif%} {%endblock%} diff --git a/uploader/ui.py b/uploader/ui.py index 1994056..41791c7 100644 --- a/uploader/ui.py +++ b/uploader/ui.py @@ -1,5 +1,5 @@ """Utilities to handle the UI""" -from flask import render_template as flask_render_template +from uploader.flask_extensions import render_template as flask_render_template def make_template_renderer(default): """Render template for species.""" |
