diff options
Diffstat (limited to 'uploader')
159 files changed, 14776 insertions, 0 deletions
diff --git a/uploader/__init__.py b/uploader/__init__.py new file mode 100644 index 0000000..cae531b --- /dev/null +++ b/uploader/__init__.py @@ -0,0 +1,91 @@ +"""The Quality-Control Web Application entry point""" +import os +import sys +import logging +from pathlib import Path + +from flask import Flask, request +from flask_session import Session + +from uploader.oauth2.client import user_logged_in, authserver_authorise_uri + +from . import session +from .base_routes import base +from .files.views import files +from .species import speciesbp +from .oauth2.views import oauth2 +from .expression_data import exprdatabp +from .errors import register_error_handlers + +def override_settings_with_envvars( + app: Flask, ignore: tuple[str, ...]=tuple()) -> None: + """Override settings in `app` with those in ENVVARS""" + for setting in (key for key in app.config if key not in ignore): + app.config[setting] = os.environ.get(setting) or app.config[setting] + + +def __log_gunicorn__(app: Flask) -> Flask: + """Set up logging for the WSGI environment with GUnicorn""" + logger = logging.getLogger("gunicorn.error") + app.logger.handlers = logger.handlers + app.logger.setLevel(logger.level) + return app + + +def __log_dev__(app: Flask) -> Flask: + """Set up logging for the development environment.""" + stderr_handler = logging.StreamHandler(stream=sys.stderr) + app.logger.addHandler(stderr_handler) + + root_logger = logging.getLogger() + root_logger.addHandler(stderr_handler) + root_logger.setLevel(app.config["LOG_LEVEL"]) + + return app + + +def setup_logging(app: Flask) -> Flask: + """Set up logging for the application.""" + software, *_version_and_comments = os.environ.get( + "SERVER_SOFTWARE", "").split('/') + return __log_gunicorn__(app) if bool(software) else __log_dev__(app) + + +def create_app(): + """The application factory""" + app = Flask(__name__) + app.config.from_pyfile( + Path(__file__).parent.joinpath("default_settings.py")) + if "UPLOADER_CONF" in os.environ: + app.config.from_envvar("UPLOADER_CONF") # Override defaults with instance path + + override_settings_with_envvars(app, ignore=tuple()) + + secretsfile = app.config.get("UPLOADER_SECRETS", "").strip() + if bool(secretsfile): + secretsfile = Path(secretsfile).absolute() + app.config["UPLOADER_SECRETS"] = secretsfile + if secretsfile.exists(): + # Silently ignore secrets if the file does not exist. + app.config.from_pyfile(secretsfile) + + setup_logging(app) + + # setup jinja2 symbols + app.add_template_global(lambda : request.url, name="request_url") + app.add_template_global(authserver_authorise_uri) + app.add_template_global(lambda: app.config["GN2_SERVER_URL"], + name="gn2server_uri") + app.add_template_global(user_logged_in) + app.add_template_global(lambda : session.user_details()["email"], name="user_email") + + Session(app) + + # setup blueprints + app.register_blueprint(base, url_prefix="/") + app.register_blueprint(files, url_prefix="/files") + app.register_blueprint(oauth2, url_prefix="/oauth2") + app.register_blueprint(speciesbp, url_prefix="/species") + + register_error_handlers(app) + return app diff --git a/uploader/authorisation.py b/uploader/authorisation.py new file mode 100644 index 0000000..a283980 --- /dev/null +++ b/uploader/authorisation.py @@ -0,0 +1,67 @@ +"""Authorisation utilities.""" +import logging +from functools import wraps + +from typing import Callable +from flask import flash, redirect +from pymonad.either import Left, Right, Either +from authlib.jose import KeySet, JsonWebToken +from authlib.jose.errors import BadSignatureError + +from uploader import session +from uploader.oauth2.client import auth_server_jwks + +def require_login(function): + """Check that the user is logged in before executing `func`.""" + @wraps(function) + def __is_session_valid__(*args, **kwargs): + """Check that the user is logged in and their token is valid.""" + def __clear_session__(_no_token): + session.clear_session_info() + flash("You need to be signed in.", "alert-danger big-alert") + return redirect("/") + + return session.user_token().either( + __clear_session__, + lambda token: function(*args, **kwargs)) + return __is_session_valid__ + + +def __validate_token__(jwks: KeySet, token: dict) -> Either: + """Check that a token is signed by a key from the authorisation server.""" + for key in jwks.keys: + try: + # Fixes CVE-2016-10555. See + # https://docs.authlib.org/en/latest/jose/jwt.html + jwt = JsonWebToken(["RS256"]) + jwt.decode(token["access_token"], key) + return Right(token) + except BadSignatureError: + pass + + return Left({"token": token}) + + +def require_token(func: Callable) -> Callable: + """ + Wrap functions that require the user be authorised to perform the operations + that the functions in question provide. + """ + def __invalid_token__(_whatever): + logging.debug("==========> Failure log: %s", _whatever) + raise Exception( + "You attempted to access a feature of the system that requires " + "authorisation. Unfortunately, we could not verify you have the " + "appropriate authorisation to perform the action you requested. " + "You might need to log in, or if you already are logged in, you " + "need to log out, then log back in to get a newer token/session.") + @wraps(func) + def __wrapper__(*args, **kwargs): + return session.user_token().then(lambda tok: { + "jwks": auth_server_jwks(), + "token": tok + }).then(lambda vals: __validate_token__(**vals)).either( + __invalid_token__, + lambda tok: func(*args, **{**kwargs, "token": tok})) + + return __wrapper__ diff --git a/uploader/base_routes.py b/uploader/base_routes.py new file mode 100644 index 0000000..326086f --- /dev/null +++ b/uploader/base_routes.py @@ -0,0 +1,60 @@ +"""Basic routes required for all pages""" +import os +from urllib.parse import urljoin + +from flask import (Blueprint, + current_app as app, + send_from_directory) + +from uploader.ui import make_template_renderer +from uploader.oauth2.client import user_logged_in + +base = Blueprint("base", __name__) +render_template = make_template_renderer("home") + + +@base.route("/favicon.ico", methods=["GET"]) +def favicon(): + """Return the favicon.""" + return send_from_directory(os.path.join(app.root_path, "static"), + "images/CITGLogo.png", + mimetype="image/png") + + +@base.route("/", methods=["GET"]) +def index(): + """Load the landing page""" + return render_template("index.html" if user_logged_in() else "login.html", + gn2server_intro=urljoin(app.config["GN2_SERVER_URL"], + "/intro")) + +def appenv(): + """Get app's guix environment path.""" + return os.environ.get("GN_UPLOADER_ENVIRONMENT") + +@base.route("/bootstrap/<path:filename>") +def bootstrap(filename): + """Fetch bootstrap files.""" + return send_from_directory( + appenv(), f"share/genenetwork2/javascript/bootstrap/{filename}") + + +@base.route("/jquery/<path:filename>") +def jquery(filename): + """Fetch jquery files.""" + return send_from_directory( + appenv(), f"share/genenetwork2/javascript/jquery/{filename}") + + +@base.route("/datatables/<path:filename>") +def datatables(filename): + """Fetch DataTables files.""" + return send_from_directory( + appenv(), f"share/genenetwork2/javascript/DataTables/{filename}") + + +@base.route("/node-modules/<path:filename>") +def node_modules(filename): + """Fetch node-js modules.""" + return send_from_directory( + appenv(), f"lib/node_modules/{filename}") diff --git a/uploader/check_connections.py b/uploader/check_connections.py new file mode 100644 index 0000000..c9b9aa3 --- /dev/null +++ b/uploader/check_connections.py @@ -0,0 +1,27 @@ +"""Check the various connection used in the application""" +import sys +import traceback + +import redis +import MySQLdb +from gn_libs.mysqldb import database_connection + +def check_redis(uri: str): + "Check the redis connection" + try: + with redis.Redis.from_url(uri) as rconn: + rconn.ping() + except redis.exceptions.ConnectionError as conn_err: + print(conn_err, file=sys.stderr) + print(traceback.format_exc(), file=sys.stderr) + sys.exit(1) + +def check_db(uri: str): + "Check the mysql connection" + try: + with database_connection(uri) as dbconn: # pylint: disable=[unused-variable] + pass + except MySQLdb.OperationalError as op_err: + print(op_err, file=sys.stderr) + print(traceback.format_exc(), file=sys.stderr) + sys.exit(1) diff --git a/uploader/datautils.py b/uploader/datautils.py new file mode 100644 index 0000000..46a55c4 --- /dev/null +++ b/uploader/datautils.py @@ -0,0 +1,38 @@ +"""Generic data utilities: Rename module.""" +import math +from functools import reduce +from typing import Union, Sequence + +def enumerate_sequence(seq: Sequence[dict], start:int = 1) -> Sequence[dict]: + """Enumerate sequence beginning at 1""" + return tuple({**item, "sequence_number": seqno} + for seqno, item in enumerate(seq, start=start)) + + +def order_by_family(items: tuple[dict, ...], + family_key: str = "Family", + order_key: str = "FamilyOrderId") -> list: + """Order the populations by their families.""" + def __family_order__(item): + orderval = item[order_key] + return math.inf if orderval is None else orderval + + def __order__(ordered, current): + _key = (__family_order__(current), current[family_key]) + return { + **ordered, + _key: ordered.get(_key, tuple()) + (current,) + } + + return sorted(tuple(reduce(__order__, items, {}).items()), + key=lambda item: item[0][0]) + + +def safe_int(val: Union[str, int, float]) -> int: + """ + Convert val into an integer: if val cannot be converted, return a zero. + """ + try: + return int(val) + except ValueError: + return 0 diff --git a/uploader/db/__init__.py b/uploader/db/__init__.py new file mode 100644 index 0000000..d2b1d9d --- /dev/null +++ b/uploader/db/__init__.py @@ -0,0 +1,2 @@ +"""Database functions""" +from .datasets import geno_datasets_by_species_and_population diff --git a/uploader/db/averaging.py b/uploader/db/averaging.py new file mode 100644 index 0000000..62bbe67 --- /dev/null +++ b/uploader/db/averaging.py @@ -0,0 +1,23 @@ +"""Functions for db interactions for averaging methods""" +from typing import Optional + +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +def averaging_methods(conn: mdb.Connection) -> tuple[dict, ...]: + """Fetch all available averaging methods""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM AvgMethod") + return tuple(dict(row) for row in cursor.fetchall()) + +def averaging_method_by_id( + conn: mdb.Connection, averageid: int) -> Optional[dict]: + """Fetch the averaging method by its ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM AvgMethod WHERE Id=%s", + (averageid,)) + result = cursor.fetchone() + if bool(result): + return dict(result) + + return None diff --git a/uploader/db/datasets.py b/uploader/db/datasets.py new file mode 100644 index 0000000..767ec41 --- /dev/null +++ b/uploader/db/datasets.py @@ -0,0 +1,133 @@ +"""Functions for accessing the database relating to datasets.""" +from datetime import date +from typing import Optional + +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +def geno_datasets_by_species_and_population( + conn: mdb.Connection, + speciesid: int, + populationid: int) -> tuple[dict, ...]: + """Retrieve all genotypes datasets by species and population""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT gf.* FROM InbredSet AS iset INNER JOIN GenoFreeze AS gf " + "ON iset.InbredSetId=gf.InbredSetId " + "WHERE iset.SpeciesId=%(sid)s AND iset.InbredSetId=%(pid)s", + {"sid": speciesid, "pid": populationid}) + return tuple(dict(row) for row in cursor.fetchall()) + +def geno_dataset_by_id(conn: mdb.Connection, dataset_id) -> Optional[dict]: + """Retrieve genotype dataset by ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s", (dataset_id,)) + _dataset = cursor.fetchone() + return dict(_dataset) if bool(_dataset) else None + +def probeset_studies_by_species_and_population( + conn: mdb.Connection, + speciesid: int, + populationid: int) -> tuple[dict, ...]: + """Retrieve all probesets""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT pf.* FROM InbredSet AS iset INNER JOIN ProbeFreeze AS pf " + "ON iset.InbredSetId=pf.InbredSetId " + "WHERE iset.SpeciesId=%(sid)s AND iset.InbredSetId=%(pid)s", + {"sid": speciesid, "pid": populationid}) + return tuple(dict(row) for row in cursor.fetchall()) + +def probeset_datasets_by_study(conn: mdb.Connection, + studyid: int) -> tuple[dict, ...]: + """Retrieve all probeset databases by study.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s", + (studyid,)) + return tuple(dict(row) for row in cursor.fetchall()) + +def probeset_study_by_id(conn: mdb.Connection, studyid) -> Optional[dict]: + """Retrieve ProbeSet study by ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM ProbeFreeze WHERE Id=%s", (studyid,)) + _study = cursor.fetchone() + return dict(_study) if bool(_study) else None + +def probeset_create_study(conn: mdb.Connection,#pylint: disable=[too-many-arguments] + populationid: int, + platformid: int, + tissueid: int, + studyname: str, + studyfullname: str = "", + studyshortname: str = ""): + """Create a new ProbeSet study.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + studydata = { + "platid": platformid, + "tissueid": tissueid, + "name": studyname, + "fname": studyfullname or studyname, + "sname": studyshortname, + "today": date.today().isoformat(), + "popid": populationid + } + cursor.execute( + """ + INSERT INTO ProbeFreeze( + ChipId, TissueId, Name, FullName, ShortName, CreateTime, + InbredSetId + ) VALUES ( + %(platid)s, %(tissueid)s, %(name)s, %(fname)s, %(sname)s, + %(today)s, %(popid)s + ) + """, + studydata) + studyid = cursor.lastrowid + cursor.execute("UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s", + (studyid, studyid)) + return {**studydata, "studyid": studyid} + +def probeset_create_dataset(conn: mdb.Connection,#pylint: disable=[too-many-arguments] + studyid: int, + averageid: int, + datasetname: str, + datasetfullname: str, + datasetshortname: str="", + public: bool = True, + datascale="log2") -> dict: + """Create a new ProbeSet dataset.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + dataset = { + "studyid": studyid, + "averageid": averageid, + "name2": datasetname, + "fname": datasetfullname, + "name": datasetshortname, + "sname": datasetshortname, + "today": date.today().isoformat(), + "public": 2 if public else 0, + "authorisedusers": "williamslab", + "datascale": datascale + } + cursor.execute( + """ + INSERT INTO ProbeSetFreeze( + ProbeFreezeId, AvgId, Name, Name2, FullName, ShortName, + CreateTime, public, AuthorisedUsers, DataScale) + VALUES( + %(studyid)s, %(averageid)s, %(name)s, %(name2)s, %(fname)s, + %(sname)s, %(today)s, %(public)s, %(authorisedusers)s, + %(datascale)s) + """, + dataset) + return {**dataset, "datasetid": cursor.lastrowid} + +def probeset_dataset_by_id(conn: mdb.Connection, datasetid) -> Optional[dict]: + """Fetch a ProbeSet dataset by its ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Id=%s", (datasetid,)) + result = cursor.fetchone() + if bool(result): + return dict(result) + + return None diff --git a/uploader/db/tissues.py b/uploader/db/tissues.py new file mode 100644 index 0000000..9fe7bab --- /dev/null +++ b/uploader/db/tissues.py @@ -0,0 +1,50 @@ +"""Handle db interactions for tissue.""" +from typing import Union, Optional + +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +def all_tissues(conn: mdb.Connection) -> tuple[dict, ...]: + """All available tissue.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Tissue ORDER BY TissueName") + return tuple(dict(row) for row in cursor.fetchall()) + + +def tissue_by_id(conn: mdb.Connection, tissueid) -> Optional[dict]: + """Retrieve a tissue by its ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Tissue WHERE Id=%s", (tissueid,)) + result = cursor.fetchone() + if bool(result): + return dict(result) + + return None + + +def create_new_tissue( + conn: mdb.Connection, + name: str, + shortname: str, + birnlexid: Optional[str] = None, + birnlexname: Optional[str] = None +) -> dict[str, Union[int, str, None]]: + """Add a new tissue, organ or biological material to the database.""" + with conn.cursor() as cursor: + cursor.execute( + "INSERT INTO " + "Tissue(TissueName, Name, Short_Name, BIRN_lex_ID, BIRN_lex_Name) " + "VALUES (%s, %s, %s, %s, %s)", + (name, name, shortname, birnlexid, birnlexname)) + tissueid = cursor.lastrowid + cursor.execute("UPDATE Tissue SET TissueId=%s WHERE Id=%s", + (tissueid, tissueid)) + return { + "Id": tissueid, + "TissueId": tissueid, + "TissueName": name, + "Name": name, + "Short_Name": shortname, + "BIRN_lex_ID": birnlexid, + "BIRN_lex_Name": birnlexname + } diff --git a/uploader/db_utils.py b/uploader/db_utils.py new file mode 100644 index 0000000..d9d521e --- /dev/null +++ b/uploader/db_utils.py @@ -0,0 +1,20 @@ +"""module contains all db related stuff""" +from typing import Any, Callable + +import MySQLdb as mdb +from redis import Redis +from flask import current_app as app +from gn_libs.mysqldb import database_connection + + +def with_db_connection(func: Callable[[mdb.Connection], Any]) -> Any: + """Call `func` with a MySQDdb database connection.""" + with database_connection(app.config["SQL_URI"]) as conn: + return func(conn) + + +def with_redis_connection(func: Callable[[Redis], Any]) -> Any: + """Call `func` with a redis connection.""" + redisuri = app.config["REDIS_URL"] + with Redis.from_url(redisuri, decode_responses=True) as rconn: + return func(rconn) diff --git a/uploader/default_settings.py b/uploader/default_settings.py new file mode 100644 index 0000000..1acb247 --- /dev/null +++ b/uploader/default_settings.py @@ -0,0 +1,20 @@ +""" +The default configuration file. The values here should be overridden in the +actual configuration file used for the production and staging systems. +""" +LOG_LEVEL = "WARNING" +SECRET_KEY = b"<Please! Please! Please! Change This!>" +UPLOAD_FOLDER = "/tmp/qc_app_files" +REDIS_URL = "redis://" +JOBS_TTL_SECONDS = 1209600 # 14 days +GNQC_REDIS_PREFIX="gn-uploader" +SQL_URI = "" + +GN2_SERVER_URL = "https://genenetwork.org/" + +SESSION_TYPE = "redis" +SESSION_PERMANENT = True +SESSION_USE_SIGNER = True + +JWKS_ROTATION_AGE_DAYS = 7 # Days (from creation) to keep a JWK in use. +JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before deleting it. diff --git a/uploader/errors.py b/uploader/errors.py new file mode 100644 index 0000000..3e7c893 --- /dev/null +++ b/uploader/errors.py @@ -0,0 +1,29 @@ +"""Application error handling.""" +import traceback +from werkzeug.exceptions import HTTPException + +import MySQLdb as mdb +from flask import Flask, request, render_template, current_app as app + +def handle_general_exception(exc: Exception): + """Handle generic exceptions.""" + trace = traceback.format_exc() + app.logger.error( + "Error (%s.%s): Generic unhandled exception!! (URI: %s)\n%s", + exc.__class__.__module__, exc.__class__.__name__, request.url, trace) + return render_template("unhandled_exception.html", trace=trace), 500 + +def handle_http_exception(exc: HTTPException): + """Handle HTTP exceptions.""" + app.logger.error( + "HTTP Error %s: %s", exc.code, exc.description, exc_info=True) + return render_template("http-error.html", + request_url=request.url, + exc=exc, + trace=traceback.format_exception(exc)), exc.code + +def register_error_handlers(appl: Flask): + """Register top-level error/exception handlers.""" + appl.register_error_handler(Exception, handle_general_exception) + appl.register_error_handler(HTTPException, handle_http_exception) + appl.register_error_handler(mdb.MySQLError, handle_general_exception) diff --git a/uploader/expression_data/__init__.py b/uploader/expression_data/__init__.py new file mode 100644 index 0000000..fc8bd41 --- /dev/null +++ b/uploader/expression_data/__init__.py @@ -0,0 +1,2 @@ +"""Package handling upload of files.""" +from .views import exprdatabp diff --git a/uploader/expression_data/dbinsert.py b/uploader/expression_data/dbinsert.py new file mode 100644 index 0000000..6d8ce80 --- /dev/null +++ b/uploader/expression_data/dbinsert.py @@ -0,0 +1,400 @@ +"Handle inserting data into the database" +import os +import json +from typing import Union +from functools import reduce +from datetime import datetime + +from redis import Redis +from MySQLdb.cursors import DictCursor +from gn_libs.mysqldb import database_connection +from flask import ( + flash, request, url_for, Blueprint, redirect, render_template, + current_app as app) + +from uploader import jobs +from uploader.authorisation import require_login +from uploader.db_utils import with_db_connection +from uploader.population.models import populations_by_species +from uploader.species.models import all_species, species_by_id +from uploader.platforms.models import platform_by_species_and_id + +dbinsertbp = Blueprint("dbinsert", __name__) + +def render_error(error_msg): + "Render the generic error page" + return render_template("dbupdate_error.html", error_message=error_msg), 400 + +def make_menu_items_grouper(grouping_fn=lambda item: item): + "Build function to be used to group menu items." + def __grouper__(acc, row): + grouping = grouping_fn(row[2]) + row_values = (row[0].strip(), row[1].strip()) + if acc.get(grouping) is None: + return {**acc, grouping: (row_values,)} + return {**acc, grouping: (acc[grouping] + (row_values,))} + return __grouper__ + +def genechips(): + "Retrieve the genechip information from the database" + def __organise_by_species__(acc, chip): + speciesid = chip["SpeciesId"] + if acc.get(speciesid) is None: + return {**acc, speciesid: (chip,)} + return {**acc, speciesid: acc[speciesid] + (chip,)} + + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GeneChip ORDER BY GeneChipName ASC") + return reduce(__organise_by_species__, cursor.fetchall(), {}) + + return {} + + +def studies_by_species_and_platform(speciesid:int, genechipid:int) -> tuple: + "Retrieve the studies by the related species and gene platform" + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + query = ( + "SELECT Species.SpeciesId, ProbeFreeze.* " + "FROM Species INNER JOIN InbredSet " + "ON Species.SpeciesId=InbredSet.SpeciesId " + "INNER JOIN ProbeFreeze " + "ON InbredSet.InbredSetId=ProbeFreeze.InbredSetId " + "WHERE Species.SpeciesId = %s " + "AND ProbeFreeze.ChipId = %s") + cursor.execute(query, (speciesid, genechipid)) + return tuple(cursor.fetchall()) + + return tuple() + +def organise_groups_by_family(acc:dict, group:dict) -> dict: + "Organise the group (InbredSet) information by the group field" + family = group["Family"] + if acc.get(family): + return {**acc, family: acc[family] + (group,)} + return {**acc, family: (group,)} + +def tissues() -> tuple: + "Retrieve type (Tissue) information from the database." + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Tissue ORDER BY Name") + return tuple(cursor.fetchall()) + + return tuple() + +@dbinsertbp.route("/platform", methods=["POST"]) +@require_login +def select_platform(): + "Select the platform (GeneChipId) used for the data." + job_id = request.form["job_id"] + with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, + database_connection(app.config["SQL_URI"]) as conn): + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + if job: + filename = job["filename"] + filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + if os.path.exists(filepath): + default_species = 1 + gchips = genechips() + return render_template( + "select_platform.html", filename=filename, + filetype=job["filetype"], totallines=int(job["currentline"]), + default_species=default_species, species=all_species(conn), + genechips=gchips[default_species], + genechips_data=json.dumps(gchips)) + return render_error(f"File '{filename}' no longer exists.") + return render_error(f"Job '{job_id}' no longer exists.") + return render_error("Unknown error") + +@dbinsertbp.route("/study", methods=["POST"]) +@require_login +def select_study(): + "View to select/create the study (ProbeFreeze) associated with the data." + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + + speciesid = form["species"] + genechipid = form["genechipid"] + + the_studies = studies_by_species_and_platform(speciesid, genechipid) + the_groups = reduce( + organise_groups_by_family, + with_db_connection( + lambda conn: populations_by_species(conn, speciesid)), + {}) + return render_template( + "select_study.html", filename=form["filename"], + filetype=form["filetype"], totallines=form["totallines"], + species=speciesid, genechipid=genechipid, studies=the_studies, + groups=the_groups, tissues = tissues(), + selected_group=int(form.get("inbredsetid", -13)), + selected_tissue=int(form.get("tissueid", -13))) + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/create-study", methods=["POST"]) +@require_login +def create_study(): + "Create a new study (ProbeFreeze)." + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyname"), "study name" + assert form.get("inbredsetid"), "group" + assert form.get("tissueid"), "type/tissue" + + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + values = ( + form["genechipid"], + form["tissueid"], + form["studyname"], + form.get("studyfullname", ""), + form.get("studyshortname", ""), + datetime.now().date().strftime("%Y-%m-%d"), + form["inbredsetid"]) + query = ( + "INSERT INTO ProbeFreeze(" + "ChipId, TissueId, Name, FullName, ShortName, CreateTime, " + "InbredSetId" + ") VALUES (%s, %s, %s, %s, %s, %s, %s)") + cursor.execute(query, values) + new_studyid = cursor.lastrowid + cursor.execute( + "UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s", + (new_studyid, new_studyid)) + flash("Study created successfully", "alert-success") + return render_template( + "continue_from_create_study.html", + filename=form["filename"], filetype=form["filetype"], + totallines=form["totallines"], species=form["species"], + genechipid=form["genechipid"], studyid=new_studyid) + except AssertionError as aserr: + flash(f"Missing data: {aserr.args[0]}", "alert-error") + return redirect(url_for("dbinsert.select_study"), code=307) + +def datasets_by_study(studyid:int) -> tuple: + "Retrieve datasets associated with a study with the ID `studyid`." + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + query = "SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s" + cursor.execute(query, (studyid,)) + return tuple(cursor.fetchall()) + + return tuple() + +def averaging_methods() -> tuple: + "Retrieve averaging methods from database" + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM AvgMethod") + return tuple(cursor.fetchall()) + + return tuple() + +def dataset_datascales() -> tuple: + "Retrieve datascales from database" + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor() as cursor: + cursor.execute( + 'SELECT DISTINCT DataScale FROM ProbeSetFreeze ' + 'WHERE DataScale IS NOT NULL AND DataScale != ""') + return tuple( + item for item in + (res[0].strip() for res in cursor.fetchall()) + if (item is not None and item != "")) + + return tuple() + +@dbinsertbp.route("/dataset", methods=["POST"]) +@require_login +def select_dataset(): + "Select the dataset to add the file contents against" + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + + studyid = form["studyid"] + datasets = datasets_by_study(studyid) + return render_template( + "select_dataset.html", **{**form, "studyid": studyid}, + datasets=datasets, avgmethods=averaging_methods(), + datascales=dataset_datascales()) + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/create-dataset", methods=["POST"]) +@require_login +def create_dataset(): + "Select the dataset to add the file contents against" + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + assert form.get("avgid"), "averaging method" + assert form.get("datasetname2"), "Dataset Name 2" + assert form.get("datasetfullname"), "Dataset Full Name" + assert form.get("datasetshortname"), "Dataset Short Name" + assert form.get("datasetpublic"), "Dataset public specification" + assert form.get("datasetconfidentiality"), "Dataset confidentiality" + assert form.get("datasetdatascale"), "Dataset Datascale" + + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + datasetname = form["datasetname"] + cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Name=%s", + (datasetname,)) + results = cursor.fetchall() + if bool(results): + flash("A dataset with that name already exists.", + "alert-error") + return redirect(url_for("dbinsert.select_dataset"), code=307) + values = ( + form["studyid"], form["avgid"], + datasetname, form["datasetname2"], + form["datasetfullname"], form["datasetshortname"], + datetime.now().date().strftime("%Y-%m-%d"), + form["datasetpublic"], form["datasetconfidentiality"], + "williamslab", form["datasetdatascale"]) + query = ( + "INSERT INTO ProbeSetFreeze(" + "ProbeFreezeId, AvgID, Name, Name2, FullName, " + "ShortName, CreateTime, OrderList, public, " + "confidentiality, AuthorisedUsers, DataScale) " + "VALUES" + "(%s, %s, %s, %s, %s, %s, %s, NULL, %s, %s, %s, %s)") + cursor.execute(query, values) + new_datasetid = cursor.lastrowid + return render_template( + "continue_from_create_dataset.html", + filename=form["filename"], filetype=form["filetype"], + species=form["species"], genechipid=form["genechipid"], + studyid=form["studyid"], datasetid=new_datasetid, + totallines=form["totallines"]) + except AssertionError as aserr: + flash(f"Missing data {aserr.args[0]}", "alert-error") + return redirect(url_for("dbinsert.select_dataset"), code=307) + +def study_by_id(studyid:int) -> Union[dict, None]: + "Get a study by its Id" + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT * FROM ProbeFreeze WHERE Id=%s", + (studyid,)) + return cursor.fetchone() + +def dataset_by_id(datasetid:int) -> Union[dict, None]: + "Retrieve a dataset by its id" + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + ("SELECT AvgMethod.Name AS AvgMethodName, ProbeSetFreeze.* " + "FROM ProbeSetFreeze INNER JOIN AvgMethod " + "ON ProbeSetFreeze.AvgId=AvgMethod.AvgMethodId " + "WHERE ProbeSetFreeze.Id=%s"), + (datasetid,)) + return cursor.fetchone() + +def selected_keys(original: dict, keys: tuple) -> dict: + "Return a new dict from the `original` dict with only `keys` present." + return {key: value for key,value in original.items() if key in keys} + +@dbinsertbp.route("/final-confirmation", methods=["POST"]) +@require_login +def final_confirmation(): + "Preview the data before triggering entry into the database" + with database_connection(app.config["SQL_URI"]) as conn: + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + assert form.get("datasetid"), "dataset" + + speciesid = form["species"] + genechipid = form["genechipid"] + studyid = form["studyid"] + datasetid=form["datasetid"] + return render_template( + "final_confirmation.html", filename=form["filename"], + filetype=form["filetype"], totallines=form["totallines"], + species=speciesid, genechipid=genechipid, studyid=studyid, + datasetid=datasetid, the_species=selected_keys( + with_db_connection(lambda conn: species_by_id(conn, speciesid)), + ("SpeciesName", "Name", "MenuName")), + platform=selected_keys( + platform_by_species_and_id(conn, speciesid, genechipid), + ("GeneChipName", "Name", "GeoPlatform", "Title", "GO_tree_value")), + study=selected_keys( + study_by_id(studyid), ("Name", "FullName", "ShortName")), + dataset=selected_keys( + dataset_by_id(datasetid), + ("AvgMethodName", "Name", "Name2", "FullName", "ShortName", + "DataScale"))) + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/insert-data", methods=["POST"]) +@require_login +def insert_data(): + "Trigger data insertion" + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + assert form.get("datasetid"), "dataset" + + filename = form["filename"] + filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + redisurl = app.config["REDIS_URL"] + if os.path.exists(filepath): + with Redis.from_url(redisurl, decode_responses=True) as rconn: + job = jobs.launch_job( + jobs.data_insertion_job( + rconn, filepath, form["filetype"], form["totallines"], + form["species"], form["genechipid"], form["datasetid"], + app.config["SQL_URI"], redisurl, + app.config["JOBS_TTL_SECONDS"]), + redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors") + + return redirect(url_for("dbinsert.insert_status", job_id=job["jobid"])) + return render_error(f"File '{filename}' no longer exists.") + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/status/<job_id>", methods=["GET"]) +def insert_status(job_id: str): + "Retrieve status of data insertion." + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + job_status = job["status"] + if job_status == "success": + return render_template("insert_success.html", job=job) + if job["status"] == "error": + return render_template("insert_error.html", job=job) + return render_template("insert_progress.html", job=job) + return render_template("no_such_job.html", job_id=job_id), 400 diff --git a/uploader/expression_data/views.py b/uploader/expression_data/views.py new file mode 100644 index 0000000..7629f3e --- /dev/null +++ b/uploader/expression_data/views.py @@ -0,0 +1,385 @@ +"""Views for expression data""" +import os +import uuid +import mimetypes +from typing import Tuple +from zipfile import ZipFile, is_zipfile + +import jsonpickle +from redis import Redis +from werkzeug.utils import secure_filename +from gn_libs.mysqldb import database_connection +from flask import (flash, + request, + url_for, + redirect, + Blueprint, + current_app as app) + +from quality_control.errors import InvalidValue, DuplicateHeading + +from uploader import jobs +from uploader.datautils import order_by_family +from uploader.ui import make_template_renderer +from uploader.authorisation import require_login +from uploader.db_utils import with_db_connection +from uploader.species.models import all_species, species_by_id +from uploader.population.models import (populations_by_species, + population_by_species_and_id) + +exprdatabp = Blueprint("expression-data", __name__) +render_template = make_template_renderer("expression-data") + +def isinvalidvalue(item): + """Check whether item is of type InvalidValue""" + return isinstance(item, InvalidValue) + + +def isduplicateheading(item): + """Check whether item is of type DuplicateHeading""" + return isinstance(item, DuplicateHeading) + + +def errors(rqst) -> Tuple[str, ...]: + """Return a tuple of the errors found in the request `rqst`. If no error is + found, then an empty tuple is returned.""" + def __filetype_error__(): + return ( + ("Invalid file type provided.",) + if rqst.form.get("filetype") not in ("average", "standard-error") + else tuple()) + + def __file_missing_error__(): + return ( + ("No file was uploaded.",) + if ("qc_text_file" not in rqst.files or + rqst.files["qc_text_file"].filename == "") + else tuple()) + + def __file_mimetype_error__(): + text_file = rqst.files["qc_text_file"] + return ( + ( + ("Invalid file! Expected a tab-separated-values file, or a zip " + "file of the a tab-separated-values file."),) + if text_file.mimetype not in ( + "text/plain", "text/tab-separated-values", + "application/zip") + else tuple()) + + return ( + __filetype_error__() + + (__file_missing_error__() or __file_mimetype_error__())) + + +def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]: + """Check the uploaded zip file for errors.""" + zfile_errors: Tuple[str, ...] = tuple() + if is_zipfile(filepath): + with ZipFile(filepath, "r") as zfile: + infolist = zfile.infolist() + if len(infolist) != 1: + zfile_errors = zfile_errors + ( + ("Expected exactly one (1) member file within the uploaded zip " + f"file. Got {len(infolist)} member files."),) + if len(infolist) == 1 and infolist[0].is_dir(): + zfile_errors = zfile_errors + ( + ("Expected a member text file in the uploaded zip file. Got a " + "directory/folder."),) + + if len(infolist) == 1 and not infolist[0].is_dir(): + zfile.extract(infolist[0], path=upload_dir) + mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}") + if mime[0] != "text/tab-separated-values": + zfile_errors = zfile_errors + ( + ("Expected the member text file in the uploaded zip file to" + " be a tab-separated file."),) + + return zfile_errors + + +@exprdatabp.route("populations/expression-data", methods=["GET"]) +@require_login +def index(): + """Display the expression data index page.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(request.args.get("species_id")): + return render_template("expression-data/index.html", + species=order_by_family(all_species(conn)), + activelink="expression-data") + species = species_by_id(conn, request.args.get("species_id")) + if not bool(species): + flash("Could not find species selected!", "alert-danger") + return redirect(url_for("species.populations.expression-data.index")) + return redirect(url_for( + "species.populations.expression-data.select_population", + species_id=species["SpeciesId"])) + + +@exprdatabp.route("<int:species_id>/populations/expression-data/select-population", + methods=["GET"]) +@require_login +def select_population(species_id: int): + """Select the expression data's population.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("No such species!", "alert-danger") + return redirect(url_for("species.populations.expression-data.index")) + + if not bool(request.args.get("population_id")): + return render_template("expression-data/select-population.html", + species=species, + populations=order_by_family( + populations_by_species(conn, species_id), + order_key="FamilyOrder"), + activelink="expression-data") + + population = population_by_species_and_id( + conn, species_id, request.args.get("population_id")) + if not bool(population): + flash("No such population!", "alert-danger") + return redirect(url_for( + "species.populations.expression-data.select_population", + species_id=species_id)) + + return redirect(url_for("species.populations.expression-data.upload_file", + species_id=species_id, + population_id=population["Id"])) + + +@exprdatabp.route("<int:species_id>/populations/<int:population_id>/" + "expression-data/upload", + methods=["GET", "POST"]) +@require_login +def upload_file(species_id: int, population_id: int): + """Enables uploading the files""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + population = population_by_species_and_id(conn, species_id, population_id) + if request.method == "GET": + return render_template("expression-data/select-file.html", + species=species, + population=population) + + upload_dir = app.config["UPLOAD_FOLDER"] + request_errors = errors(request) + if request_errors: + for error in request_errors: + flash(error, "alert-danger error-expr-data") + return redirect(url_for("species.populations.expression-data.upload_file")) + + filename = secure_filename( + request.files["qc_text_file"].filename)# type: ignore[arg-type] + if not os.path.exists(upload_dir): + os.mkdir(upload_dir) + + filepath = os.path.join(upload_dir, filename) + request.files["qc_text_file"].save(os.path.join(upload_dir, filename)) + + zip_errors = zip_file_errors(filepath, upload_dir) + if zip_errors: + for error in zip_errors: + flash(error, "alert-danger error-expr-data") + return redirect(url_for("species.populations.expression-data.index.upload_file")) + + return redirect(url_for("species.populations.expression-data.parse_file", + species_id=species_id, + population_id=population_id, + filename=filename, + filetype=request.form["filetype"])) + + +@exprdatabp.route("/data-review", methods=["GET"]) +@require_login +def data_review(): + """Provide some help on data expectations to the user.""" + return render_template("expression-data/data-review.html") + + +@exprdatabp.route( + "<int:species_id>/populations/<int:population_id>/expression-data/parse", + methods=["GET"]) +@require_login +def parse_file(species_id: int, population_id: int): + """Trigger file parsing""" + _errors = False + filename = request.args.get("filename") + filetype = request.args.get("filetype") + + species = with_db_connection(lambda con: species_by_id(con, species_id)) + if not bool(species): + flash("No such species.", "alert-danger") + _errors = True + + if filename is None: + flash("No file provided", "alert-danger") + _errors = True + + if filetype is None: + flash("No filetype provided", "alert-danger") + _errors = True + + if filetype not in ("average", "standard-error"): + flash("Invalid filetype provided", "alert-danger") + _errors = True + + if filename: + filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename) + if not os.path.exists(filepath): + flash("Selected file does not exist (any longer)", "alert-danger") + _errors = True + + if _errors: + return redirect(url_for("species.populations.expression-data.upload_file")) + + redisurl = app.config["REDIS_URL"] + with Redis.from_url(redisurl, decode_responses=True) as rconn: + job = jobs.launch_job( + jobs.build_file_verification_job( + rconn, app.config["SQL_URI"], redisurl, + species_id, filepath, filetype,# type: ignore[arg-type] + app.config["JOBS_TTL_SECONDS"]), + redisurl, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + + return redirect(url_for("species.populations.expression-data.parse_status", + species_id=species_id, + population_id=population_id, + job_id=job["jobid"])) + + +@exprdatabp.route( + "<int:species_id>/populations/<int:population_id>/expression-data/parse/" + "status/<uuid:job_id>", + methods=["GET"]) +@require_login +def parse_status(species_id: int, population_id: int, job_id: str): + "Retrieve the status of the job" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + except jobs.JobNotFound as _exc: + return render_template("no_such_job.html", job_id=job_id), 400 + + error_filename = jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + if os.path.exists(error_filename): + stat = os.stat(error_filename) + if stat.st_size > 0: + return redirect(url_for("parse.fail", job_id=job_id)) + + job_id = job["jobid"] + progress = float(job["percent"]) + status = job["status"] + filename = job.get("filename", "uploaded file") + _errors = jsonpickle.decode( + job.get("errors", jsonpickle.encode(tuple()))) + if status in ("success", "aborted"): + return redirect(url_for("species.populations.expression-data.results", + species_id=species_id, + population_id=population_id, + job_id=job_id)) + + if status == "parse-error": + return redirect(url_for("species.populations.expression-data.fail", job_id=job_id)) + + app.jinja_env.globals.update( + isinvalidvalue=isinvalidvalue, + isduplicateheading=isduplicateheading) + return render_template( + "expression-data/job-progress.html", + job_id = job_id, + job_status = status, + progress = progress, + message = job.get("message", ""), + job_name = f"Parsing '{filename}'", + errors=_errors, + species=with_db_connection( + lambda conn: species_by_id(conn, species_id)), + population=with_db_connection( + lambda conn: population_by_species_and_id( + conn, species_id, population_id))) + + +@exprdatabp.route( + "<int:species_id>/populations/<int:population_id>/expression-data/parse/" + "<uuid:job_id>/results", + methods=["GET"]) +@require_login +def results(species_id: int, population_id: int, job_id: uuid.UUID): + """Show results of parsing...""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + filename = job["filename"] + _errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple()))) + app.jinja_env.globals.update( + isinvalidvalue=isinvalidvalue, + isduplicateheading=isduplicateheading) + return render_template( + "expression-data/parse-results.html", + errors=_errors, + job_name = f"Parsing '{filename}'", + user_aborted = job.get("user_aborted"), + job_id=job["jobid"], + species=with_db_connection( + lambda conn: species_by_id(conn, species_id)), + population=with_db_connection( + lambda conn: population_by_species_and_id( + conn, species_id, population_id))) + + return render_template("expression-data/no-such-job.html", job_id=job_id) + + +@exprdatabp.route( + "<int:species_id>/populations/<int:population_id>/expression-data/parse/" + "<uuid:job_id>/fail", + methods=["GET"]) +@require_login +def fail(species_id: int, population_id: int, job_id: str): + """Handle parsing failure""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + error_filename = jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + if os.path.exists(error_filename): + stat = os.stat(error_filename) + if stat.st_size > 0: + return render_template( + "worker_failure.html", job_id=job_id) + + return render_template("parse_failure.html", job=job) + + return render_template("expression-data/no-such-job.html", + **with_db_connection(lambda conn: { + "species_id": species_by_id(conn, species_id), + "population_id": population_by_species_and_id( + conn, species_id, population_id)}), + job_id=job_id) + + +@exprdatabp.route( + "<int:species_id>/populations/<int:population_id>/expression-data/parse/" + "abort", + methods=["POST"]) +@require_login +def abort(species_id: int, population_id: int): + """Handle user request to abort file processing""" + job_id = request.form["job_id"] + + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id), + key="user_aborted", + value=int(True)) + + return redirect(url_for("species.populations.expression-data.parse_status", + species_id=species_id, + population_id=population_id, + job_id=job_id)) diff --git a/uploader/files/__init__.py b/uploader/files/__init__.py new file mode 100644 index 0000000..53c3176 --- /dev/null +++ b/uploader/files/__init__.py @@ -0,0 +1,5 @@ +"""General files and chunks utilities.""" +from .chunks import chunked_binary_read +from .functions import (fullpath, + save_file, + sha256_digest_over_file) diff --git a/uploader/files/chunks.py b/uploader/files/chunks.py new file mode 100644 index 0000000..c4360b5 --- /dev/null +++ b/uploader/files/chunks.py @@ -0,0 +1,32 @@ +"""Functions dealing with chunking of files.""" +from pathlib import Path +from typing import Iterator + +from flask import current_app as app +from werkzeug.utils import secure_filename + + +def chunked_binary_read(filepath: Path, chunksize: int = 2048) -> Iterator: + """Read a file in binary mode in chunks.""" + with open(filepath, "rb") as inputfile: + while True: + data = inputfile.read(chunksize) + if data != b"": + yield data + continue + break + +def chunk_name(uploadfilename: str, chunkno: int) -> str: + """Generate chunk name from original filename and chunk number""" + if uploadfilename == "": + raise ValueError("Name cannot be empty!") + if chunkno < 1: + raise ValueError("Chunk number must be greater than zero") + return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}" + + +def chunks_directory(uniqueidentifier: str) -> Path: + """Compute the directory where chunks are temporarily stored.""" + if uniqueidentifier == "": + raise ValueError("Unique identifier cannot be empty!") + return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}") diff --git a/uploader/files/functions.py b/uploader/files/functions.py new file mode 100644 index 0000000..7b9f06b --- /dev/null +++ b/uploader/files/functions.py @@ -0,0 +1,42 @@ +"""Utilities to deal with uploaded files.""" +import hashlib +from pathlib import Path +from datetime import datetime + +from flask import current_app + +from werkzeug.utils import secure_filename +from werkzeug.datastructures import FileStorage + +from .chunks import chunked_binary_read + +def save_file(fileobj: FileStorage, upload_dir: Path, hashed: bool = True) -> Path: + """Save the uploaded file and return the path.""" + assert bool(fileobj), "Invalid file object!" + hashed_name = ( + hashlib.sha512( + f"{fileobj.filename}::{datetime.now().isoformat()}".encode("utf8") + ).hexdigest() + if hashed else + fileobj.filename) + filename = Path(secure_filename(hashed_name)) # type: ignore[arg-type] + if not upload_dir.exists(): + upload_dir.mkdir() + + filepath = Path(upload_dir, filename) + fileobj.save(filepath) + return filepath + + +def fullpath(filename: str): + """Get a file's full path. This makes use of `flask.current_app`.""" + return Path(current_app.config["UPLOAD_FOLDER"], filename).absolute() + + +def sha256_digest_over_file(filepath: Path) -> str: + """Compute the sha256 digest over a file's contents.""" + filehash = hashlib.sha256() + for chunk in chunked_binary_read(filepath): + filehash.update(chunk) + + return filehash.hexdigest() diff --git a/uploader/files/views.py b/uploader/files/views.py new file mode 100644 index 0000000..8d81654 --- /dev/null +++ b/uploader/files/views.py @@ -0,0 +1,116 @@ +"""Module for generic files endpoints.""" +import traceback +from pathlib import Path + +from flask import request, jsonify, Blueprint, current_app as app + +from .chunks import chunk_name, chunks_directory + +files = Blueprint("files", __name__) + +def target_file(fileid: str) -> Path: + """Compute the full path for the target file.""" + return Path(app.config["UPLOAD_FOLDER"], fileid) + + +@files.route("/upload/resumable", methods=["GET"]) +def resumable_upload_get(): + """Used for checking whether **ALL** chunks have been uploaded.""" + fileid = request.args.get("resumableIdentifier", type=str) or "" + filename = request.args.get("resumableFilename", type=str) or "" + chunk = request.args.get("resumableChunkNumber", type=int) or 0 + if not(fileid or filename or chunk): + return jsonify({ + "message": "At least one required query parameter is missing.", + "error": "BadRequest", + "statuscode": 400 + }), 400 + + # If the complete target file exists, return 200 for all chunks. + _targetfile = target_file(fileid) + if _targetfile.exists(): + return jsonify({ + "uploaded-file": _targetfile.name, + "original-name": filename, + "chunk": chunk, + "message": "The complete file already exists.", + "statuscode": 200 + }), 200 + + if Path(chunks_directory(fileid), + chunk_name(filename, chunk)).exists(): + return jsonify({ + "chunk": chunk, + "message": f"Chunk {chunk} exists.", + "statuscode": 200 + }), 200 + + return jsonify({ + "message": f"Chunk {chunk} was not found.", + "error": "NotFound", + "statuscode": 404 + }), 404 + + +def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path: + """Merge the chunks into a single file.""" + with open(targetfile, "ab") as _target: + for chunkfile in chunkpaths: + with open(chunkfile, "rb") as _chunkdata: + _target.write(_chunkdata.read()) + + chunkfile.unlink() + return targetfile + + +@files.route("/upload/resumable", methods=["POST"]) +def resumable_upload_post(): + """Do the actual chunks upload here.""" + _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0 + _chunk = request.form.get("resumableChunkNumber", default=1, type=int) + _uploadfilename = request.form.get( + "resumableFilename", default="", type=str) or "" + _fileid = request.form.get( + "resumableIdentifier", default="", type=str) or "" + _targetfile = target_file(_fileid) + + if _targetfile.exists(): + return jsonify({ + "uploaded-file": _targetfile.name, + "original-name": _uploadfilename, + "message": "File was uploaded successfully!", + "statuscode": 200 + }), 200 + + try: + chunks_directory(_fileid).mkdir(exist_ok=True, parents=True) + request.files["file"].save(Path(chunks_directory(_fileid), + chunk_name(_uploadfilename, _chunk))) + + # Check whether upload is complete + chunkpaths = tuple( + Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk)) + for _achunk in range(1, _totalchunks+1)) + if all(_file.exists() for _file in chunkpaths): + # merge_files and clean up chunks + __merge_chunks__(_targetfile, chunkpaths) + chunks_directory(_fileid).rmdir() + return jsonify({ + "uploaded-file": _targetfile.name, + "original-name": _uploadfilename, + "message": "File was uploaded successfully!", + "statuscode": 200 + }), 200 + return jsonify({ + "message": f"Chunk {int(_chunk)} uploaded successfully.", + "statuscode": 201 + }), 201 + except Exception as exc:# pylint: disable=[broad-except] + msg = "Error processing uploaded file chunks." + app.logger.error(msg, exc_info=True, stack_info=True) + return jsonify({ + "message": msg, + "error": type(exc).__name__, + "error-description": " ".join(str(arg) for arg in exc.args), + "error-trace": traceback.format_exception(exc) + }), 500 diff --git a/uploader/genotypes/__init__.py b/uploader/genotypes/__init__.py new file mode 100644 index 0000000..d0025d6 --- /dev/null +++ b/uploader/genotypes/__init__.py @@ -0,0 +1 @@ +"""The Genotypes module.""" diff --git a/uploader/genotypes/models.py b/uploader/genotypes/models.py new file mode 100644 index 0000000..4c3e634 --- /dev/null +++ b/uploader/genotypes/models.py @@ -0,0 +1,102 @@ +"""Functions for handling genotypes.""" +from typing import Optional +from datetime import datetime + +import MySQLdb as mdb +from MySQLdb.cursors import Cursor, DictCursor +from flask import current_app as app + +from gn_libs.mysqldb import debug_query + +def genocode_by_population( + conn: mdb.Connection, population_id: int) -> tuple[dict, ...]: + """Get the allele/genotype codes.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GenoCode WHERE InbredSetId=%s", + (population_id,)) + return tuple(dict(item) for item in cursor.fetchall()) + + +def genotype_markers_count(conn: mdb.Connection, species_id: int) -> int: + """Find the total count of the genotype markers for a species.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT COUNT(Name) AS markers_count FROM Geno WHERE SpeciesId=%s", + (species_id,)) + return int(cursor.fetchone()["markers_count"]) + + +def genotype_markers( + conn: mdb.Connection, + species_id: int, + offset: int = 0, + limit: Optional[int] = None +) -> tuple[dict, ...]: + """Retrieve markers from the database.""" + _query = "SELECT * FROM Geno WHERE SpeciesId=%s" + if bool(limit) and limit > 0:# type: ignore[operator] + _query = _query + f" LIMIT {limit} OFFSET {offset}" + + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(_query, (species_id,)) + debug_query(cursor, app.logger) + return tuple(dict(row) for row in cursor.fetchall()) + + +def genotype_dataset( + conn: mdb.Connection, + species_id: int, + population_id: int, + dataset_id: Optional[int] = None +) -> Optional[dict]: + """Retrieve genotype datasets from the database. + + Apparently, you should only ever have one genotype dataset for a population. + """ + _query = ( + "SELECT gf.* FROM Species AS s INNER JOIN InbredSet AS iset " + "ON s.Id=iset.SpeciesId INNER JOIN GenoFreeze AS gf " + "ON iset.Id=gf.InbredSetId " + "WHERE s.Id=%s AND iset.Id=%s") + _params = (species_id, population_id) + if bool(dataset_id): + _query = _query + " AND gf.Id=%s" + _params = _params + (dataset_id,)# type: ignore[assignment] + + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(_query, _params) + debug_query(cursor, app.logger) + result = cursor.fetchone() + if bool(result): + return dict(result) + return None + + +def save_new_dataset( + cursor: Cursor, + population_id: int, + name: str, + fullname: str, + shortname: str +) -> dict: + """Save a new genotype dataset into the database.""" + params = { + "InbredSetId": population_id, + "Name": name, + "FullName": fullname, + "ShortName": shortname, + "CreateTime": datetime.now().date().isoformat(), + "public": 2, + "confidentiality": 0, + "AuthorisedUsers": None + } + cursor.execute( + "INSERT INTO GenoFreeze(" + "Name, FullName, ShortName, CreateTime, public, InbredSetId, " + "confidentiality, AuthorisedUsers" + ") VALUES (" + "%(Name)s, %(FullName)s, %(ShortName)s, %(CreateTime)s, %(public)s, " + "%(InbredSetId)s, %(confidentiality)s, %(AuthorisedUsers)s" + ")", + params) + return {**params, "Id": cursor.lastrowid} diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py new file mode 100644 index 0000000..5105730 --- /dev/null +++ b/uploader/genotypes/views.py @@ -0,0 +1,225 @@ +"""Views for the genotypes.""" +from MySQLdb.cursors import DictCursor +from gn_libs.mysqldb import database_connection +from flask import (flash, + request, + url_for, + redirect, + Blueprint, + render_template, + current_app as app) + +from uploader.ui import make_template_renderer +from uploader.oauth2.client import oauth2_post +from uploader.authorisation import require_login +from uploader.species.models import all_species, species_by_id +from uploader.monadic_requests import make_either_error_handler +from uploader.request_checks import with_species, with_population +from uploader.datautils import safe_int, order_by_family, enumerate_sequence +from uploader.population.models import (populations_by_species, + population_by_species_and_id) + +from .models import (genotype_markers, + genotype_dataset, + save_new_dataset, + genotype_markers_count, + genocode_by_population) + +genotypesbp = Blueprint("genotypes", __name__) +render_template = make_template_renderer("genotypes") + +@genotypesbp.route("populations/genotypes", methods=["GET"]) +@require_login +def index(): + """Direct entry-point for genotypes.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(request.args.get("species_id")): + return render_template("genotypes/index.html", + species=all_species(conn), + activelink="genotypes") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.genotypes.select_population")) + + species = species_by_id(conn, request.args.get("species_id")) + if not bool(species): + flash(f"Could not find species with ID '{request.args.get('species_id')}'!", + "alert-danger") + return redirect(url_for("species.populations.genotypes.index")) + return redirect(url_for("species.populations.genotypes.select_population", + species_id=species["SpeciesId"])) + + +@genotypesbp.route("/<int:species_id>/populations/genotypes/select-population", + methods=["GET"]) +@require_login +@with_species(redirect_uri="species.populations.genotypes.index") +def select_population(species: dict, species_id: int): + """Select the population under which the genotypes go.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(request.args.get("population_id")): + return render_template("genotypes/select-population.html", + species=species, + populations=populations_by_species( + conn, species_id), + activelink="genotypes") + + population_id = request.args["population_id"] + if population_id == "CREATE-POPULATION": + return redirect(url_for( + "species.populations.create_population", + species_id=species["SpeciesId"], + return_to="species.populations.samples.list_genotypes")) + + population = population_by_species_and_id( + conn, species_id, request.args.get("population_id")) + if not bool(population): + flash("Invalid population selected!", "alert-danger") + return redirect(url_for( + "species.populations.genotypes.select_population", + species_id=species_id)) + + return redirect(url_for("species.populations.genotypes.list_genotypes", + species_id=species_id, + population_id=population["Id"])) + + +@genotypesbp.route( + "/<int:species_id>/populations/<int:population_id>/genotypes", + methods=["GET"]) +@require_login +@with_population(species_redirect_uri="species.populations.genotypes.index", + redirect_uri="species.populations.genotypes.select_population") +def list_genotypes(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] + """List genotype details for species and population.""" + with database_connection(app.config["SQL_URI"]) as conn: + return render_template("genotypes/list-genotypes.html", + species=species, + population=population, + genocode=genocode_by_population( + conn, population["Id"]), + total_markers=genotype_markers_count( + conn, species["SpeciesId"]), + dataset=genotype_dataset(conn, + species["SpeciesId"], + population["Id"]), + activelink="list-genotypes") + + +@genotypesbp.route( + "/<int:species_id>/populations/<int:population_id>/genotypes/list-markers", + methods=["GET"]) +@require_login +@with_population(species_redirect_uri="species.populations.genotypes.index", + redirect_uri="species.populations.genotypes.select_population") +def list_markers( + species: dict, + population: dict, + **kwargs +):# pylint: disable=[unused-argument] + """List a species' genetic markers.""" + with database_connection(app.config["SQL_URI"]) as conn: + start_from = max(safe_int(request.args.get("start_from") or 0), 0) + count = safe_int(request.args.get("count") or 20) + return render_template("genotypes/list-markers.html", + species=species, + population=population, + total_markers=genotype_markers_count( + conn, species["SpeciesId"]), + start_from=start_from, + count=count, + markers=enumerate_sequence( + genotype_markers(conn, + species["SpeciesId"], + offset=start_from, + limit=count), + start=start_from+1), + activelink="list-markers") + +@genotypesbp.route( + "/<int:species_id>/populations/<int:population_id>/genotypes/datasets/" + "<int:dataset_id>/view", + methods=["GET"]) +@require_login +def view_dataset(species_id: int, population_id: int, dataset_id: int): + """View details regarding a specific dataset.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("Invalid species provided!", "alert-danger") + return redirect(url_for("species.populations.genotypes.index")) + + population = population_by_species_and_id( + conn, species_id, population_id) + if not bool(population): + flash("Invalid population selected!", "alert-danger") + return redirect(url_for( + "species.populations.genotypes.select_population", + species_id=species_id)) + + dataset = genotype_dataset(conn, species_id, population_id, dataset_id) + if not bool(dataset): + flash("Could not find such a dataset!", "alert-danger") + return redirect(url_for( + "species.populations.genotypes.list_genotypes", + species_id=species_id, + population_id=population_id)) + + return render_template("genotypes/view-dataset.html", + species=species, + population=population, + dataset=dataset, + activelink="view-dataset") + + +@genotypesbp.route( + "/<int:species_id>/populations/<int:population_id>/genotypes/datasets/" + "create", + methods=["GET", "POST"]) +@require_login +@with_population(species_redirect_uri="species.populations.genotypes.index", + redirect_uri="species.populations.genotypes.select_population") +def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] + """Create a genotype dataset.""" + with (database_connection(app.config["SQL_URI"]) as conn, + conn.cursor(cursorclass=DictCursor) as cursor): + if request.method == "GET": + return render_template("genotypes/create-dataset.html", + species=species, + population=population, + activelink="create-dataset") + + form = request.form + new_dataset = save_new_dataset( + cursor, + population["Id"], + form["geno-dataset-name"], + form["geno-dataset-fullname"], + form["geno-dataset-shortname"]) + + def __success__(_success): + flash("Successfully created genotype dataset.", "alert-success") + return redirect(url_for( + "species.populations.genotypes.list_genotypes", + species_id=species["SpeciesId"], + population_id=population["Id"])) + + return oauth2_post( + "auth/resource/genotypes/create", + json={ + **dict(request.form), + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": new_dataset["Id"], + "dataset_name": form["geno-dataset-name"], + "dataset_fullname": form["geno-dataset-fullname"], + "dataset_shortname": form["geno-dataset-shortname"], + "public": "on" + } + ).either( + make_either_error_handler( + "There was an error creating the genotype dataset."), + __success__) diff --git a/uploader/input_validation.py b/uploader/input_validation.py new file mode 100644 index 0000000..627c69e --- /dev/null +++ b/uploader/input_validation.py @@ -0,0 +1,71 @@ +"""Input validation utilities""" +import re +import json +import base64 +from typing import Any + +def is_empty_string(value: str) -> bool: + """Check whether as string is empty""" + return (isinstance(value, str) and value.strip() == "") + + +def is_empty_input(value: Any) -> bool: + """Check whether user provided an empty value.""" + return (value is None or is_empty_string(value)) + + +def is_integer_input(value: Any) -> bool: + """ + Check whether user provided a value that can be parsed into an integer. + """ + def __is_int__(val, base): + try: + int(val, base=base) + except ValueError: + return False + return True + return isinstance(value, int) or ( + (not is_empty_input(value)) and ( + isinstance(value, str) and ( + __is_int__(value, 10) + or __is_int__(value, 8) + or __is_int__(value, 16)))) + + +def is_valid_representative_name(repr_name: str) -> bool: + """ + Check whether the given representative name is a valid according to our rules. + + Parameters + ---------- + repr_name: a string of characters. + + Checks For + ---------- + * The name MUST start with an alphabet [a-zA-Z] + * The name MUST end with an alphabet [a-zA-Z] or number [0-9] + * The name MUST be composed of alphabets [a-zA-Z], numbers [0-9], + underscores (_) and/or hyphens (-). + + Returns + ------- + Boolean indicating whether or not the name is valid. + """ + pattern = re.compile(r"^[a-zA-Z]+[a-zA-Z0-9_-]*[a-zA-Z0-9]$") + return bool(pattern.match(repr_name)) + + +def encode_errors(errors: tuple[tuple[str, str], ...], form) -> bytes: + """Encode form errors into base64 string.""" + return base64.b64encode( + json.dumps({ + "errors": dict(errors), + "original_formdata": dict(form) + }).encode("utf8")) + + +def decode_errors(errorstr) -> dict[str, dict]: + """Decode errors from base64 string""" + if not bool(errorstr): + return {"errors": {}, "original_formdata": {}} + return json.loads(base64.b64decode(errorstr.encode("utf8")).decode("utf8")) diff --git a/uploader/jobs.py b/uploader/jobs.py new file mode 100644 index 0000000..e86ee05 --- /dev/null +++ b/uploader/jobs.py @@ -0,0 +1,164 @@ +"""Handle jobs""" +import os +import sys +import uuid +import json +import shlex +import subprocess +from uuid import UUID, uuid4 +from datetime import timedelta +from typing import Union, Optional + +from redis import Redis +from flask import current_app as app + +from functional_tools import take + +JOBS_PREFIX = "jobs" + +class JobNotFound(Exception): + """Raised if we try to retrieve a non-existent job.""" + +def jobsnamespace(): + """ + Return the jobs namespace prefix. It depends on app configuration. + + Calling this function outside of an application context will cause an + exception to be raised. It is mostly a convenience utility to use within the + application. + """ + return f"{app.config['GNQC_REDIS_PREFIX']}:{JOBS_PREFIX}" + +def job_key(namespaceprefix: str, jobid: Union[str, UUID]) -> str: + """Build the key by appending it to the namespace prefix.""" + return f"{namespaceprefix}:{jobid}" + +def raise_jobnotfound(rprefix:str, jobid: Union[str,UUID]): + """Utility to raise a `NoSuchJobError`""" + raise JobNotFound(f"Could not retrieve job '{jobid}' from '{rprefix}.") + +def error_filename(jobid, error_dir): + "Compute the path of the file where errors will be dumped." + return f"{error_dir}/job_{jobid}.error" + +def initialise_job(# pylint: disable=[too-many-arguments] + rconn: Redis, rprefix: str, jobid: str, command: list, job_type: str, + ttl_seconds: int = 86400, extra_meta: Optional[dict] = None) -> dict: + "Initialise a job 'object' and put in on redis" + the_job = { + "jobid": jobid, "command": shlex.join(command), "status": "pending", + "percent": 0, "job-type": job_type, **(extra_meta or {}) + } + rconn.hset(job_key(rprefix, jobid), mapping=the_job) + rconn.expire( + name=job_key(rprefix, jobid), time=timedelta(seconds=ttl_seconds)) + return the_job + +def build_file_verification_job(#pylint: disable=[too-many-arguments] + redis_conn: Redis, + dburi: str, + redisuri: str, + speciesid: int, + filepath: str, + filetype: str, + ttl_seconds: int): + "Build a file verification job" + jobid = str(uuid4()) + command = [ + sys.executable, "-m", "scripts.validate_file", + dburi, redisuri, jobsnamespace(), jobid, + "--redisexpiry", str(ttl_seconds), + str(speciesid), filetype, filepath, + ] + return initialise_job( + redis_conn, jobsnamespace(), jobid, command, "file-verification", + ttl_seconds, { + "filetype": filetype, + "filename": os.path.basename(filepath), "percent": 0 + }) + +def data_insertion_job(# pylint: disable=[too-many-arguments] + redis_conn: Redis, filepath: str, filetype: str, totallines: int, + speciesid: int, platformid: int, datasetid: int, databaseuri: str, + redisuri: str, ttl_seconds: int) -> dict: + "Build a data insertion job" + jobid = str(uuid4()) + command = [ + sys.executable, "-m", "scripts.insert_data", filetype, filepath, + speciesid, platformid, datasetid, databaseuri, redisuri + ] + return initialise_job( + redis_conn, jobsnamespace(), jobid, command, "data-insertion", + ttl_seconds, { + "filename": os.path.basename(filepath), "filetype": filetype, + "totallines": totallines + }) + +def launch_job(the_job: dict, redisurl: str, error_dir): + """Launch a job in the background""" + if not os.path.exists(error_dir): + os.mkdir(error_dir) + + jobid = the_job["jobid"] + with open(error_filename(jobid, error_dir), + "w", + encoding="utf-8") as errorfile: + subprocess.Popen( # pylint: disable=[consider-using-with] + [sys.executable, "-m", "scripts.worker", redisurl, jobsnamespace(), + jobid], + stderr=errorfile, + env={"PYTHONPATH": ":".join(sys.path)}) + + return the_job + +def job(rconn: Redis, rprefix: str, jobid: Union[str,UUID]): + "Retrieve the job" + thejob = (rconn.hgetall(job_key(rprefix, jobid)) or + raise_jobnotfound(rprefix, jobid)) + return thejob + +def update_status( + rconn: Redis, rprefix: str, jobid: Union[str, UUID], status: str): + """Update status of job in redis.""" + rconn.hset(name=job_key(rprefix, jobid), key="status", value=status) + +def update_stdout_stderr(rconn: Redis, + rprefix: str, + jobid: Union[str, UUID], + bytes_read: bytes, + stream: str): + "Update the stdout/stderr keys according to the value of `stream`." + thejob = job(rconn, rprefix, jobid) + contents = thejob.get(stream, '') + new_contents = contents + bytes_read.decode("utf-8") + rconn.hset(name=job_key(rprefix, jobid), key=stream, value=new_contents) + + +def job_errors( + rconn: Redis, + prefix: str, + job_id: Union[str, uuid.UUID], + count: int = 100 +) -> list: + """Fetch job errors""" + return take( + ( + json.loads(error) + for key in rconn.keys(f"{prefix}:{str(job_id)}:*:errors:*") + for error in rconn.lrange(key, 0, -1)), + count) + + +def job_files_metadata( + rconn: Redis, + prefix: str, + job_id: Union[str, uuid.UUID] +) -> dict: + """Get the metadata for specific job file.""" + return { + key.split(":")[-1]: { + **rconn.hgetall(key), + "filetype": key.split(":")[-3] + } + for key in rconn.keys(f"{prefix}:{str(job_id)}:*:metadata*") + } diff --git a/uploader/monadic_requests.py b/uploader/monadic_requests.py new file mode 100644 index 0000000..c492df5 --- /dev/null +++ b/uploader/monadic_requests.py @@ -0,0 +1,104 @@ +"""Wrap requests functions with monads.""" +import traceback +from typing import Union, Optional, Callable + +import requests +from requests.models import Response +from pymonad.either import Left, Right, Either +from flask import (flash, + request, + redirect, + render_template, + current_app as app, + escape as flask_escape) + +# HTML Status codes indicating a successful request. +SUCCESS_CODES = (200, 201, 202, 203, 204, 205, 206, 207, 208, 226) + +# Possible error(s) that can be encontered while attempting to do a request. +PossibleError = Union[Response, Exception] + + +def make_error_handler( + redirect_to: Optional[Response] = None, + cleanup_thunk: Callable = lambda *args: None +) -> Callable[[PossibleError], Response]: + """ + Build a function to gracefully handle errors encountered while doing + requests. + + :rtype: Callable + """ + redirect_to = redirect_to or redirect(request.url) + def __handler__(resp_or_exc: PossibleError) -> Response: + cleanup_thunk() + if issubclass(type(resp_or_exc), Exception): + # Is an exception! + return render_template( + "unhandled_exception.html", + trace=traceback.format_exception(resp_or_exc)) + if isinstance(resp_or_exc, Response): + flash("The authorisation server responded with " + f"({flask_escape(resp_or_exc.status_code)}, " + f"{flask_escape(resp_or_exc.reason)}) for the request to " + f"'{flask_escape(resp_or_exc.request.url)}'", + "alert-danger") + return redirect_to + + flash("Unspecified error!", "alert-danger") + app.logger.debug("Error (%s): %s", type(resp_or_exc), resp_or_exc) + return redirect_to + return __handler__ + + +def get(url, params=None, **kwargs) -> Either: + """ + A wrapper around `requests.get` function. + + Takes the same arguments as `requests.get`. + + :rtype: pymonad.either.Either + """ + try: + resp = requests.get(url, params=params, **kwargs) + if resp.status_code in SUCCESS_CODES: + return Right(resp.json()) + return Left(resp) + except requests.exceptions.RequestException as exc: + return Left(exc) + + +def post(url, data=None, json=None, **kwargs) -> Either: + """ + A wrapper around `requests.post` function. + + Takes the same arguments as `requests.post`. + + :rtype: pymonad.either.Either + """ + try: + resp = requests.post(url, data=data, json=json, **kwargs) + if resp.status_code in SUCCESS_CODES: + return Right(resp.json()) + return Left(resp) + except requests.exceptions.RequestException as exc: + return Left(exc) + + +def make_either_error_handler(msg): + """Make generic error handler for pymonads Either objects.""" + def __fail__(error): + if issubclass(type(error), Exception): + app.logger.debug("\n\n%s (Exception)\n\n", msg, exc_info=True) + raise error + if issubclass(type(error), Response): + try: + _data = error.json() + except Exception as _exc: + raise Exception(error.content) from _exc + raise Exception(_data) + + app.logger.debug("\n\n%s\n\n", msg) + raise Exception(error) + + return __fail__ diff --git a/uploader/oauth2/__init__.py b/uploader/oauth2/__init__.py new file mode 100644 index 0000000..aaea638 --- /dev/null +++ b/uploader/oauth2/__init__.py @@ -0,0 +1 @@ +"""Package to handle OAuth2 authentication/authorisation issues.""" diff --git a/uploader/oauth2/client.py b/uploader/oauth2/client.py new file mode 100644 index 0000000..1efa299 --- /dev/null +++ b/uploader/oauth2/client.py @@ -0,0 +1,231 @@ +"""OAuth2 client utilities.""" +import json +import time +import random +from datetime import datetime, timedelta +from urllib.parse import urljoin, urlparse + +import requests +from flask import request, current_app as app + +from pymonad.either import Left, Right, Either + +from authlib.common.urls import url_decode +from authlib.jose.errors import BadSignatureError +from authlib.jose import KeySet, JsonWebKey, JsonWebToken +from authlib.integrations.requests_client import OAuth2Session + +from uploader import session +import uploader.monadic_requests as mrequests + +SCOPE = ("profile group role resource register-client user masquerade " + "introspect migrate-data") + + +def authserver_uri(): + """Return URI to authorisation server.""" + return app.config["AUTH_SERVER_URL"] + + +def oauth2_clientid(): + """Return the client id.""" + return app.config["OAUTH2_CLIENT_ID"] + + +def oauth2_clientsecret(): + """Return the client secret.""" + return app.config["OAUTH2_CLIENT_SECRET"] + + +def __fetch_auth_server_jwks__() -> KeySet: + """Fetch the JWKs from the auth server.""" + return KeySet([ + JsonWebKey.import_key(key) + for key in requests.get( + urljoin(authserver_uri(), "auth/public-jwks") + ).json()["jwks"]]) + + +def __update_auth_server_jwks__(jwks) -> KeySet: + """Update the JWKs from the servers if necessary.""" + last_updated = jwks["last-updated"] + now = datetime.now().timestamp() + # Maybe the `two_hours` variable below can be made into a configuration + # variable and passed in to this function + two_hours = timedelta(hours=2).seconds + if bool(last_updated) and (now - last_updated) < two_hours: + return jwks["jwks"] + + return session.set_auth_server_jwks(__fetch_auth_server_jwks__()) + + +def auth_server_jwks() -> KeySet: + """Fetch the auth-server JSON Web Keys information.""" + _jwks = session.session_info().get("auth_server_jwks") or {} + if bool(_jwks): + return __update_auth_server_jwks__({ + "last-updated": _jwks["last-updated"], + "jwks": KeySet([ + JsonWebKey.import_key(key) for key in _jwks.get( + "jwks", {"keys": []})["keys"]]) + }) + + return __update_auth_server_jwks__({ + "last-updated": (datetime.now() - timedelta(hours=3)).timestamp() + }) + + +def oauth2_client(): + """Build the OAuth2 client for use fetching data.""" + def __update_token__(token, refresh_token=None, access_token=None):# pylint: disable=[unused-argument] + """Update the token when refreshed.""" + session.set_user_token(token) + + def __json_auth__(client, _method, uri, headers, body): + return ( + uri, + {**headers, "Content-Type": "application/json"}, + json.dumps({ + **dict(url_decode(body)), + "client_id": client.client_id, + "client_secret": client.client_secret + })) + + def __client__(token) -> OAuth2Session: + client = OAuth2Session( + oauth2_clientid(), + oauth2_clientsecret(), + scope=SCOPE, + token_endpoint=urljoin(authserver_uri(), "/auth/token"), + token_endpoint_auth_method="client_secret_post", + token=token, + update_token=__update_token__) + client.register_client_auth_method( + ("client_secret_post", __json_auth__)) + return client + + def __token_expired__(token): + """Check whether the token has expired.""" + jwks = auth_server_jwks() + if bool(jwks): + for jwk in jwks.keys: + try: + jwt = JsonWebToken(["RS256"]).decode( + token["access_token"], key=jwk) + if bool(jwt.get("exp")): + return datetime.now().timestamp() > jwt["exp"] + except BadSignatureError as _bse: + pass + + return False + + def __delay__(): + """Do a tiny delay.""" + time.sleep(random.choice(tuple(i/1000.0 for i in range(0,100)))) + + def __refresh_token__(token): + """Refresh the token if necessary — synchronise amongst threads.""" + if __token_expired__(token): + __delay__() + if session.is_token_refreshing(): + while session.is_token_refreshing(): + __delay__() + + return session.user_token().either(None, lambda _tok: _tok) + + session.toggle_token_refreshing() + _client = __client__(token) + _client.get(urljoin(authserver_uri(), "auth/user/")) + session.toggle_token_refreshing() + return _client.token + + return token + + return session.user_token().then(__refresh_token__).either( + lambda _notok: __client__(None), + __client__) + + +def user_logged_in(): + """Check whether the user has logged in.""" + suser = session.session_info()["user"] + return suser["logged_in"] and suser["token"].is_right() + + +def authserver_authorise_uri(): + """Build up the authorisation URI.""" + req_baseurl = urlparse(request.base_url, scheme=request.scheme) + host_uri = f"{req_baseurl.scheme}://{req_baseurl.netloc}/" + return urljoin( + authserver_uri(), + "auth/authorise?response_type=code" + f"&client_id={oauth2_clientid()}" + f"&redirect_uri={urljoin(host_uri, 'oauth2/code')}") + + +def __no_token__(_err) -> Left: + """Handle situation where request is attempted with no token.""" + resp = requests.models.Response() + resp._content = json.dumps({#pylint: disable=[protected-access] + "error": "AuthenticationError", + "error-description": ("You need to authenticate to access requested " + "information.")}).encode("utf-8") + resp.status_code = 400 + return Left(resp) + + +def oauth2_get(url, **kwargs) -> Either: + """Do a get request to the authentication/authorisation server.""" + def __get__(_token) -> Either: + _uri = urljoin(authserver_uri(), url) + try: + resp = oauth2_client().get( + _uri, + **{ + **kwargs, + "headers": { + **kwargs.get("headers", {}), + "Content-Type": "application/json" + } + }) + if resp.status_code in mrequests.SUCCESS_CODES: + return Right(resp.json()) + return Left(resp) + except Exception as exc:#pylint: disable=[broad-except] + app.logger.error("Error retrieving data from auth server: (GET %s)", + _uri, + exc_info=True) + return Left(exc) + return session.user_token().either(__no_token__, __get__) + + +def oauth2_post(url, data=None, json=None, **kwargs):#pylint: disable=[redefined-outer-name] + """Do a POST request to the authentication/authorisation server.""" + def __post__(_token) -> Either: + _uri = urljoin(authserver_uri(), url) + _headers = ({ + **kwargs.get("headers", {}), + "Content-Type": "application/json" + } + if bool(json) else kwargs.get("headers", {})) + try: + request_data = { + **(data or {}), + **(json or {}), + "client_id": oauth2_clientid(), + "client_secret": oauth2_clientsecret() + } + resp = oauth2_client().post( + _uri, + data=(request_data if bool(data) else None), + json=(request_data if bool(json) else None), + **{**kwargs, "headers": _headers}) + if resp.status_code in mrequests.SUCCESS_CODES: + return Right(resp.json()) + return Left(resp) + except Exception as exc:#pylint: disable=[broad-except] + app.logger.error("Error retrieving data from auth server: (POST %s)", + _uri, + exc_info=True) + return Left(exc) + return session.user_token().either(__no_token__, __post__) diff --git a/uploader/oauth2/jwks.py b/uploader/oauth2/jwks.py new file mode 100644 index 0000000..efd0499 --- /dev/null +++ b/uploader/oauth2/jwks.py @@ -0,0 +1,86 @@ +"""Utilities dealing with JSON Web Keys (JWK)""" +import os +from pathlib import Path +from typing import Any, Union +from datetime import datetime, timedelta + +from flask import Flask +from authlib.jose import JsonWebKey +from pymonad.either import Left, Right, Either + +def jwks_directory(app: Flask, configname: str) -> Path: + """Compute the directory where the JWKs are stored.""" + appsecretsdir = Path(app.config[configname]).parent + if appsecretsdir.exists() and appsecretsdir.is_dir(): + jwksdir = Path(appsecretsdir, "jwks/") + if not jwksdir.exists(): + jwksdir.mkdir() + return jwksdir + raise ValueError( + "The `appsecretsdir` value should be a directory that actually exists.") + + +def generate_and_save_private_key( + storagedir: Path, + kty: str = "RSA", + crv_or_size: Union[str, int] = 2048, + options: tuple[tuple[str, Any]] = (("iat", datetime.now().timestamp()),) +) -> JsonWebKey: + """Generate a private key and save to `storagedir`.""" + privatejwk = JsonWebKey.generate_key( + kty, crv_or_size, dict(options), is_private=True) + keyname = f"{privatejwk.thumbprint()}.private.pem" + with open(Path(storagedir, keyname), "wb") as pemfile: + pemfile.write(privatejwk.as_pem(is_private=True)) + + return privatejwk + + +def pem_to_jwk(filepath: Path) -> JsonWebKey: + """Parse a PEM file into a JWK object.""" + with open(filepath, "rb") as pemfile: + return JsonWebKey.import_key(pemfile.read()) + + +def __sorted_jwks_paths__(storagedir: Path) -> tuple[tuple[float, Path], ...]: + """A sorted list of the JWK file paths with their creation timestamps.""" + return tuple(sorted(((os.stat(keypath).st_ctime, keypath) + for keypath in (Path(storagedir, keyfile) + for keyfile in os.listdir(storagedir) + if keyfile.endswith(".pem"))), + key=lambda tpl: tpl[0])) + + +def list_jwks(storagedir: Path) -> tuple[JsonWebKey, ...]: + """ + List all the JWKs in a particular directory in the order they were created. + """ + return tuple(pem_to_jwk(keypath) for ctime,keypath in + __sorted_jwks_paths__(storagedir)) + + +def newest_jwk(storagedir: Path) -> Either: + """ + Return an Either monad with the newest JWK or a message if none exists. + """ + existingkeys = __sorted_jwks_paths__(storagedir) + if len(existingkeys) > 0: + return Right(pem_to_jwk(existingkeys[-1][1])) + return Left("No JWKs exist") + + +def newest_jwk_with_rotation(jwksdir: Path, keyage: int) -> JsonWebKey: + """ + Retrieve the latests JWK, creating a new one if older than `keyage` days. + """ + def newer_than_days(jwkey): + filestat = os.stat(Path( + jwksdir, f"{jwkey.as_dict()['kid']}.private.pem")) + oldesttimeallowed = (datetime.now() - timedelta(days=keyage)) + if filestat.st_ctime < (oldesttimeallowed.timestamp()): + return Left("JWK is too old!") + return jwkey + + return newest_jwk(jwksdir).then(newer_than_days).either( + lambda _errmsg: generate_and_save_private_key(jwksdir), + lambda key: key) diff --git a/uploader/oauth2/views.py b/uploader/oauth2/views.py new file mode 100644 index 0000000..a7211cb --- /dev/null +++ b/uploader/oauth2/views.py @@ -0,0 +1,138 @@ +"""Views for OAuth2 related functionality.""" +import uuid +from datetime import datetime, timedelta +from urllib.parse import urljoin, urlparse, urlunparse + +from authlib.jose import jwt +from flask import ( + flash, + jsonify, + url_for, + request, + redirect, + Blueprint, + current_app as app) + +from uploader import session +from uploader import monadic_requests as mrequests +from uploader.monadic_requests import make_error_handler + +from . import jwks +from .client import ( + SCOPE, + oauth2_get, + user_logged_in, + authserver_uri, + oauth2_clientid, + oauth2_clientsecret) + +oauth2 = Blueprint("oauth2", __name__) + +@oauth2.route("/code") +def authorisation_code(): + """Receive authorisation code from auth server and use it to get token.""" + def __process_error__(resp_or_exception): + app.logger.debug("ERROR: (%s)", resp_or_exception) + flash("There was an error retrieving the authorisation token.", + "alert-danger") + return redirect("/") + + def __fail_set_user_details__(_failure): + app.logger.debug("Fetching user details fails: %s", _failure) + flash("Could not retrieve the user details", "alert-danger") + return redirect("/") + + def __success_set_user_details__(_success): + app.logger.debug("Session info: %s", _success) + return redirect("/") + + def __success__(token): + session.set_user_token(token) + return oauth2_get("auth/user/").then( + lambda usrdets: session.set_user_details({ + "user_id": uuid.UUID(usrdets["user_id"]), + "name": usrdets["name"], + "email": usrdets["email"], + "token": session.user_token(), + "logged_in": True})).either( + __fail_set_user_details__, + __success_set_user_details__) + + code = request.args.get("code", "").strip() + if not bool(code): + flash("AuthorisationError: No code was provided.", "alert-danger") + return redirect("/") + + baseurl = urlparse(request.base_url, scheme=request.scheme) + issued = datetime.now() + jwtkey = jwks.newest_jwk_with_rotation( + jwks.jwks_directory(app, "UPLOADER_SECRETS"), + int(app.config["JWKS_ROTATION_AGE_DAYS"])) + return mrequests.post( + urljoin(authserver_uri(), "auth/token"), + json={ + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "code": code, + "scope": SCOPE, + "redirect_uri": urljoin( + urlunparse(baseurl), + url_for("oauth2.authorisation_code")), + "assertion": jwt.encode( + header={ + "alg": "RS256", + "typ": "JWT", + "kid": jwtkey.as_dict()["kid"] + }, + payload={ + "iss": str(oauth2_clientid()), + "sub": request.args["user_id"], + "aud": urljoin(authserver_uri(),"auth/token"), + "exp": (issued + timedelta(minutes=5)).timestamp(), + "nbf": int(issued.timestamp()), + "iat": int(issued.timestamp()), + "jti": str(uuid.uuid4()) + }, + key=jwtkey).decode("utf8"), + "client_id": oauth2_clientid() + }).either(__process_error__, __success__) + +@oauth2.route("/public-jwks") +def public_jwks(): + """List the available JWKs""" + return jsonify({ + "documentation": ( + "The keys are listed in order of creation, from the oldest (first) " + "to the newest (last)."), + "jwks": tuple(key.as_dict() for key + in jwks.list_jwks(jwks.jwks_directory( + app, "UPLOADER_SECRETS"))) + }) + + +@oauth2.route("/logout", methods=["GET"]) +def logout(): + """Log out of any active sessions.""" + def __unset_session__(session_info): + _user = session_info["user"] + _user_str = f"{_user['name']} ({_user['email']})" + session.clear_session_info() + flash("Successfully signed out.", "alert-success") + return redirect("/") + + if user_logged_in(): + return session.user_token().then( + lambda _tok: mrequests.post( + urljoin(authserver_uri(), "auth/revoke"), + json={ + "token": _tok["refresh_token"], + "token_type_hint": "refresh_token", + "client_id": oauth2_clientid(), + "client_secret": oauth2_clientsecret() + })).either( + make_error_handler( + redirect_to=redirect("/"), + cleanup_thunk=lambda: __unset_session__( + session.session_info())), + lambda res: __unset_session__(session.session_info())) + flash("There is no user that is currently logged in.", "alert-info") + return redirect("/") diff --git a/uploader/phenotypes/__init__.py b/uploader/phenotypes/__init__.py new file mode 100644 index 0000000..c17d32c --- /dev/null +++ b/uploader/phenotypes/__init__.py @@ -0,0 +1,2 @@ +"""Package for handling ('classical') phenotype data""" +from .views import phenotypesbp diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py new file mode 100644 index 0000000..e1ec0c9 --- /dev/null +++ b/uploader/phenotypes/models.py @@ -0,0 +1,256 @@ +"""Database and utility functions for phenotypes.""" +from typing import Optional +from functools import reduce +from datetime import datetime + +import MySQLdb as mdb +from MySQLdb.cursors import Cursor, DictCursor +from flask import current_app as app + +from gn_libs.mysqldb import debug_query + +def datasets_by_population( + conn: mdb.Connection, + species_id: int, + population_id: int +) -> tuple[dict, ...]: + """Retrieve all of a population's phenotype studies.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT s.SpeciesId, pf.* FROM Species AS s " + "INNER JOIN InbredSet AS iset ON s.Id=iset.SpeciesId " + "INNER JOIN PublishFreeze AS pf ON iset.Id=pf.InbredSetId " + "WHERE s.Id=%s AND iset.Id=%s;", + (species_id, population_id)) + return tuple(dict(row) for row in cursor.fetchall()) + + +def dataset_by_id(conn: mdb.Connection, + species_id: int, + population_id: int, + dataset_id: int) -> dict: + """Fetch dataset details by identifier""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT s.SpeciesId, pf.* FROM Species AS s " + "INNER JOIN InbredSet AS iset ON s.Id=iset.SpeciesId " + "INNER JOIN PublishFreeze AS pf ON iset.Id=pf.InbredSetId " + "WHERE s.Id=%s AND iset.Id=%s AND pf.Id=%s", + (species_id, population_id, dataset_id)) + return dict(cursor.fetchone()) + + +def phenotypes_count(conn: mdb.Connection, + population_id: int, + dataset_id: int) -> int: + """Count the number of phenotypes in the dataset.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT COUNT(*) AS total_phenos FROM Phenotype AS pheno " + "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " + "INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId " + "WHERE pxr.InbredSetId=%s AND pf.Id=%s", + (population_id, dataset_id)) + return int(cursor.fetchone()["total_phenos"]) + + +def phenotype_publication_data(conn, phenotype_id) -> Optional[dict]: + """Retrieve the publication data for a phenotype if it exists.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT DISTINCT pxr.PhenotypeId, pub.* FROM PublishXRef AS pxr " + "INNER JOIN Publication as pub ON pxr.PublicationId=pub.Id " + "WHERE pxr.PhenotypeId=%s", + (phenotype_id,)) + res = cursor.fetchone() + if res is None: + return res + return dict(res) + + +def dataset_phenotypes(conn: mdb.Connection, + population_id: int, + dataset_id: int, + offset: int = 0, + limit: Optional[int] = None) -> tuple[dict, ...]: + """Fetch the actual phenotypes.""" + _query = ( + "SELECT pheno.*, pxr.Id AS xref_id, ist.InbredSetCode FROM Phenotype AS pheno " + "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " + "INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId " + "INNER JOIN InbredSet AS ist ON pf.InbredSetId=ist.Id " + "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + ( + f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(_query, (population_id, dataset_id)) + debug_query(cursor, app.logger) + return tuple(dict(row) for row in cursor.fetchall()) + + +def __phenotype_se__(cursor: Cursor, xref_id, dataids_and_strainids): + """Fetch standard-error values (if they exist) for a phenotype.""" + paramstr = ", ".join(["(%s, %s)"] * len(dataids_and_strainids)) + flat = tuple(item for sublist in dataids_and_strainids for item in sublist) + cursor.execute("SELECT * FROM PublishSE WHERE (DataId, StrainId) IN " + f"({paramstr})", + flat) + debug_query(cursor, app.logger) + _se = { + (row["DataId"], row["StrainId"]): { + "DataId": row["DataId"], + "StrainId": row["StrainId"], + "error": row["error"] + } + for row in cursor.fetchall() + } + + cursor.execute("SELECT * FROM NStrain WHERE (DataId, StrainId) IN " + f"({paramstr})", + flat) + debug_query(cursor, app.logger) + _n = { + (row["DataId"], row["StrainId"]): { + "DataId": row["DataId"], + "StrainId": row["StrainId"], + "count": row["count"] + } + for row in cursor.fetchall() + } + + keys = set(tuple(_se.keys()) + tuple(_n.keys())) + return { + key: {"xref_id": xref_id, **_se.get(key,{}), **_n.get(key,{})} + for key in keys + } + +def __organise_by_phenotype__(pheno, row): + """Organise disparate data rows into phenotype 'objects'.""" + _pheno = pheno.get(row["Id"]) + return { + **pheno, + row["Id"]: { + "Id": row["Id"], + "Pre_publication_description": row["Pre_publication_description"], + "Post_publication_description": row["Post_publication_description"], + "Original_description": row["Original_description"], + "Units": row["Units"], + "Pre_publication_abbreviation": row["Pre_publication_abbreviation"], + "Post_publication_abbreviation": row["Post_publication_abbreviation"], + "xref_id": row["pxr.Id"], + "data": { + **(_pheno["data"] if bool(_pheno) else {}), + (row["DataId"], row["StrainId"]): { + "DataId": row["DataId"], + "StrainId": row["StrainId"], + "mean": row["mean"], + "Locus": row["Locus"], + "LRS": row["LRS"], + "additive": row["additive"], + "Sequence": row["Sequence"], + "comments": row["comments"], + "value": row["value"], + "StrainName": row["Name"], + "StrainName2": row["Name2"], + "StrainSymbol": row["Symbol"], + "StrainAlias": row["Alias"] + } + } + } + } + + +def __merge_pheno_data_and_se__(data, sedata) -> dict: + """Merge phenotype data with the standard errors.""" + return { + key: {**value, **sedata.get(key, {})} + for key, value in data.items() + } + + +def phenotype_by_id( + conn: mdb.Connection, + species_id: int, + population_id: int, + dataset_id: int, + xref_id +) -> Optional[dict]: + """Fetch a specific phenotype.""" + _dataquery = ("SELECT pheno.*, pxr.*, pd.*, str.*, iset.InbredSetCode " + "FROM Phenotype AS pheno " + "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " + "INNER JOIN PublishData AS pd ON pxr.DataId=pd.Id " + "INNER JOIN Strain AS str ON pd.StrainId=str.Id " + "INNER JOIN StrainXRef AS sxr ON str.Id=sxr.StrainId " + "INNER JOIN PublishFreeze AS pf ON sxr.InbredSetId=pf.InbredSetId " + "INNER JOIN InbredSet AS iset ON pf.InbredSetId=iset.InbredSetId " + "WHERE " + "(str.SpeciesId, pxr.InbredSetId, pf.Id, pxr.Id)=(%s, %s, %s, %s)") + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(_dataquery, + (species_id, population_id, dataset_id, xref_id)) + _pheno: dict = reduce(__organise_by_phenotype__, cursor.fetchall(), {}) + if bool(_pheno) and len(_pheno.keys()) == 1: + _pheno = tuple(_pheno.values())[0] + return { + **_pheno, + "data": tuple(__merge_pheno_data_and_se__( + _pheno["data"], + __phenotype_se__( + cursor, xref_id, tuple(_pheno["data"].keys())) + ).values()) + } + if bool(_pheno) and len(_pheno.keys()) > 1: + raise Exception( + "We found more than one phenotype with the same identifier!") + + return None + + +def phenotypes_data(conn: mdb.Connection, + population_id: int, + dataset_id: int, + offset: int = 0, + limit: Optional[int] = None) -> tuple[dict, ...]: + """Fetch the data for the phenotypes.""" + # — Phenotype -> PublishXRef -> PublishData -> Strain -> StrainXRef -> PublishFreeze + _query = ("SELECT pheno.*, pxr.*, pd.*, str.*, iset.InbredSetCode " + "FROM Phenotype AS pheno " + "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " + "INNER JOIN PublishData AS pd ON pxr.DataId=pd.Id " + "INNER JOIN Strain AS str ON pd.StrainId=str.Id " + "INNER JOIN StrainXRef AS sxr ON str.Id=sxr.StrainId " + "INNER JOIN PublishFreeze AS pf ON sxr.InbredSetId=pf.InbredSetId " + "INNER JOIN InbredSet AS iset ON pf.InbredSetId=iset.InbredSetId " + "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + ( + f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(_query, (population_id, dataset_id)) + debug_query(cursor, app.logger) + return tuple(dict(row) for row in cursor.fetchall()) + + +def save_new_dataset(cursor: Cursor, + population_id: int, + dataset_name: str, + dataset_fullname: str, + dataset_shortname: str) -> dict: + """Create a new phenotype dataset.""" + params = { + "population_id": population_id, + "dataset_name": dataset_name, + "dataset_fullname": dataset_fullname, + "dataset_shortname": dataset_shortname, + "created": datetime.now().date().isoformat(), + "public": 2, + "confidentiality": 0, + "users": None + } + cursor.execute( + "INSERT INTO PublishFreeze(Name, FullName, ShortName, CreateTime, " + "public, InbredSetId, confidentiality, AuthorisedUsers) " + "VALUES(%(dataset_name)s, %(dataset_fullname)s, %(dataset_shortname)s, " + "%(created)s, %(public)s, %(population_id)s, %(confidentiality)s, " + "%(users)s)", + params) + debug_query(cursor, app.logger) + return {**params, "Id": cursor.lastrowid} diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py new file mode 100644 index 0000000..bcbb3a9 --- /dev/null +++ b/uploader/phenotypes/views.py @@ -0,0 +1,864 @@ +"""Views handling ('classical') phenotypes.""" +import sys +import uuid +import json +import datetime +from typing import Any +from pathlib import Path +from zipfile import ZipFile +from functools import wraps, reduce +from logging import INFO, ERROR, DEBUG, FATAL, CRITICAL, WARNING + +from redis import Redis +from pymonad.either import Left +from requests.models import Response +from MySQLdb.cursors import DictCursor +from gn_libs.mysqldb import database_connection +from flask import (flash, + request, + url_for, + jsonify, + redirect, + Blueprint, + current_app as app) + +# from r_qtl import r_qtl2 as rqtl2 +from r_qtl import r_qtl2_qc as rqc +from r_qtl import exceptions as rqe + +from uploader import jobs +from uploader.files import save_file#, fullpath +from uploader.ui import make_template_renderer +from uploader.oauth2.client import oauth2_post +from uploader.authorisation import require_login +from uploader.species.models import all_species, species_by_id +from uploader.monadic_requests import make_either_error_handler +from uploader.request_checks import with_species, with_population +from uploader.datautils import safe_int, order_by_family, enumerate_sequence +from uploader.population.models import (populations_by_species, + population_by_species_and_id) +from uploader.input_validation import (encode_errors, + decode_errors, + is_valid_representative_name) + +from .models import (dataset_by_id, + phenotype_by_id, + phenotypes_count, + save_new_dataset, + dataset_phenotypes, + datasets_by_population, + phenotype_publication_data) + +phenotypesbp = Blueprint("phenotypes", __name__) +render_template = make_template_renderer("phenotypes") + +_FAMILIES_WITH_SE_AND_N_ = ( + "Reference Populations (replicate average, SE, N)",) + +@phenotypesbp.route("/phenotypes", methods=["GET"]) +@require_login +def index(): + """Direct entry-point for phenotypes data handling.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(request.args.get("species_id")): + return render_template("phenotypes/index.html", + species=all_species(conn), + activelink="phenotypes") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.phenotypes.select_population")) + + species = species_by_id(conn, species_id) + if not bool(species): + flash("No such species!", "alert-danger") + return redirect(url_for("species.populations.phenotypes.index")) + return redirect(url_for("species.populations.phenotypes.select_population", + species_id=species["SpeciesId"])) + + +@phenotypesbp.route("<int:species_id>/phenotypes/select-population", + methods=["GET"]) +@require_login +@with_species(redirect_uri="species.populations.phenotypes.index") +def select_population(species: dict, **kwargs):# pylint: disable=[unused-argument] + """Select the population for your phenotypes.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(request.args.get("population_id")): + return render_template("phenotypes/select-population.html", + species=species, + populations=populations_by_species( + conn, species["SpeciesId"]), + activelink="phenotypes") + + population_id = request.args["population_id"] + if population_id == "CREATE-POPULATION": + return redirect(url_for( + "species.populations.create_population", + species_id=species["SpeciesId"], + return_to="species.populations.phenotypes.list_datasets")) + population = population_by_species_and_id( + conn, species["SpeciesId"], int(population_id)) + if not bool(population): + flash("No such population found!", "alert-danger") + return redirect(url_for( + "species.populations.phenotypes.select_population", + species_id=species["SpeciesId"])) + + return redirect(url_for("species.populations.phenotypes.list_datasets", + species_id=species["SpeciesId"], + population_id=population["Id"])) + + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets", + methods=["GET"]) +@require_login +@with_population(species_redirect_uri="species.populations.phenotypes.index", + redirect_uri="species.populations.phenotypes.select_population") +def list_datasets(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] + """List available phenotype datasets.""" + with database_connection(app.config["SQL_URI"]) as conn: + datasets = datasets_by_population( + conn, species["SpeciesId"], population["Id"]) + if len(datasets) == 1: + return redirect(url_for( + "species.populations.phenotypes.view_dataset", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=datasets[0]["Id"])) + return render_template("phenotypes/list-datasets.html", + species=species, + population=population, + datasets=datasets, + activelink="list-datasets") + + +def with_dataset( + species_redirect_uri: str, + population_redirect_uri: str, + redirect_uri: str +): + """Ensure the dataset actually exists.""" + def __decorator__(func): + @wraps(func) + @with_population(species_redirect_uri, population_redirect_uri) + def __with_dataset__(**kwargs): + try: + _spcid = int(kwargs["species_id"]) + _popid = int(kwargs["population_id"]) + _dsetid = int(kwargs.get("dataset_id")) + select_dataset_uri = redirect(url_for( + redirect_uri, species_id=_spcid, population_id=_popid)) + if not bool(_dsetid): + flash("You need to select a valid 'dataset_id' value.", + "alert-danger") + return select_dataset_uri + with database_connection(app.config["SQL_URI"]) as conn: + dataset = dataset_by_id(conn, _spcid, _popid, _dsetid) + if not bool(dataset): + flash("You must select a valid dataset.", + "alert-danger") + return select_dataset_uri + except ValueError as _verr: + app.logger.debug( + "Exception converting 'dataset_id' to integer: %s", + kwargs.get("dataset_id"), + exc_info=True) + flash("Expected 'dataset_id' value to be an integer." + "alert-danger") + return select_dataset_uri + return func(dataset=dataset, **kwargs) + return __with_dataset__ + return __decorator__ + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/view", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def view_dataset(# pylint: disable=[unused-argument] + species: dict, population: dict, dataset: dict, **kwargs): + """View a specific dataset""" + with database_connection(app.config["SQL_URI"]) as conn: + dataset = dataset_by_id( + conn, species["SpeciesId"], population["Id"], dataset["Id"]) + if not bool(dataset): + flash("Could not find such a phenotype dataset!", "alert-danger") + return redirect(url_for( + "species.populations.phenotypes.list_datasets", + species_id=species["SpeciesId"], + population_id=population["Id"])) + + start_at = max(safe_int(request.args.get("start_at") or 0), 0) + count = int(request.args.get("count") or 20) + return render_template("phenotypes/view-dataset.html", + species=species, + population=population, + dataset=dataset, + phenotype_count=phenotypes_count( + conn, population["Id"], dataset["Id"]), + phenotypes=enumerate_sequence( + dataset_phenotypes( + conn, + population["Id"], + dataset["Id"])), + start_from=start_at, + count=count, + activelink="view-dataset") + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/phenotype/<xref_id>", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def view_phenotype(# pylint: disable=[unused-argument] + species: dict, + population: dict, + dataset: dict, + xref_id: int, + **kwargs +): + """View an individual phenotype from the dataset.""" + def __render__(privileges): + phenotype = phenotype_by_id(conn, + species["SpeciesId"], + population["Id"], + dataset["Id"], + xref_id) + def __non_empty__(value) -> bool: + if isinstance(value, str): + return value.strip() != "" + return bool(value) + + return render_template( + "phenotypes/view-phenotype.html", + species=species, + population=population, + dataset=dataset, + xref_id=xref_id, + phenotype=phenotype, + has_se=any(bool(item.get("error")) for item in phenotype["data"]), + publish_data={ + key.replace("_", " "): val + for key,val in + (phenotype_publication_data(conn, phenotype["Id"]) or {}).items() + if (key in ("PubMed_ID", "Authors", "Title", "Journal") + and __non_empty__(val)) + }, + privileges=(privileges + ### For demo! Do not commit this part + + ("group:resource:edit-resource", + "group:resource:delete-resource",) + ### END: For demo! Do not commit this part + ), + activelink="view-phenotype") + + def __fail__(error): + if isinstance(error, Response) and error.json() == "No linked resource!": + return __render__(tuple()) + return make_either_error_handler( + "There was an error fetching the roles and privileges.")(error) + + with database_connection(app.config["SQL_URI"]) as conn: + return oauth2_post( + "/auth/resource/phenotypes/individual/linked-resource", + json={ + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": dataset["Id"], + "xref_id": xref_id + } + ).then( + lambda resource: tuple( + privilege["privilege_id"] for role in resource["roles"] + for privilege in role["privileges"]) + ).then(__render__).either(__fail__, lambda resp: resp) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets/create", + methods=["GET", "POST"]) +@require_login +@with_population( + species_redirect_uri="species.populations.phenotypes.index", + redirect_uri="species.populations.phenotypes.select_population") +def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] + """Create a new phenotype dataset.""" + with (database_connection(app.config["SQL_URI"]) as conn, + conn.cursor(cursorclass=DictCursor) as cursor): + if request.method == "GET": + return render_template("phenotypes/create-dataset.html", + activelink="create-dataset", + species=species, + population=population, + **decode_errors( + request.args.get("error_values", ""))) + + form = request.form + _errors: tuple[tuple[str, str], ...] = tuple() + if not is_valid_representative_name( + (form.get("dataset-name") or "").strip()): + _errors = _errors + (("dataset-name", "Invalid dataset name."),) + + if not bool((form.get("dataset-fullname") or "").strip()): + _errors = _errors + (("dataset-fullname", + "You must provide a value for 'Full Name'."),) + + if bool(_errors) > 0: + return redirect(url_for( + "species.populations.phenotypes.create_dataset", + species_id=species["SpeciesId"], + population_id=population["Id"], + error_values=encode_errors(_errors, form))) + + dataset_shortname = ( + form["dataset-shortname"] or form["dataset-name"]).strip() + _pheno_dataset = save_new_dataset( + cursor, + population["Id"], + form["dataset-name"].strip(), + form["dataset-fullname"].strip(), + dataset_shortname) + return redirect(url_for("species.populations.phenotypes.list_datasets", + species_id=species["SpeciesId"], + population_id=population["Id"])) + + +def process_phenotypes_rqtl2_bundle(error_uri): + """Process phenotypes from the uploaded R/qtl2 bundle.""" + _redisuri = app.config["REDIS_URL"] + _sqluri = app.config["SQL_URI"] + try: + ## Handle huge files here... + phenobundle = save_file(request.files["phenotypes-bundle"], + Path(app.config["UPLOAD_FOLDER"])) + rqc.validate_bundle(phenobundle) + return phenobundle + except AssertionError as _aerr: + app.logger.debug("File upload error!", exc_info=True) + flash("Expected a zipped bundle of files with phenotypes' " + "information.", + "alert-danger") + return error_uri + except rqe.RQTLError as rqtlerr: + app.logger.debug("Bundle validation error!", exc_info=True) + flash("R/qtl2 Error: " + " ".join(rqtlerr.args), "alert-danger") + return error_uri + + +def process_phenotypes_individual_files(error_uri): + """Process the uploaded individual files.""" + form = request.form + cdata = { + "sep": form["file-separator"], + "comment.char": form["file-comment-character"], + "na.strings": form["file-na"].split(" "), + } + bundlepath = Path(app.config["UPLOAD_FOLDER"], + f"{str(uuid.uuid4()).replace('-', '')}.zip") + with ZipFile(bundlepath,mode="w") as zfile: + for rqtlkey, formkey in (("phenocovar", "phenotype-descriptions"), + ("pheno", "phenotype-data"), + ("phenose", "phenotype-se"), + ("phenonum", "phenotype-n")): + if form.get("resumable-upload", False): + # Chunked upload of large files was used + filedata = json.loads(form[formkey]) + zfile.write( + Path(app.config["UPLOAD_FOLDER"], filedata["uploaded-file"]), + arcname=filedata["original-name"]) + cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filedata["original-name"]] + else: + # TODO: Check this path: fix any bugs. + _sentfile = request.files[formkey] + if not bool(_sentfile): + flash(f"Expected file ('{formkey}') was not provided.", + "alert-danger") + return error_uri + + filepath = save_file( + _sentfile, Path(app.config["UPLOAD_FOLDER"]), hashed=False) + zfile.write( + Path(app.config["UPLOAD_FOLDER"], filepath), + arcname=filepath.name) + cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filepath.name] + + zfile.writestr("control_data.json", data=json.dumps(cdata, indent=2)) + + return bundlepath + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/add-phenotypes", + methods=["GET", "POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# pylint: disable=[unused-argument, too-many-locals] + """Add one or more phenotypes to the dataset.""" + use_bundle = request.args.get("use_bundle", "").lower() == "true" + add_phenos_uri = redirect(url_for( + "species.populations.phenotypes.add_phenotypes", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"])) + _redisuri = app.config["REDIS_URL"] + _sqluri = app.config["SQL_URI"] + with (Redis.from_url(_redisuri, decode_responses=True) as rconn, + # database_connection(_sqluri) as conn, + # conn.cursor(cursorclass=DictCursor) as cursor + ): + if request.method == "GET": + today = datetime.date.today() + return render_template( + ("phenotypes/add-phenotypes-with-rqtl2-bundle.html" + if use_bundle else "phenotypes/add-phenotypes-raw-files.html"), + species=species, + population=population, + dataset=dataset, + monthnames=( + "January", "February", "March", "April", + "May", "June", "July", "August", + "September", "October", "November", + "December"), + current_month=today.strftime("%B"), + current_year=int(today.strftime("%Y")), + families_with_se_and_n=_FAMILIES_WITH_SE_AND_N_, + use_bundle=use_bundle, + activelink="add-phenotypes") + + phenobundle = (process_phenotypes_rqtl2_bundle(add_phenos_uri) + if use_bundle else + process_phenotypes_individual_files(add_phenos_uri)) + + _jobid = uuid.uuid4() + _namespace = jobs.jobsnamespace() + _ttl_seconds = app.config["JOBS_TTL_SECONDS"] + _job = jobs.launch_job( + jobs.initialise_job( + rconn, + _namespace, + str(_jobid), + [sys.executable, "-m", "scripts.rqtl2.phenotypes_qc", _sqluri, + _redisuri, _namespace, str(_jobid), str(species["SpeciesId"]), + str(population["Id"]), + # str(dataset["Id"]), + str(phenobundle), + "--loglevel", + { + INFO: "INFO", + ERROR: "ERROR", + DEBUG: "DEBUG", + FATAL: "FATAL", + CRITICAL: "CRITICAL", + WARNING: "WARNING" + }[app.logger.getEffectiveLevel()], + "--redisexpiry", + str(_ttl_seconds)], "phenotype_qc", _ttl_seconds, + {"job-metadata": json.dumps({ + "speciesid": species["SpeciesId"], + "populationid": population["Id"], + "datasetid": dataset["Id"], + "bundle": str(phenobundle.absolute())})}), + _redisuri, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + + app.logger.debug("JOB DETAILS: %s", _job) + jobstatusuri = url_for("species.populations.phenotypes.job_status", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"], + job_id=str(_job["jobid"])) + return ((jsonify({ + "redirect-to": jobstatusuri, + "statuscode": 200, + "message": ("Follow the 'redirect-to' URI to see the state " + "of the quality-control job started for your " + "uploaded files.") + }), 200) + if request.form.get("resumable-upload", False) else + redirect(jobstatusuri)) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/job/<uuid:job_id>", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def job_status( + species: dict, + population: dict, + dataset: dict, + job_id: uuid.UUID, + **kwargs +):# pylint: disable=[unused-argument] + """Retrieve current status of a particular phenotype QC job.""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id)) + except jobs.JobNotFound as _jnf: + job = None + return render_template("phenotypes/job-status.html", + species=species, + population=population, + dataset=dataset, + job_id=job_id, + job=job, + errors=jobs.job_errors( + rconn, jobs.jobsnamespace(), job['jobid']), + metadata=jobs.job_files_metadata( + rconn, jobs.jobsnamespace(), job['jobid']), + activelink="add-phenotypes") + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/job/<uuid:job_id>/review", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def review_job_data( + species: dict, + population: dict, + dataset: dict, + job_id: uuid.UUID, + **kwargs +):# pylint: disable=[unused-argument] + """Review data one more time before entering it into the database.""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id)) + except jobs.JobNotFound as _jnf: + job = None + + def __metadata_by_type__(by_type, item): + filetype = item[1]["filetype"] + return { + **by_type, + filetype: (by_type.get(filetype, tuple()) + + ({"filename": item[0], **item[1]},)) + } + metadata: dict[str, Any] = reduce( + __metadata_by_type__, + (jobs.job_files_metadata( + rconn, jobs.jobsnamespace(), job['jobid']) + if job else {}).items(), + {}) + + def __desc__(filetype): + match filetype: + case "phenocovar": + desc = "phenotypes" + case "pheno": + desc = "phenotypes data" + case "phenose": + desc = "phenotypes standard-errors" + case "phenonum": + desc = "phenotypes samples" + case _: + desc = f"unknown file type '{filetype}'." + + return desc + + def __summarise__(filetype, files): + return { + "filetype": filetype, + "number-of-files": len(files), + "total-data-rows": sum( + int(afile["linecount"]) - 1 for afile in files), + "description": __desc__(filetype) + } + + summary = { + filetype: __summarise__(filetype, meta) + for filetype,meta in metadata.items() + } + return render_template("phenotypes/review-job-data.html", + species=species, + population=population, + dataset=dataset, + job_id=job_id, + job=job, + summary=summary, + activelink="add-phenotypes") + + +def update_phenotype_metadata(conn, metadata: dict): + """Update a phenotype's basic metadata values.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Phenotype WHERE Id=%(phenotype-id)s", + metadata) + res = { + **{ + _key: _val for _key,_val in { + key.lower().replace("_", "-"): value + for key, value in (cursor.fetchone() or {}).items() + }.items() + if _key in metadata.keys() + }, + "phenotype-id": metadata.get("phenotype-id") + } + if res == metadata: + return False + + cursor.execute( + "UPDATE Phenotype SET " + "Pre_publication_description=%(pre-publication-description)s, " + "Post_publication_description=%(post-publication-description)s, " + "Original_description=%(original-description)s, " + "Units=%(units)s, " + "Pre_publication_abbreviation=%(pre-publication-abbreviation)s, " + "Post_publication_abbreviation=%(post-publication-abbreviation)s " + "WHERE Id=%(phenotype-id)s", + metadata) + return cursor.rowcount + + +def update_phenotype_values(conn, values): + """Update a phenotype's data values.""" + with conn.cursor() as cursor: + cursor.executemany( + "UPDATE PublishData SET value=%(new)s " + "WHERE Id=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in values if item["new"] is not None)) + cursor.executemany( + "DELETE FROM PublishData " + "WHERE Id=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in values if item["new"] is None)) + return len(values) + return 0 + + +def update_phenotype_se(conn, serrs): + """Update a phenotype's standard-error values.""" + with conn.cursor() as cursor: + cursor.executemany( + "INSERT INTO PublishSE(DataId, StrainId, error) " + "VALUES(%(data_id)s, %(strain_id)s, %(new)s) " + "ON DUPLICATE KEY UPDATE error=VALUES(error)", + tuple(item for item in serrs if item["new"] is not None)) + cursor.executemany( + "DELETE FROM PublishSE " + "WHERE DataId=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in serrs if item["new"] is None)) + return len(serrs) + return 0 + + +def update_phenotype_n(conn, counts): + """Update a phenotype's strain counts.""" + with conn.cursor() as cursor: + cursor.executemany( + "INSERT INTO NStrain(DataId, StrainId, count) " + "VALUES(%(data_id)s, %(strain_id)s, %(new)s) " + "ON DUPLICATE KEY UPDATE count=VALUES(count)", + tuple(item for item in counts if item["new"] is not None)) + cursor.executemany( + "DELETE FROM NStrain " + "WHERE DataId=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in counts if item["new"] is None)) + return len(counts) + + return 0 + + +def update_phenotype_data(conn, data: dict): + """Update the numeric data for a phenotype.""" + def __organise_by_dataid_and_strainid__(acc, current): + _key, dataid, strainid = current[0].split("::") + _keysrc, _keytype = _key.split("-") + newkey = f"{dataid}::{strainid}" + newitem = acc.get(newkey, {}) + newitem[_keysrc] = newitem.get(_keysrc, {}) + newitem[_keysrc][_keytype] = current[1] + return {**acc, newkey: newitem} + + def __separate_items__(acc, row): + key, val = row + return ({ + **acc[0], + key: { + **val["value"], + "changed?": (not val["value"]["new"] == val["value"]["original"]) + } + }, { + **acc[1], + key: { + **val["se"], + "changed?": (not val["se"]["new"] == val["se"]["original"]) + } + },{ + **acc[2], + key: { + **val["n"], + "changed?": (not val["n"]["new"] == val["n"]["original"]) + } + }) + + values, serrs, counts = tuple( + tuple({ + "data_id": row[0].split("::")[0], + "strain_id": row[0].split("::")[1], + "new": row[1]["new"] + } for row in item) + for item in ( + filter(lambda val: val[1]["changed?"], item.items())# type: ignore[arg-type] + for item in reduce(# type: ignore[var-annotated] + __separate_items__, + reduce(__organise_by_dataid_and_strainid__, + data.items(), + {}).items(), + ({}, {}, {})))) + + return (update_phenotype_values(conn, values), + update_phenotype_se(conn, serrs), + update_phenotype_n(conn, counts)) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/phenotype/<int:xref_id>/edit", + methods=["GET", "POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def edit_phenotype_data(# pylint: disable=[unused-argument] + species: dict, + population: dict, + dataset: dict, + xref_id: int, + **kwargs +): + """Edit the data for a particular phenotype.""" + def __render__(**kwargs): + processed_kwargs = { + **kwargs, + "privileges": (kwargs.get("privileges", tuple()) + ### For demo! Do not commit this part + + ("group:resource:edit-resource", + "group:resource:delete-resource",) + ### END: For demo! Do not commit this part + ) + } + return render_template( + "phenotypes/edit-phenotype.html", + species=species, + population=population, + dataset=dataset, + xref_id=xref_id, + families_with_se_and_n=_FAMILIES_WITH_SE_AND_N_, + **processed_kwargs, + activelink="edit-phenotype") + + with database_connection(app.config["SQL_URI"]) as conn: + if request.method == "GET": + def __fetch_phenotype__(privileges): + phenotype = phenotype_by_id(conn, + species["SpeciesId"], + population["Id"], + dataset["Id"], + xref_id) + if phenotype is None: + msg = ("Could not find the phenotype with cross-reference ID" + f" '{xref_id}' from dataset '{dataset['FullName']}' " + f" from the '{population['FullName']}' population of " + f" species '{species['FullName']}'.") + return Left({"privileges": privileges, "phenotype-error": msg}) + return {"privileges": privileges, "phenotype": phenotype} + + def __fetch_publication_data__(**kwargs): + pheno = kwargs["phenotype"] + return { + **kwargs, + "publication_data": phenotype_publication_data( + conn, pheno["Id"]) + } + + def __fail__(failure_object): + # process the object + return __render__(failure_object=failure_object) + + return oauth2_post( + "/auth/resource/phenotypes/individual/linked-resource", + json={ + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": dataset["Id"], + "xref_id": xref_id + } + ).then( + lambda resource: tuple( + privilege["privilege_id"] for role in resource["roles"] + for privilege in role["privileges"]) + ).then( + __fetch_phenotype__ + ).then( + lambda args: __fetch_publication_data__(**args) + ).either(__fail__, lambda args: __render__(**args)) + + ## POST + _change = False + match request.form.get("submit", "invalid-action"): + case "update basic metadata": + _change = update_phenotype_metadata(conn, { + key: value.strip() if bool(value.strip()) else None + for key, value in request.form.items() + if key not in ("submit",) + }) + msg = "Basic metadata was updated successfully." + case "update data": + _update = update_phenotype_data(conn, { + key: value.strip() if bool(value.strip()) else None + for key, value in request.form.items() + if key not in ("submit",) + }) + msg = (f"{_update[0]} value rows, {_update[1]} standard-error " + f"rows and {_update[2]} 'N' rows were updated.") + _change = any(item != 0 for item in _update) + case "update publication": + flash("NOT IMPLEMENTED: Would update publication data.", "alert-success") + case _: + flash("Invalid phenotype editing action.", "alert-danger") + + if _change: + flash(msg, "alert-success") + return redirect(url_for( + "species.populations.phenotypes.view_phenotype", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"], + xref_id=xref_id)) + + flash("No change was made by the user.", "alert-info") + return redirect(url_for( + "species.populations.phenotypes.edit_phenotype_data", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"], + xref_id=xref_id)) diff --git a/uploader/platforms/__init__.py b/uploader/platforms/__init__.py new file mode 100644 index 0000000..8cb89c9 --- /dev/null +++ b/uploader/platforms/__init__.py @@ -0,0 +1,2 @@ +"""Module to handle management of genetic platforms.""" +from .views import platformsbp diff --git a/uploader/platforms/models.py b/uploader/platforms/models.py new file mode 100644 index 0000000..a859371 --- /dev/null +++ b/uploader/platforms/models.py @@ -0,0 +1,95 @@ +"""Handle db interactions for platforms.""" +from typing import Optional + +import MySQLdb as mdb +from MySQLdb.cursors import Cursor, DictCursor + +def platforms_by_species( + conn: mdb.Connection, + speciesid: int, + offset: int = 0, + limit: Optional[int] = None +) -> tuple[dict, ...]: + """Retrieve platforms by the species""" + _query = ("SELECT * FROM GeneChip WHERE SpeciesId=%s " + "ORDER BY GeneChipName ASC") + if bool(limit) and limit > 0:# type: ignore[operator] + _query = f"{_query} LIMIT {limit} OFFSET {offset}" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(_query, (speciesid,)) + return tuple(dict(row) for row in cursor.fetchall()) + + +def species_platforms_count(conn: mdb.Connection, species_id: int) -> int: + """Get the number of platforms in the database for a particular species.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT COUNT(GeneChipName) AS count FROM GeneChip " + "WHERE SpeciesId=%s", + (species_id,)) + return int(cursor.fetchone()["count"]) + + +def platform_by_id(conn: mdb.Connection, platformid: int) -> Optional[dict]: + """Retrieve a platform by its ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GeneChip WHERE Id=%s", + (platformid,)) + result = cursor.fetchone() + if bool(result): + return dict(result) + + return None + + +def platform_by_species_and_id( + conn: mdb.Connection, species_id: int, platformid: int +) -> Optional[dict]: + """Retrieve a platform by its species and ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GeneChip WHERE SpeciesId=%s AND Id=%s", + (species_id, platformid)) + result = cursor.fetchone()#pylint: disable=[duplicate-code] + if bool(result): + return dict(result) + + return None + + +def save_new_platform(# pylint: disable=[too-many-arguments] + cursor: Cursor, + species_id: int, + geo_platform: str, + platform_name: str, + platform_shortname: str, + platform_title: str, + go_tree_value: Optional[str] +) -> dict: + """Save a new platform to the database.""" + params = { + "species_id": species_id, + "GeoPlatform": geo_platform, + "GeneChipName": platform_name, + "Name": platform_shortname, + "Title": platform_title, + "GO_tree_value": go_tree_value + } + cursor.execute("SELECT SpeciesId, GeoPlatform FROM GeneChip") + assert (species_id, geo_platform) not in ( + (row["SpeciesId"], row["GeoPlatform"]) for row in cursor.fetchall()) + cursor.execute( + "INSERT INTO " + "GeneChip(SpeciesId, GeneChipName, Name, GeoPlatform, Title, GO_tree_value) " + "VALUES(" + "%(species_id)s, %(GeneChipName)s, %(Name)s, %(GeoPlatform)s, " + "%(Title)s, %(GO_tree_value)s" + ")", + params) + new_id = cursor.lastrowid + cursor.execute("UPDATE GeneChip SET GeneChipId=%s WHERE Id=%s", + (new_id, new_id)) + return { + **params, + "Id": new_id, + "GeneChipId": new_id + } diff --git a/uploader/platforms/views.py b/uploader/platforms/views.py new file mode 100644 index 0000000..114c1a9 --- /dev/null +++ b/uploader/platforms/views.py @@ -0,0 +1,118 @@ +"""The endpoints for the platforms""" +from MySQLdb.cursors import DictCursor +from gn_libs.mysqldb import database_connection +from flask import ( + flash, + request, + url_for, + redirect, + Blueprint, + current_app as app) + +from uploader.ui import make_template_renderer +from uploader.authorisation import require_login +from uploader.species.models import all_species, species_by_id +from uploader.datautils import safe_int, order_by_family, enumerate_sequence + +from .models import (save_new_platform, + platforms_by_species, + species_platforms_count) + +platformsbp = Blueprint("platforms", __name__) +render_template = make_template_renderer("platforms") + +@platformsbp.route("platforms", methods=["GET"]) +@require_login +def index(): + """Entry-point to the platforms feature.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(request.args.get("species_id")): + return render_template( + "platforms/index.html", + species=all_species(conn), + activelink="platforms") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.platforms.list_platforms")) + + species = species_by_id(conn, request.args["species_id"]) + if not bool(species): + flash("No species selected.", "alert-danger") + return redirect(url_for("species.platforms.index")) + + return redirect(url_for("species.platforms.list_platforms", + species_id=species["SpeciesId"])) + + +@platformsbp.route("<int:species_id>/platforms", methods=["GET"]) +@require_login +def list_platforms(species_id: int): + """List all the available genetic sequencing platforms.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("No species provided.", "alert-danger") + return redirect(url_for("species.platforms.index")) + + start_from = max(safe_int(request.args.get("start_from") or 0), 0) + count = safe_int(request.args.get("count") or 20) + return render_template( + "platforms/list-platforms.html", + species=species, + platforms=enumerate_sequence( + platforms_by_species(conn, + species_id, + offset=start_from, + limit=count), + start=start_from+1), + start_from=start_from, + count=count, + total_platforms=species_platforms_count(conn, species_id), + activelink="list-platforms") + + +@platformsbp.route("<int:species_id>/platforms/create", methods=["GET", "POST"]) +@require_login +def create_platform(species_id: int): + """Create a new genetic sequencing platform.""" + with (database_connection(app.config["SQL_URI"]) as conn, + conn.cursor(cursorclass=DictCursor) as cursor): + species = species_by_id(conn, species_id) + if not bool(species): + flash("No species provided.", "alert-danger") + return redirect(url_for("species.platforms.index")) + + if request.method == "GET": + return render_template( + "platforms/create-platform.html", + species=species, + activelink="create-platform") + + try: + form = request.form + _new_platform = save_new_platform( + cursor, + species_id, + form["geo-platform"], + form["platform-name"], + form["platform-shortname"], + form["platform-title"], + form.get("go-tree-value") or None) + except KeyError as _kerr: + flash(f"Required value for field {_kerr.args[0]} was not provided.", + "alert-danger") + return redirect(url_for("species.platforms.create_platform", + species_id=species_id)) + except AssertionError as _aerr: + flash(f"Platform with GeoPlatform value of '{form['geo-platform']}'" + f" already exists for species '{species['FullName']}'.", + "alert-danger") + return redirect(url_for("species.platforms.create_platform", + species_id=species_id)) + + flash("Platform created successfully", "alert-success") + return redirect(url_for("species.platforms.list_platforms", + species_id=species_id)) diff --git a/uploader/population/__init__.py b/uploader/population/__init__.py new file mode 100644 index 0000000..bf6bf3c --- /dev/null +++ b/uploader/population/__init__.py @@ -0,0 +1,3 @@ +"""Package to handle creation and management of Populations/InbredSets""" + +from .views import popbp diff --git a/uploader/population/models.py b/uploader/population/models.py new file mode 100644 index 0000000..d78a821 --- /dev/null +++ b/uploader/population/models.py @@ -0,0 +1,87 @@ +"""Functions for accessing the database relating to species populations.""" +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +def population_by_id(conn: mdb.Connection, population_id) -> dict: + """Get the grouping/population by id.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM InbredSet WHERE InbredSetId=%s", + (population_id,)) + return cursor.fetchone() + +def population_by_species_and_id( + conn: mdb.Connection, species_id, population_id) -> dict: + """Retrieve a population by its identifier and species.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM InbredSet WHERE SpeciesId=%s AND Id=%s", + (species_id, population_id)) + return cursor.fetchone() + +def populations_by_species(conn: mdb.Connection, speciesid) -> tuple: + "Retrieve group (InbredSet) information from the database." + with conn.cursor(cursorclass=DictCursor) as cursor: + query = "SELECT * FROM InbredSet WHERE SpeciesId=%s" + cursor.execute(query, (speciesid,)) + return tuple(cursor.fetchall()) + + return tuple() + + +def population_families(conn) -> tuple: + """Fetch the families under which populations are grouped.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT DISTINCT(Family) FROM InbredSet WHERE Family IS NOT NULL") + return tuple(row["Family"] for row in cursor.fetchall()) + + +def population_genetic_types(conn) -> tuple: + """Fetch the families under which populations are grouped.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT DISTINCT(GeneticType) FROM InbredSet WHERE GeneticType IS " + "NOT NULL") + return tuple(row["GeneticType"] for row in cursor.fetchall()) + + +def save_population(cursor: mdb.cursors.Cursor, population_details: dict) -> dict: + """Save the population details to the db.""" + cursor.execute("SELECT DISTINCT(Family), FamilyOrder FROM InbredSet " + "WHERE Family IS NOT NULL AND Family != '' " + "AND FamilyOrder IS NOT NULL " + "ORDER BY FamilyOrder ASC") + _families = { + row["Family"]: int(row["FamilyOrder"]) + for row in cursor.fetchall() + } + params = { + "MenuOrderId": 0, + "InbredSetId": 0, + "public": 2, + **population_details, + "FamilyOrder": _families.get( + population_details["Family"], + max((0,) + tuple(_families.values()))+1) + } + cursor.execute( + "INSERT INTO InbredSet(" + "InbredSetId, InbredSetName, Name, SpeciesId, FullName, " + "public, MappingMethodId, GeneticType, Family, FamilyOrder," + " MenuOrderId, InbredSetCode, Description" + ") " + "VALUES (" + "%(InbredSetId)s, %(InbredSetName)s, %(Name)s, %(SpeciesId)s, " + "%(FullName)s, %(public)s, %(MappingMethodId)s, %(GeneticType)s, " + "%(Family)s, %(FamilyOrder)s, %(MenuOrderId)s, %(InbredSetCode)s, " + "%(Description)s" + ")", + params) + new_id = cursor.lastrowid + cursor.execute("UPDATE InbredSet SET InbredSetId=%s WHERE Id=%s", + (new_id, new_id)) + return { + **params, + "Id": new_id, + "InbredSetId": new_id, + "population_id": new_id + } diff --git a/uploader/population/rqtl2.py b/uploader/population/rqtl2.py new file mode 100644 index 0000000..044cdd4 --- /dev/null +++ b/uploader/population/rqtl2.py @@ -0,0 +1,953 @@ +"""Module to handle uploading of R/qtl2 bundles."""#pylint: disable=[too-many-lines] +import sys +import json +import traceback +from pathlib import Path +from uuid import UUID, uuid4 +from functools import partial +from zipfile import ZipFile, is_zipfile +from typing import Union, Callable, Optional + +import MySQLdb as mdb +from redis import Redis +from MySQLdb.cursors import DictCursor +from gn_libs.mysqldb import database_connection +from flask import ( + flash, + escape, + request, + url_for, + redirect, + Response, + Blueprint, + render_template, + current_app as app) + +from r_qtl import r_qtl2 + +from uploader import jobs +from uploader.files import save_file, fullpath +from uploader.species.models import all_species +from uploader.db_utils import with_db_connection + +from uploader.authorisation import require_login +from uploader.platforms.models import platform_by_id, platforms_by_species +from uploader.db.averaging import averaging_methods, averaging_method_by_id +from uploader.db.tissues import all_tissues, tissue_by_id, create_new_tissue +from uploader.population.models import (populations_by_species, + population_by_species_and_id) +from uploader.species.models import species_by_id +from uploader.db.datasets import ( + geno_dataset_by_id, + geno_datasets_by_species_and_population, + + probeset_study_by_id, + probeset_create_study, + probeset_dataset_by_id, + probeset_create_dataset, + probeset_datasets_by_study, + probeset_studies_by_species_and_population) + +rqtl2 = Blueprint("rqtl2", __name__) + + +@rqtl2.route("/", methods=["GET", "POST"]) +@rqtl2.route("/select-species", methods=["GET", "POST"]) +@require_login +def select_species(): + """Select the species.""" + if request.method == "GET": + return render_template("expression-data/rqtl2/index.html", + species=with_db_connection(all_species)) + + species_id = request.form.get("species_id") + species = with_db_connection( + lambda conn: species_by_id(conn, species_id)) + if bool(species): + return redirect(url_for( + "species.populations.expression-data.rqtl2.select_population", + species_id=species_id)) + flash("Invalid species or no species selected!", "alert-error error-rqtl2") + return redirect(url_for("expression-data.rqtl2.select_species")) + + +@rqtl2.route("<int:species_id>/expression-data/rqtl2/select-population", + methods=["GET", "POST"]) +@require_login +def select_population(species_id: int): + """Select/Create the population to organise data under.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("Invalid species selected!", "alert-error error-rqtl2") + return redirect(url_for("expression-data.rqtl2.select_species")) + + if request.method == "GET": + return render_template( + "expression-data/rqtl2/select-population.html", + species=species, + populations=populations_by_species(conn, species_id)) + + population = population_by_species_and_id( + conn, species["SpeciesId"], request.form.get("inbredset_id")) + if not bool(population): + flash("Invalid Population!", "alert-error error-rqtl2") + return redirect( + url_for("expression-data.rqtl2.select_population", pgsrc="error"), + code=307) + + return redirect(url_for("expression-data.rqtl2.upload_rqtl2_bundle", + species_id=species["SpeciesId"], + population_id=population["InbredSetId"])) + + +class __RequestError__(Exception): #pylint: disable=[invalid-name] + """Internal class to avoid pylint's `too-many-return-statements` error.""" + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle"), + methods=["GET", "POST"]) +@require_login +def upload_rqtl2_bundle(species_id: int, population_id: int): + """Allow upload of R/qtl2 bundle.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + population = population_by_species_and_id( + conn, species["SpeciesId"], population_id) + if not bool(species): + flash("Invalid species!", "alert-error error-rqtl2") + return redirect(url_for("expression-data.rqtl2.select_species")) + if not bool(population): + flash("Invalid Population!", "alert-error error-rqtl2") + return redirect( + url_for("expression-data.rqtl2.select_population", pgsrc="error"), + code=307) + if request.method == "GET" or ( + request.method == "POST" + and bool(request.args.get("pgsrc"))): + return render_template( + "expression-data/rqtl2/upload-rqtl2-bundle-step-01.html", + species=species, + population=population) + + try: + app.logger.debug("Files in the form: %s", request.files) + the_file = save_file(request.files["rqtl2_bundle_file"], + Path(app.config["UPLOAD_FOLDER"])) + except AssertionError: + app.logger.debug(traceback.format_exc()) + flash("Please provide a valid R/qtl2 zip bundle.", + "alert-error error-rqtl2") + return redirect(url_for("expression-data.rqtl2.upload_rqtl2_bundle", + species_id=species_id, + population_id=population_id)) + + if not is_zipfile(str(the_file)): + app.logger.debug("The file is not a zip file.") + raise __RequestError__("Invalid file! Expected a zip file.") + + jobid = trigger_rqtl2_bundle_qc( + species_id, + population_id, + the_file, + request.files["rqtl2_bundle_file"].filename)#type: ignore[arg-type] + return redirect(url_for( + "expression-data.rqtl2.rqtl2_bundle_qc_status", jobid=jobid)) + + +def trigger_rqtl2_bundle_qc( + species_id: int, + population_id: int, + rqtl2bundle: Path, + originalfilename: str +) -> UUID: + """Trigger QC on the R/qtl2 bundle.""" + redisuri = app.config["REDIS_URL"] + with Redis.from_url(redisuri, decode_responses=True) as rconn: + jobid = uuid4() + redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"] + jobs.launch_job( + jobs.initialise_job( + rconn, + jobs.jobsnamespace(), + str(jobid), + [sys.executable, "-m", "scripts.qc_on_rqtl2_bundle", + app.config["SQL_URI"], app.config["REDIS_URL"], + jobs.jobsnamespace(), str(jobid), str(species_id), + str(population_id), "--redisexpiry", + str(redis_ttl_seconds)], + "rqtl2-bundle-qc-job", + redis_ttl_seconds, + {"job-metadata": json.dumps({ + "speciesid": species_id, + "populationid": population_id, + "rqtl2-bundle-file": str(rqtl2bundle.absolute()), + "original-filename": originalfilename})}), + redisuri, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + return jobid + + +@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>", + methods=["GET", "POST"]) +@require_login +def rqtl2_bundle_qc_status(jobid: UUID): + """Check the status of the QC jobs.""" + with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, + database_connection(app.config["SQL_URI"]) as dbconn): + try: + thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid) + messagelistname = thejob.get("log-messagelist") + logmessages = (rconn.lrange(messagelistname, 0, -1) + if bool(messagelistname) else []) + jobstatus = thejob["status"] + if jobstatus == "error": + return render_template( + "expression-data/rqtl2/rqtl2-qc-job-error.html", + job=thejob, + errorsgeneric=json.loads( + thejob.get("errors-generic", "[]")), + errorsgeno=json.loads( + thejob.get("errors-geno", "[]")), + errorspheno=json.loads( + thejob.get("errors-pheno", "[]")), + errorsphenose=json.loads( + thejob.get("errors-phenose", "[]")), + errorsphenocovar=json.loads( + thejob.get("errors-phenocovar", "[]")), + messages=logmessages) + if jobstatus == "success": + jobmeta = json.loads(thejob["job-metadata"]) + species = species_by_id(dbconn, jobmeta["speciesid"]) + return render_template( + "expression-data/rqtl2/rqtl2-qc-job-results.html", + species=species, + population=population_by_species_and_id( + dbconn, species["SpeciesId"], jobmeta["populationid"]), + rqtl2bundle=Path(jobmeta["rqtl2-bundle-file"]).name, + rqtl2bundleorig=jobmeta["original-filename"]) + + def compute_percentage(thejob, filetype) -> Union[str, None]: + if f"{filetype}-linecount" in thejob: + return "100" + if f"{filetype}-filesize" in thejob: + percent = ((int(thejob.get(f"{filetype}-checked", 0)) + / + int(thejob.get(f"{filetype}-filesize", 1))) + * 100) + return f"{percent:.2f}" + return None + + return render_template( + "expression-data/rqtl2/rqtl2-qc-job-status.html", + job=thejob, + geno_percent=compute_percentage(thejob, "geno"), + pheno_percent=compute_percentage(thejob, "pheno"), + phenose_percent=compute_percentage(thejob, "phenose"), + messages=logmessages) + except jobs.JobNotFound: + return render_template("expression-data/rqtl2/no-such-job.html", jobid=jobid) + + +def redirect_on_error(flaskroute, **kwargs): + """Utility to redirect on error""" + return redirect(url_for(flaskroute, **kwargs, pgsrc="error"), + code=(307 if request.method == "POST" else 302)) + + +def check_species(conn: mdb.Connection, formargs: dict) -> Optional[ + tuple[str, Response]]: + """ + Check whether the 'species_id' value is provided, and whether a + corresponding species exists in the database. + + Maybe give the function a better name...""" + speciespage = redirect_on_error("expression-data.rqtl2.select_species") + if "species_id" not in formargs: + return "You MUST provide the Species identifier.", speciespage + + if not bool(species_by_id(conn, formargs["species_id"])): + return "No species with the provided identifier exists.", speciespage + + return None + + +def check_population(conn: mdb.Connection, + formargs: dict, + species_id) -> Optional[tuple[str, Response]]: + """ + Check whether the 'population_id' value is provided, and whether a + corresponding population exists in the database. + + Maybe give the function a better name...""" + poppage = redirect_on_error( + "expression-data.rqtl2.select_species", species_id=species_id) + if "population_id" not in formargs: + return "You MUST provide the Population identifier.", poppage + + if not bool(population_by_species_and_id( + conn, species_id, formargs["population_id"])): + return "No population with the provided identifier exists.", poppage + + return None + + +def check_r_qtl2_bundle(formargs: dict, + species_id, + population_id) -> Optional[tuple[str, Response]]: + """Check for the existence of the R/qtl2 bundle.""" + fileuploadpage = redirect_on_error("expression-data.rqtl2.upload_rqtl2_bundle", + species_id=species_id, + population_id=population_id) + if not "rqtl2_bundle_file" in formargs: + return ( + "You MUST provide a R/qtl2 zip bundle for expression-data.", fileuploadpage) + + if not Path(fullpath(formargs["rqtl2_bundle_file"])).exists(): + return "No R/qtl2 bundle with the given name exists.", fileuploadpage + + return None + + +def check_geno_dataset(conn: mdb.Connection, + formargs: dict, + species_id, + population_id) -> Optional[tuple[str, Response]]: + """Check for the Genotype dataset.""" + genodsetpg = redirect_on_error("expression-data.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id) + if not bool(formargs.get("geno-dataset-id")): + return ( + "You MUST provide a valid Genotype dataset identifier", genodsetpg) + + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s", + (formargs["geno-dataset-id"],)) + results = cursor.fetchall() + if not bool(results): + return ("No genotype dataset with the provided identifier exists.", + genodsetpg) + if len(results) > 1: + return ( + "Data corruption: More than one genotype dataset with the same " + "identifier.", + genodsetpg) + + return None + +def check_tissue( + conn: mdb.Connection,formargs: dict) -> Optional[tuple[str, Response]]: + """Check for tissue/organ/biological material.""" + selectdsetpg = redirect_on_error("expression-data.rqtl2.select_dataset_info", + species_id=formargs["species_id"], + population_id=formargs["population_id"]) + if not bool(formargs.get("tissueid", "").strip()): + return ("No tissue/organ/biological material provided.", selectdsetpg) + + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Tissue WHERE Id=%s", + (formargs["tissueid"],)) + results = cursor.fetchall() + if not bool(results): + return ("No tissue/organ with the provided identifier exists.", + selectdsetpg) + + if len(results) > 1: + return ( + "Data corruption: More than one tissue/organ with the same " + "identifier.", + selectdsetpg) + + return None + + +def check_probe_study(conn: mdb.Connection, + formargs: dict, + species_id, + population_id) -> Optional[tuple[str, Response]]: + """Check for the ProbeSet study.""" + dsetinfopg = redirect_on_error("expression-data.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id) + if not bool(formargs.get("probe-study-id")): + return "No probeset study was selected!", dsetinfopg + + if not bool(probeset_study_by_id(conn, formargs["probe-study-id"])): + return ("No probeset study with the provided identifier exists", + dsetinfopg) + + return None + + +def check_probe_dataset(conn: mdb.Connection, + formargs: dict, + species_id, + population_id) -> Optional[tuple[str, Response]]: + """Check for the ProbeSet dataset.""" + dsetinfopg = redirect_on_error("expression-data.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id) + if not bool(formargs.get("probe-dataset-id")): + return "No probeset dataset was selected!", dsetinfopg + + if not bool(probeset_dataset_by_id(conn, formargs["probe-dataset-id"])): + return ("No probeset dataset with the provided identifier exists", + dsetinfopg) + + return None + + +def with_errors(endpointthunk: Callable, *checkfns): + """Run 'endpointthunk' with error checking.""" + formargs = {**dict(request.args), **dict(request.form)} + errors = tuple(item for item in (_fn(formargs=formargs) for _fn in checkfns) + if item is not None) + if len(errors) > 0: + flash(errors[0][0], "alert-error error-rqtl2") + return errors[0][1] + + return endpointthunk() + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/select-geno-dataset"), + methods=["POST"]) +@require_login +def select_geno_dataset(species_id: int, population_id: int): + """Select from existing geno datasets.""" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + geno_dset = geno_datasets_by_species_and_population( + conn, species_id, population_id) + if not bool(geno_dset): + flash("No genotype dataset was provided!", + "alert-error error-rqtl2") + return redirect(url_for("expression-data.rqtl2.select_geno_dataset", + species_id=species_id, + population_id=population_id, + pgsrc="error"), + code=307) + + flash("Genotype accepted", "alert-success error-rqtl2") + return redirect(url_for("expression-data.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id, + pgsrc="expression-data.rqtl2.select_geno_dataset"), + code=307) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/select-tissue"), + methods=["POST"]) +@require_login +def select_tissue(species_id: int, population_id: int): + """Select from existing tissues.""" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + if not bool(request.form.get("tissueid", "").strip()): + flash("Invalid tissue selection!", + "alert-error error-select-tissue error-rqtl2") + + return redirect(url_for("expression-data.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id, + pgsrc="expression-data.rqtl2.select_geno_dataset"), + code=307) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id)) + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/create-tissue"), + methods=["POST"]) +@require_login +def create_tissue(species_id: int, population_id: int): + """Add new tissue, organ or biological material to the system.""" + form = request.form + datasetinfopage = redirect( + url_for("expression-data.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id, + pgsrc="expression-data.rqtl2.select_geno_dataset"), + code=307) + with database_connection(app.config["SQL_URI"]) as conn: + tissuename = form.get("tissuename", "").strip() + tissueshortname = form.get("tissueshortname", "").strip() + if not bool(tissuename): + flash("Organ/Tissue name MUST be provided.", + "alert-error error-create-tissue error-rqtl2") + return datasetinfopage + + if not bool(tissueshortname): + flash("Organ/Tissue short name MUST be provided.", + "alert-error error-create-tissue error-rqtl2") + return datasetinfopage + + try: + tissue = create_new_tissue(conn, tissuename, tissueshortname) + flash("Tissue created successfully!", "alert-success") + return render_template( + "expression-data/rqtl2/create-tissue-success.html", + species=species_by_id(conn, species_id), + population=population_by_species_and_id( + conn, species_id, population_id), + rqtl2_bundle_file=request.form["rqtl2_bundle_file"], + geno_dataset=geno_dataset_by_id( + conn, + int(request.form["geno-dataset-id"])), + tissue=tissue) + except mdb.IntegrityError as _ierr: + flash("Tissue/Organ with that short name already exists!", + "alert-error error-create-tissue error-rqtl2") + return datasetinfopage + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/select-probeset-study"), + methods=["POST"]) +@require_login +def select_probeset_study(species_id: int, population_id: int): + """Select or create a probeset study.""" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + summary_page = redirect(url_for("expression-data.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id), + code=307) + if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))): + flash("Invalid study selected!", "alert-error error-rqtl2") + return summary_page + + return summary_page + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_tissue, conn=conn), + partial(check_probe_study, + conn=conn, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/select-probeset-dataset"), + methods=["POST"]) +@require_login +def select_probeset_dataset(species_id: int, population_id: int): + """Select or create a probeset dataset.""" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + summary_page = redirect(url_for("expression-data.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id), + code=307) + if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))): + flash("Invalid study selected!", "alert-error error-rqtl2") + return summary_page + + return summary_page + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_tissue, conn=conn), + partial(check_probe_study, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_probe_dataset, + conn=conn, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/create-probeset-study"), + methods=["POST"]) +@require_login +def create_probeset_study(species_id: int, population_id: int): + """Create a new probeset study.""" + errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-study" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + form = request.form + dataset_info_page = redirect( + url_for("expression-data.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id), + code=307) + + if not (bool(form.get("platformid")) and + bool(platform_by_id(conn, int(form["platformid"])))): + flash("Invalid platform selected.", errorclasses) + return dataset_info_page + + if not (bool(form.get("tissueid")) and + bool(tissue_by_id(conn, int(form["tissueid"])))): + flash("Invalid tissue selected.", errorclasses) + return dataset_info_page + + studyname = form["studyname"] + try: + study = probeset_create_study( + conn, population_id, int(form["platformid"]), int(form["tissueid"]), + studyname, form.get("studyfullname") or "", + form.get("studyshortname") or "") + except mdb.IntegrityError as _ierr: + flash(f"ProbeSet study with name '{escape(studyname)}' already " + "exists.", + errorclasses) + return dataset_info_page + return render_template( + "expression-data/rqtl2/create-probe-study-success.html", + species=species_by_id(conn, species_id), + population=population_by_species_and_id( + conn, species_id, population_id), + rqtl2_bundle_file=request.form["rqtl2_bundle_file"], + geno_dataset=geno_dataset_by_id( + conn, + int(request.form["geno-dataset-id"])), + study=study) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_tissue, conn=conn)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/create-probeset-dataset"), + methods=["POST"]) +@require_login +def create_probeset_dataset(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements] + """Create a new probeset dataset.""" + errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-dataset" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__():#pylint: disable=[too-many-return-statements] + form = request.form + summary_page = redirect(url_for("expression-data.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id), + code=307) + if not bool(form.get("averageid")): + flash("Averaging method not selected!", errorclasses) + return summary_page + if not bool(form.get("datasetname")): + flash("Dataset name not provided!", errorclasses) + return summary_page + if not bool(form.get("datasetfullname")): + flash("Dataset full name not provided!", errorclasses) + return summary_page + + tissue = tissue_by_id(conn, form.get("tissueid", "").strip()) + + study = probeset_study_by_id(conn, int(form["probe-study-id"])) + if not bool(study): + flash("Invalid ProbeSet study provided!", errorclasses) + return summary_page + + avgmethod = averaging_method_by_id(conn, int(form["averageid"])) + if not bool(avgmethod): + flash("Invalid averaging method provided!", errorclasses) + return summary_page + + try: + dset = probeset_create_dataset(conn, + int(form["probe-study-id"]), + int(form["averageid"]), + form["datasetname"], + form["datasetfullname"], + form["datasetshortname"], + form["datasetpublic"] == "on", + form.get( + "datasetdatascale", "log2")) + except mdb.IntegrityError as _ierr: + app.logger.debug("Possible integrity error: %s", traceback.format_exc()) + flash(("IntegrityError: The data you provided has some errors: " + f"{_ierr.args}"), + errorclasses) + return summary_page + except Exception as _exc:# pylint: disable=[broad-except] + app.logger.debug("Error creating ProbeSet dataset: %s", + traceback.format_exc()) + flash(("There was a problem creating your dataset. Please try " + "again."), + errorclasses) + return summary_page + return render_template( + "expression-data/rqtl2/create-probe-dataset-success.html", + species=species_by_id(conn, species_id), + population=population_by_species_and_id( + conn, species_id, population_id), + rqtl2_bundle_file=request.form["rqtl2_bundle_file"], + geno_dataset=geno_dataset_by_id( + conn, + int(request.form["geno-dataset-id"])), + tissue=tissue, + study=study, + avgmethod=avgmethod, + dataset=dset) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_tissue, conn=conn), + partial(check_probe_study, + conn=conn, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/dataset-info"), + methods=["POST"]) +@require_login +def select_dataset_info(species_id: int, population_id: int): + """ + If `geno` files exist in the R/qtl2 bundle, prompt user to provide the + dataset the genotypes belong to. + """ + form = request.form + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + species = species_by_id(conn, species_id) + population = population_by_species_and_id( + conn, species_id, population_id) + thefile = fullpath(form["rqtl2_bundle_file"]) + with ZipFile(str(thefile), "r") as zfile: + cdata = r_qtl2.control_data(zfile) + + geno_dataset = geno_dataset_by_id( + conn,form.get("geno-dataset-id", "").strip()) + if "geno" in cdata and not bool(form.get("geno-dataset-id")): + return render_template( + "expression-data/rqtl2/select-geno-dataset.html", + species=species, + population=population, + rqtl2_bundle_file=thefile.name, + datasets=geno_datasets_by_species_and_population( + conn, species_id, population_id)) + + tissue = tissue_by_id(conn, form.get("tissueid", "").strip()) + if "pheno" in cdata and not bool(tissue): + return render_template( + "expression-data/rqtl2/select-tissue.html", + species=species, + population=population, + rqtl2_bundle_file=thefile.name, + geno_dataset=geno_dataset, + studies=probeset_studies_by_species_and_population( + conn, species_id, population_id), + platforms=platforms_by_species(conn, species_id), + tissues=all_tissues(conn)) + + probeset_study = probeset_study_by_id( + conn, form.get("probe-study-id", "").strip()) + if "pheno" in cdata and not bool(probeset_study): + return render_template( + "expression-data/rqtl2/select-probeset-study-id.html", + species=species, + population=population, + rqtl2_bundle_file=thefile.name, + geno_dataset=geno_dataset, + studies=probeset_studies_by_species_and_population( + conn, species_id, population_id), + platforms=platforms_by_species(conn, species_id), + tissue=tissue) + probeset_study = probeset_study_by_id( + conn, int(form["probe-study-id"])) + + probeset_dataset = probeset_dataset_by_id( + conn, form.get("probe-dataset-id", "").strip()) + if "pheno" in cdata and not bool(probeset_dataset): + return render_template( + "expression-data/rqtl2/select-probeset-dataset.html", + species=species, + population=population, + rqtl2_bundle_file=thefile.name, + geno_dataset=geno_dataset, + probe_study=probeset_study, + tissue=tissue, + datasets=probeset_datasets_by_study( + conn, int(form["probe-study-id"])), + avgmethods=averaging_methods(conn)) + + return render_template("expression-data/rqtl2/summary-info.html", + species=species, + population=population, + rqtl2_bundle_file=thefile.name, + geno_dataset=geno_dataset, + tissue=tissue, + probe_study=probeset_study, + probe_dataset=probeset_dataset) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/confirm-bundle-details"), + methods=["POST"]) +@require_login +def confirm_bundle_details(species_id: int, population_id: int): + """Confirm the details and trigger R/qtl2 bundle processing...""" + redisuri = app.config["REDIS_URL"] + with (database_connection(app.config["SQL_URI"]) as conn, + Redis.from_url(redisuri, decode_responses=True) as rconn): + def __thunk__(): + redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"] + jobid = str(uuid4()) + _job = jobs.launch_job( + jobs.initialise_job( + rconn, + jobs.jobsnamespace(), + jobid, + [ + sys.executable, "-m", "scripts.process_rqtl2_bundle", + app.config["SQL_URI"], app.config["REDIS_URL"], + jobs.jobsnamespace(), jobid, "--redisexpiry", + str(redis_ttl_seconds)], + "R/qtl2 Bundle Upload", + redis_ttl_seconds, + { + "bundle-metadata": json.dumps({ + "speciesid": species_id, + "populationid": population_id, + "rqtl2-bundle-file": str(fullpath( + request.form["rqtl2_bundle_file"])), + "geno-dataset-id": request.form.get( + "geno-dataset-id", ""), + "probe-study-id": request.form.get( + "probe-study-id", ""), + "probe-dataset-id": request.form.get( + "probe-dataset-id", ""), + **({ + "platformid": probeset_study_by_id( + conn, + int(request.form["probe-study-id"]))["ChipId"] + } if bool(request.form.get("probe-study-id")) else {}) + }) + }), + redisuri, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + + return redirect(url_for("expression-data.rqtl2.rqtl2_processing_status", + jobid=jobid)) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_probe_study, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_probe_dataset, + conn=conn, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route("/status/<uuid:jobid>") +def rqtl2_processing_status(jobid: UUID): + """Retrieve the status of the job processing the uploaded R/qtl2 bundle.""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid) + + messagelistname = thejob.get("log-messagelist") + logmessages = (rconn.lrange(messagelistname, 0, -1) + if bool(messagelistname) else []) + + if thejob["status"] == "error": + return render_template( + "expression-data/rqtl2/rqtl2-job-error.html", + job=thejob, + messages=logmessages) + if thejob["status"] == "success": + return render_template( + "expression-data/rqtl2/rqtl2-job-results.html", + job=thejob, + messages=logmessages) + + return render_template( + "expression-data/rqtl2/rqtl2-job-status.html", + job=thejob, + messages=logmessages) + except jobs.JobNotFound as _exc: + return render_template("expression-data/rqtl2/no-such-job.html", + jobid=jobid) diff --git a/uploader/population/views.py b/uploader/population/views.py new file mode 100644 index 0000000..f42e547 --- /dev/null +++ b/uploader/population/views.py @@ -0,0 +1,217 @@ +"""Views dealing with populations/inbredsets""" +import json +import base64 + +from MySQLdb.cursors import DictCursor +from gn_libs.mysqldb import database_connection +from flask import (flash, + escape, + request, + url_for, + redirect, + Blueprint, + current_app as app) + +from uploader.samples.views import samplesbp +from uploader.oauth2.client import oauth2_post +from uploader.ui import make_template_renderer +from uploader.authorisation import require_login +from uploader.genotypes.views import genotypesbp +from uploader.datautils import enumerate_sequence +from uploader.phenotypes.views import phenotypesbp +from uploader.expression_data.views import exprdatabp +from uploader.monadic_requests import make_either_error_handler +from uploader.input_validation import is_valid_representative_name +from uploader.species.models import (all_species, + species_by_id, + order_species_by_family) + +from .models import (save_population, + population_families, + populations_by_species, + population_genetic_types, + population_by_species_and_id) + +__active_link__ = "populations" +popbp = Blueprint("populations", __name__) +popbp.register_blueprint(samplesbp, url_prefix="/") +popbp.register_blueprint(genotypesbp, url_prefix="/") +popbp.register_blueprint(phenotypesbp, url_prefix="/") +popbp.register_blueprint(exprdatabp, url_prefix="/") +render_template = make_template_renderer("populations") + + +@popbp.route("/populations", methods=["GET", "POST"]) +@require_login +def index(): + """Entry point for populations.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(request.args.get("species_id")): + return render_template( + "populations/index.html", + species=all_species(conn), + activelink="populations") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.list_species_populations")) + + species = species_by_id(conn, request.args.get("species_id")) + if not bool(species): + flash("Invalid species identifier provided!", "alert-danger") + return redirect(url_for("species.populations.index")) + return redirect(url_for("species.populations.list_species_populations", + species_id=species["SpeciesId"])) + +@popbp.route("/<int:species_id>/populations", methods=["GET"]) +@require_login +def list_species_populations(species_id: int): + """List a particular species' populations.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("No species was found for given ID.", "alert-danger") + return redirect(url_for("species.populations.index")) + return render_template( + "populations/list-populations.html", + species=species, + populations=enumerate_sequence(populations_by_species( + conn, species_id)), + activelink="list-populations") + + +@popbp.route("/<int:species_id>/populations/create", methods=["GET", "POST"]) +@require_login +def create_population(species_id: int): + """Create a new population.""" + with (database_connection(app.config["SQL_URI"]) as conn, + conn.cursor(cursorclass=DictCursor) as cursor): + species = species_by_id(conn, species_id) + + if request.method == "GET": + error_values = request.args.get("error_values") + if not bool(error_values): + error_values = base64.b64encode( + '{"errors":{}, "error_values": {}}'.encode("utf8") + ).decode("utf8") + + error_values = json.loads(base64.b64decode( + error_values.encode("utf8")).decode("utf8"))# type: ignore[union-attr] + return render_template( + "populations/create-population.html", + species=species, + families = population_families(conn), + genetic_types = population_genetic_types(conn), + mapping_methods=( + {"id": "0", "value": "No mapping support"}, + {"id": "1", "value": "GEMMA, QTLReaper, R/qtl"}, + {"id": "2", "value": "GEMMA"}, + {"id": "3", "value": "R/qtl"}, + {"id": "4", "value": "GEMMA, PLINK"}), + return_to=(request.args.get("return_to") or ""), + activelink="create-population", + **error_values) + + if not bool(species): + flash("You must select a species.", "alert-danger") + return redirect(url_for("species.populations.index")) + + errors: tuple[tuple[str, str], ...] = tuple() + + population_name = (request.form.get( + "population_name") or "").strip() + if not bool(population_name): + errors = errors + (("population_name", + "You must provide a name for the population!"),) + + if not is_valid_representative_name(population_name): + errors = errors + (( + "population_name", + "The population name can only contain letters, numbers, " + "hyphens and underscores."),) + + population_fullname = (request.form.get( + "population_fullname") or "").strip() + if not bool(population_fullname): + errors = errors + ( + ("population_fullname", "Full Name MUST be provided."),) + + if bool(errors): + values = base64.b64encode( + json.dumps({ + "errors": dict(errors), + "error_values": dict(request.form) + }).encode("utf8")) + return redirect(url_for("species.populations.create_population", + species_id=species["SpeciesId"], + error_values=values)) + + new_population = save_population(cursor, { + "SpeciesId": species["SpeciesId"], + "Name": population_name, + "InbredSetName": population_fullname, + "FullName": population_fullname, + "InbredSetCode": request.form.get("population_code") or None, + "Description": request.form.get("population_description") or None, + "Family": request.form.get("population_family") or None, + "MappingMethodId": request.form.get("population_mapping_method_id"), + "GeneticType": request.form.get("population_genetic_type") or None + }) + + def __flash_success__(_success): + flash("Successfully created population " + f"{escape(new_population['FullName'])}.", + "alert-success") + return_to = request.form.get("return_to") or "" + if return_to: + return redirect(url_for( + return_to, + species_id=species["SpeciesId"], + population_id=new_population["InbredSetId"])) + return redirect(url_for( + "species.populations.view_population", + species_id=species["SpeciesId"], + population_id=new_population["InbredSetId"])) + + app.logger.debug("We begin setting up the privileges here…") + return oauth2_post( + "auth/resource/populations/create", + json={ + **dict(request.form), + "species_id": species_id, + "population_id": new_population["Id"], + "public": "on" + } + ).either( + make_either_error_handler( + "There was an error creating the population"), + __flash_success__) + + +@popbp.route("/<int:species_id>/populations/<int:population_id>", + methods=["GET"]) +@require_login +def view_population(species_id: int, population_id: int): + """View the details of a population.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + population = population_by_species_and_id(conn, species_id, population_id) + error = False + + if not bool(species): + flash("You must select a species.", "alert-danger") + error = True + + if not bool(population): + flash("You must select a population.", "alert-danger") + error = True + + if error: + return redirect(url_for("species.populations.index")) + + return render_template("populations/view-population.html", + species=species, + population=population, + activelink="view-population") diff --git a/uploader/request_checks.py b/uploader/request_checks.py new file mode 100644 index 0000000..f1d8027 --- /dev/null +++ b/uploader/request_checks.py @@ -0,0 +1,75 @@ +"""Functions to perform common checks. + +These are useful for reusability, and hence maintainability of the code. +""" +from functools import wraps + +from gn_libs.mysqldb import database_connection +from flask import flash, url_for, redirect, current_app as app + +from uploader.species.models import species_by_id +from uploader.population.models import population_by_species_and_id + +def with_species(redirect_uri: str): + """Ensure the species actually exists.""" + def __decorator__(function): + @wraps(function) + def __with_species__(**kwargs): + try: + species_id = int(kwargs.get("species_id")) + if not bool(species_id): + flash("Expected species_id value to be present!", + "alert-danger") + return redirect(url_for(redirect_uri)) + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("Could not find species with that ID", + "alert-danger") + return redirect(url_for(redirect_uri)) + except ValueError as _verr: + app.logger.debug( + "Exception converting value to integer: %s", + kwargs.get("species_id"), + exc_info=True) + flash("Expected an integer for 'species_id' value.", + "alert-danger") + return redirect(url_for(redirect_uri)) + return function(**{**kwargs, "species": species}) + return __with_species__ + return __decorator__ + + +def with_population(species_redirect_uri: str, redirect_uri: str): + """Ensure the population actually exists.""" + def __decorator__(function): + @wraps(function) + @with_species(redirect_uri=species_redirect_uri) + def __with_population__(**kwargs): + try: + species_id = int(kwargs["species_id"]) + population_id = int(kwargs.get("population_id")) + select_population_uri = redirect(url_for( + redirect_uri, species_id=species_id)) + if not bool(population_id): + flash("Expected population_id value to be present!", + "alert-danger") + return select_population_uri + with database_connection(app.config["SQL_URI"]) as conn: + population = population_by_species_and_id( + conn, species_id, population_id) + if not bool(population): + flash("Could not find population with that ID", + "alert-danger") + return select_population_uri + except ValueError as _verr: + app.logger.debug( + "Exception converting value to integer: %s", + kwargs.get("population_id"), + exc_info=True) + flash("Expected an integer for 'population_id' value.", + "alert-danger") + return select_population_uri + return function(**{**kwargs, "population": population}) + return __with_population__ + return __decorator__ diff --git a/uploader/samples/__init__.py b/uploader/samples/__init__.py new file mode 100644 index 0000000..1bd6d2d --- /dev/null +++ b/uploader/samples/__init__.py @@ -0,0 +1 @@ +"""Samples package. Handle samples uploads and editing.""" diff --git a/uploader/samples/models.py b/uploader/samples/models.py new file mode 100644 index 0000000..d7d5384 --- /dev/null +++ b/uploader/samples/models.py @@ -0,0 +1,104 @@ +"""Functions for handling samples.""" +import csv +from typing import Iterator + +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +from functional_tools import take + +def samples_by_species_and_population( + conn: mdb.Connection, + species_id: int, + population_id: int +) -> tuple[dict, ...]: + """Fetch the samples by their species and population.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT iset.InbredSetId, s.* FROM InbredSet AS iset " + "INNER JOIN StrainXRef AS sxr ON iset.InbredSetId=sxr.InbredSetId " + "INNER JOIN Strain AS s ON sxr.StrainId=s.Id " + "WHERE s.SpeciesId=%(species_id)s " + "AND iset.InbredSetId=%(population_id)s", + {"species_id": species_id, "population_id": population_id}) + return tuple(cursor.fetchall()) + + +def read_samples_file(filepath, separator: str, firstlineheading: bool, **kwargs) -> Iterator[dict]: + """Read the samples file.""" + with open(filepath, "r", encoding="utf-8") as inputfile: + reader = csv.DictReader( + inputfile, + fieldnames=( + None if firstlineheading + else ("Name", "Name2", "Symbol", "Alias")), + delimiter=separator, + quotechar=kwargs.get("quotechar", '"')) + for row in reader: + yield row + + +def save_samples_data(conn: mdb.Connection, + speciesid: int, + file_data: Iterator[dict]): + """Save the samples to DB.""" + data = ({**row, "SpeciesId": speciesid} for row in file_data) + total = 0 + with conn.cursor() as cursor: + while True: + batch = take(data, 5000) + if len(batch) == 0: + break + cursor.executemany( + "INSERT INTO Strain(Name, Name2, SpeciesId, Symbol, Alias) " + "VALUES(" + " %(Name)s, %(Name2)s, %(SpeciesId)s, %(Symbol)s, %(Alias)s" + ") ON DUPLICATE KEY UPDATE Name=Name", + batch) + total += len(batch) + print(f"\tSaved {total} samples total so far.") + + +def cross_reference_samples(conn: mdb.Connection, + species_id: int, + population_id: int, + strain_names: Iterator[str]): + """Link samples to their population.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT MAX(OrderId) AS loid FROM StrainXRef WHERE InbredSetId=%s", + (population_id,)) + last_order_id = (cursor.fetchone()["loid"] or 10) + total = 0 + while True: + batch = take(strain_names, 5000) + if len(batch) == 0: + break + params_str = ", ".join(["%s"] * len(batch)) + ## This query is slow -- investigate. + cursor.execute( + "SELECT s.Id FROM Strain AS s LEFT JOIN StrainXRef AS sx " + "ON s.Id = sx.StrainId WHERE s.SpeciesId=%s AND s.Name IN " + f"({params_str}) AND sx.StrainId IS NULL", + (species_id,) + tuple(batch)) + strain_ids = (sid["Id"] for sid in cursor.fetchall()) + params = tuple({ + "pop_id": population_id, + "strain_id": strain_id, + "order_id": last_order_id + (order_id * 10), + "mapping": "N", + "pedigree": None + } for order_id, strain_id in enumerate(strain_ids, start=1)) + cursor.executemany( + "INSERT INTO StrainXRef( " + " InbredSetId, StrainId, OrderId, Used_for_mapping, PedigreeStatus" + ")" + "VALUES (" + " %(pop_id)s, %(strain_id)s, %(order_id)s, %(mapping)s, " + " %(pedigree)s" + ")", + params) + last_order_id += (len(params) * 10) + total += len(batch) + print(f"\t{total} total samples cross-referenced to the population " + "so far.") diff --git a/uploader/samples/views.py b/uploader/samples/views.py new file mode 100644 index 0000000..95a6f8c --- /dev/null +++ b/uploader/samples/views.py @@ -0,0 +1,292 @@ +"""Code regarding samples""" +import os +import sys +import uuid +from pathlib import Path + +from redis import Redis +from flask import (flash, + request, + url_for, + redirect, + Blueprint, + current_app as app) + +from uploader import jobs +from uploader.files import save_file +from uploader.ui import make_template_renderer +from uploader.authorisation import require_login +from uploader.request_checks import with_population +from uploader.input_validation import is_integer_input +from uploader.datautils import safe_int, order_by_family, enumerate_sequence +from uploader.population.models import population_by_id, populations_by_species +from uploader.db_utils import (with_db_connection, + database_connection, + with_redis_connection) +from uploader.species.models import (all_species, + species_by_id, + order_species_by_family) + +from .models import samples_by_species_and_population + +samplesbp = Blueprint("samples", __name__) +render_template = make_template_renderer("samples") + +@samplesbp.route("/samples", methods=["GET"]) +@require_login +def index(): + """Direct entry-point for uploading/handling the samples.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(request.args.get("species_id")): + return render_template( + "samples/index.html", + species=all_species(conn), + activelink="samples") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.samples.select_population")) + + species = species_by_id(conn, request.args.get("species_id")) + if not bool(species): + flash("No such species!", "alert-danger") + return redirect(url_for("species.populations.samples.index")) + return redirect(url_for("species.populations.samples.select_population", + species_id=species["SpeciesId"])) + + +@samplesbp.route("<int:species_id>/samples/select-population", methods=["GET"]) +@require_login +def select_population(species_id: int): + """Select the population to use for the samples.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("Invalid species!", "alert-danger") + return redirect(url_for("species.populations.samples.index")) + + if not bool(request.args.get("population_id")): + return render_template("samples/select-population.html", + species=species, + populations=populations_by_species( + conn, + species_id), + activelink="samples") + + population_id = request.args["population_id"] + if population_id == "CREATE-POPULATION": + return redirect(url_for( + "species.populations.create_population", + species_id=species["SpeciesId"], + return_to="species.populations.samples.list_samples")) + + population = population_by_id(conn, request.args.get("population_id")) + if not bool(population): + flash("Population not found!", "alert-danger") + return redirect(url_for( + "species.populations.samples.select_population", + species_id=species_id)) + + return redirect(url_for("species.populations.samples.list_samples", + species_id=species_id, + population_id=population["Id"])) + +@samplesbp.route("<int:species_id>/populations/<int:population_id>/samples") +@require_login +def list_samples(species_id: int, population_id: int): + """ + List the samples in a particular population and give the ability to upload + new ones. + """ + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("Invalid species!", "alert-danger") + return redirect(url_for("species.populations.samples.index")) + + population = population_by_id(conn, population_id) + if not bool(population): + flash("Population not found!", "alert-danger") + return redirect(url_for( + "species.populations.samples.select_population", + species_id=species_id)) + + all_samples = enumerate_sequence(samples_by_species_and_population( + conn, species_id, population_id)) + total_samples = len(all_samples) + offset = max(safe_int(request.args.get("from") or 0), 0) + count = int(request.args.get("count") or 20) + return render_template("samples/list-samples.html", + species=species, + population=population, + samples=all_samples[offset:offset+count], + offset=offset, + count=count, + total_samples=total_samples, + activelink="list-samples") + + +def build_sample_upload_job(# pylint: disable=[too-many-arguments] + speciesid: int, + populationid: int, + samplesfile: Path, + separator: str, + firstlineheading: bool, + quotechar: str): + """Define the async command to run the actual samples data upload.""" + return [ + sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"], + str(speciesid), str(populationid), str(samplesfile.absolute()), + separator, f"--redisuri={app.config['REDIS_URL']}", + f"--quotechar={quotechar}" + ] + (["--firstlineheading"] if firstlineheading else []) + + +@samplesbp.route("<int:species_id>/populations/<int:population_id>/upload-samples", + methods=["GET", "POST"]) +@require_login +def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements] + """Upload the samples.""" + samples_uploads_page = redirect(url_for( + "species.populations.samples.upload_samples", + species_id=species_id, + population_id=population_id)) + if not is_integer_input(species_id): + flash("You did not provide a valid species. Please select one to " + "continue.", + "alert-danger") + return redirect(url_for("expression-data.samples.select_species")) + species = with_db_connection(lambda conn: species_by_id(conn, species_id)) + if not bool(species): + flash("Species with given ID was not found.", "alert-danger") + return redirect(url_for("expression-data.samples.select_species")) + + if not is_integer_input(population_id): + flash("You did not provide a valid population. Please select one " + "to continue.", + "alert-danger") + return redirect(url_for("species.populations.samples.select_population", + species_id=species_id), + code=307) + population = with_db_connection( + lambda conn: population_by_id(conn, int(population_id))) + if not bool(population): + flash("Invalid grouping/population!", "alert-error") + return redirect(url_for("species.populations.samples.select_population", + species_id=species_id), + code=307) + + if request.method == "GET" or request.files.get("samples_file") is None: + return render_template("samples/upload-samples.html", + species=species, + population=population) + + try: + samples_file = save_file(request.files["samples_file"], + Path(app.config["UPLOAD_FOLDER"])) + except AssertionError: + flash("You need to provide a file with the samples data.", + "alert-error") + return samples_uploads_page + + firstlineheading = (request.form.get("first_line_heading") == "on") + + separator = request.form.get("separator", ",") + if separator == "other": + separator = request.form.get("other_separator", ",") + if not bool(separator): + flash("You need to provide a separator character.", "alert-error") + return samples_uploads_page + + quotechar = (request.form.get("field_delimiter", '"') or '"') + + redisuri = app.config["REDIS_URL"] + with Redis.from_url(redisuri, decode_responses=True) as rconn: + #TODO: Add a QC step here — what do we check? + # 1. Does any sample in the uploaded file exist within the database? + # If yes, what is/are its/their species and population? + # 2. If yes 1. above, provide error with notes on which species and + # populations already own the samples. + the_job = jobs.launch_job( + jobs.initialise_job( + rconn, + jobs.jobsnamespace(), + str(uuid.uuid4()), + build_sample_upload_job( + species["SpeciesId"], + population["InbredSetId"], + samples_file, + separator, + firstlineheading, + quotechar), + "samples_upload", + app.config["JOBS_TTL_SECONDS"], + {"job_name": f"Samples Upload: {samples_file.name}"}), + redisuri, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + return redirect(url_for( + "species.populations.samples.upload_status", + species_id=species_id, + population_id=population_id, + job_id=the_job["jobid"])) + + +@samplesbp.route("<int:species_id>/populations/<int:population_id>/" + "upload-samples/status/<uuid:job_id>", + methods=["GET"]) +@require_login +@with_population(species_redirect_uri="species.populations.samples.index", + redirect_uri="species.populations.samples.select_population") +def upload_status(species: dict, population: dict, job_id: uuid.UUID, **kwargs):# pylint: disable=[unused-argument] + """Check on the status of a samples upload job.""" + job = with_redis_connection(lambda rconn: jobs.job( + rconn, jobs.jobsnamespace(), job_id)) + if job: + status = job["status"] + if status == "success": + return render_template("samples/upload-success.html", + job=job, + species=species, + population=population,) + + if status == "error": + return redirect(url_for( + "species.populations.samples.upload_failure", job_id=job_id)) + + error_filename = Path(jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) + if error_filename.exists(): + stat = os.stat(error_filename) + if stat.st_size > 0: + return redirect(url_for( + "samples.upload_failure", job_id=job_id)) + + return render_template("samples/upload-progress.html", + species=species, + population=population, + job=job) # maybe also handle this? + + return render_template("no_such_job.html", + job_id=job_id, + species=species, + population=population), 400 + +@samplesbp.route("/upload/failure/<uuid:job_id>", methods=["GET"]) +@require_login +def upload_failure(job_id: uuid.UUID): + """Display the errors of the samples upload failure.""" + job = with_redis_connection(lambda rconn: jobs.job( + rconn, jobs.jobsnamespace(), job_id)) + if not bool(job): + return render_template("no_such_job.html", job_id=job_id), 400 + + error_filename = Path(jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) + if error_filename.exists(): + stat = os.stat(error_filename) + if stat.st_size > 0: + return render_template("worker_failure.html", job_id=job_id) + + return render_template("samples/upload-failure.html", job=job) diff --git a/uploader/session.py b/uploader/session.py new file mode 100644 index 0000000..b538187 --- /dev/null +++ b/uploader/session.py @@ -0,0 +1,118 @@ +"""Deal with user sessions""" +from uuid import UUID, uuid4 +from datetime import datetime +from typing import Any, Optional, TypedDict + +from authlib.jose import KeySet +from flask import request, session +from pymonad.either import Left, Right, Either + + +class UserDetails(TypedDict): + """Session information relating specifically to the user.""" + user_id: UUID + name: str + email: str + token: Either + logged_in: bool + + +class SessionInfo(TypedDict): + """All Session information we save.""" + session_id: UUID + user: UserDetails + anon_id: UUID + user_agent: str + ip_addr: str + masquerade: Optional[UserDetails] + auth_server_jwks: Optional[dict[str, Any]] + + +__SESSION_KEY__ = "GN::uploader::session_info" # Do not use this outside this module!! + + +def clear_session_info(): + """Clears the session.""" + session.pop(__SESSION_KEY__) + + +def save_session_info(sess_info: SessionInfo) -> SessionInfo: + """Save `session_info`.""" + # T0d0: if it is an existing session, verify that certain important security + # bits have not changed before saving. + # old_session_info = session.get(__SESSION_KEY__) + # if bool(old_session_info): + # if old_session_info["user_agent"] == request.headers.get("User-Agent"): + # session[__SESSION_KEY__] = sess_info + # return sess_info + # # request session verification + # return verify_session(sess_info) + # New session + session[__SESSION_KEY__] = sess_info + return sess_info + + +def session_info() -> SessionInfo: + """Retrieve the session information""" + anon_id = uuid4() + return save_session_info( + session.get(__SESSION_KEY__, { + "session_id": uuid4(), + "user": { + "user_id": anon_id, + "name": "Anonymous User", + "email": "anon@ymous.user", + "token": Left("INVALID-TOKEN"), + "logged_in": False + }, + "anon_id": anon_id, + "user_agent": request.headers.get("User-Agent"), + "ip_addr": request.environ.get("HTTP_X_FORWARDED_FOR", + request.remote_addr), + "masquerading": None + })) + + +def set_user_token(token: str) -> SessionInfo: + """Set the user's token.""" + info = session_info() + return save_session_info({ + **info, "user": {**info["user"], "token": Right(token)}})#type: ignore[misc] + + +def set_user_details(userdets: UserDetails) -> SessionInfo: + """Set the user details information""" + return save_session_info({**session_info(), "user": userdets})#type: ignore[misc] + +def user_details() -> UserDetails: + """Retrieve user details.""" + return session_info()["user"] + +def user_token() -> Either: + """Retrieve the user token.""" + return session_info()["user"]["token"] + + +def set_auth_server_jwks(keyset: KeySet) -> KeySet: + """Update the JSON Web Keys in the session.""" + save_session_info({ + **session_info(),# type: ignore[misc] + "auth_server_jwks": { + "last-updated": datetime.now().timestamp(), + "jwks": keyset.as_dict() + } + }) + return keyset + + +def toggle_token_refreshing(): + """Toggle the state of the token_refreshing variable.""" + _session = session_info() + return save_session_info({ + **_session, + "token_refreshing": not _session.get("token_refreshing", False)}) + + +def is_token_refreshing(): + """Returns whether the token is being refreshed or not.""" + return session_info().get("token_refreshing", False) diff --git a/uploader/species/__init__.py b/uploader/species/__init__.py new file mode 100644 index 0000000..83f2165 --- /dev/null +++ b/uploader/species/__init__.py @@ -0,0 +1,2 @@ +"""Package to handle creation and management of species.""" +from .views import speciesbp diff --git a/uploader/species/models.py b/uploader/species/models.py new file mode 100644 index 0000000..9477aa8 --- /dev/null +++ b/uploader/species/models.py @@ -0,0 +1,154 @@ +"""Database functions for species.""" +import math +from typing import Optional +from functools import reduce + +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +def all_species(conn: mdb.Connection) -> tuple: + "Retrieve the species from the database." + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT Id AS SpeciesId, SpeciesName, LOWER(Name) AS Name, " + "MenuName, FullName, TaxonomyId, Family, FamilyOrderId, OrderId " + "FROM Species ORDER BY FamilyOrderId ASC, OrderID ASC") + return tuple(cursor.fetchall()) + + return tuple() + +def order_species_by_family(species: tuple[dict, ...]) -> list: + """Order the species by their family""" + def __family_order_id__(item): + orderid = item["FamilyOrderId"] + return math.inf if orderid is None else orderid + def __order__(ordered, current): + _key = (__family_order_id__(current), current["Family"]) + return { + **ordered, + _key: ordered.get(_key, tuple()) + (current,) + } + ordered = reduce(__order__, species, {})# type: ignore[var-annotated] + return sorted(tuple(ordered.items()), key=lambda item: item[0][0]) + + +def species_by_id(conn: mdb.Connection, speciesid) -> dict: + "Retrieve the species from the database by id." + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT Id AS SpeciesId, SpeciesName, LOWER(Name) AS Name, " + "MenuName, FullName, TaxonomyId, Family, FamilyOrderId, OrderId " + "FROM Species WHERE SpeciesId=%s", + (speciesid,)) + return cursor.fetchone() + + +def save_species(conn: mdb.Connection, + common_name: str, + scientific_name: str, + family: str, + taxon_id: Optional[str] = None) -> dict: + """ + Save a new species to the database. + + Parameters + ---------- + conn: A connection to the MariaDB database. + taxon_id: The taxonomy identifier for the new species. + common_name: The species' common name. + scientific_name; The species' scientific name. + """ + genus, *species_name = scientific_name.split(" ") + species_name = " ".join(species_name) + families = species_families(conn) + with conn.cursor() as cursor: + cursor.execute("SELECT MAX(OrderId) FROM Species") + species = { + "common_name": common_name, + "common_name_lower": common_name.lower(), + "menu_name": f"{common_name} ({genus[0]}. {species_name.lower()})", + "scientific_name": scientific_name, + "family": family, + "family_order": families.get(family, 999999), + "taxon_id": taxon_id, + "species_order": cursor.fetchone()[0] + 5 + } + cursor.execute( + "INSERT INTO Species(" + "SpeciesName, Name, MenuName, FullName, Family, FamilyOrderId, " + "TaxonomyId, OrderId" + ") VALUES (" + "%(common_name)s, %(common_name_lower)s, %(menu_name)s, " + "%(scientific_name)s, %(family)s, %(family_order)s, %(taxon_id)s, " + "%(species_order)s" + ")", + species) + species_id = cursor.lastrowid + cursor.execute("UPDATE Species SET SpeciesId=%s WHERE Id=%s", + (species_id, species_id)) + return { + **species, + "species_id": species_id + } + + +def update_species(# pylint: disable=[too-many-arguments] + conn: mdb.Connection, + species_id: int, + common_name: str, + scientific_name: str, + family: str, + family_order: int, + species_order: int +): + """Update a species' details. + + Parameters + ---------- + conn: A connection to the MariaDB database. + species_id: The species identifier + + Key-Word Arguments + ------------------ + common_name: A layman's name for the species + scientific_name: A binomial nomenclature name for the species + family: The grouping under which the species falls + family_order: The ordering for the "family" above + species_order: The ordering of this species in relation to others + """ + with conn.cursor(cursorclass=DictCursor) as cursor: + genus, *species_name = scientific_name.split(" ") + species_name = " ".join(species_name) + species = { + "species_id": species_id, + "common_name": common_name, + "common_name_lower": common_name.lower(), + "menu_name": f"{common_name} ({genus[0]}. {species_name.lower()})", + "scientific_name": scientific_name, + "family": family, + "family_order": family_order, + "species_order": species_order + } + cursor.execute( + "UPDATE Species SET " + "SpeciesName=%(common_name)s, " + "Name=%(common_name_lower)s, " + "MenuName=%(menu_name)s, " + "FullName=%(scientific_name)s, " + "Family=%(family)s, " + "FamilyOrderId=%(family_order)s, " + "OrderId=%(species_order)s " + "WHERE Id=%(species_id)s", + species) + + +def species_families(conn: mdb.Connection) -> dict: + """Retrieve the families under which species are grouped.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT DISTINCT(Family), FamilyOrderId FROM Species " + "WHERE Family IS NOT NULL") + return { + fam["Family"]: fam["FamilyOrderId"] + for fam in cursor.fetchall() + } diff --git a/uploader/species/views.py b/uploader/species/views.py new file mode 100644 index 0000000..f0798d6 --- /dev/null +++ b/uploader/species/views.py @@ -0,0 +1,210 @@ +"""Endpoints handling species.""" +from pymonad.either import Left, Right, Either +from gn_libs.mysqldb import database_connection +from flask import (flash, + escape, + request, + url_for, + redirect, + Blueprint, + current_app as app) + +from uploader.population import popbp +from uploader.platforms import platformsbp +from uploader.ui import make_template_renderer +from uploader.oauth2.client import oauth2_get, oauth2_post +from uploader.authorisation import require_login, require_token +from uploader.datautils import order_by_family, enumerate_sequence + +from .models import (all_species, + save_species, + species_by_id, + update_species, + species_families) + + +speciesbp = Blueprint("species", __name__) +speciesbp.register_blueprint(popbp, url_prefix="/") +speciesbp.register_blueprint(platformsbp, url_prefix="/") +render_template = make_template_renderer("species") + + +@speciesbp.route("/", methods=["GET"]) +@require_login +def list_species(): + """List and display all the species in the database.""" + with database_connection(app.config["SQL_URI"]) as conn: + return render_template("species/list-species.html", + allspecies=enumerate_sequence(all_species(conn))) + +@speciesbp.route("/<int:species_id>", methods=["GET"]) +@require_login +def view_species(species_id: int): + """View details of a particular species and menus to act upon it.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if bool(species): + return render_template("species/view-species.html", + species=species, + activelink="view-species") + flash("Could not find a species with the given identifier.", + "alert-danger") + return redirect(url_for("species.view_species")) + +@speciesbp.route("/create", methods=["GET", "POST"]) +@require_login +def create_species(): + """Create a new species.""" + # We can use uniprot's API to fetch the details with something like + # https://rest.uniprot.org/taxonomy/<taxonID> e.g. + # https://rest.uniprot.org/taxonomy/6239 + with (database_connection(app.config["SQL_URI"]) as conn, + conn.cursor() as cursor): + if request.method == "GET": + return render_template("species/create-species.html", + families=species_families(conn), + return_to=( + request.args.get("return_to") or ""), + activelink="create-species") + + error = False + taxon_id = request.form.get("species_taxonomy_id", "").strip() or None + + common_name = request.form.get("common_name", "").strip() + if not bool(common_name): + flash("The common species name MUST be provided.", "alert-danger") + error = True + + scientific_name = request.form.get("scientific_name", "").strip() + if not bool(scientific_name): + flash("The species' scientific name MUST be provided.", + "alert-danger") + error = True + + parts = tuple(name.strip() for name in scientific_name.split(" ")) + if (len(parts) != 2 and len(parts) != 3) or not all(bool(name) for name in parts): + flash("The scientific name you provided is invalid.", "alert-danger") + error = True + + cursor.execute( + "SELECT * FROM Species WHERE FullName=%s", (scientific_name,)) + res = cursor.fetchone() + if bool(res): + flash("A species already exists with the provided scientific name.", + "alert-danger") + error = True + + family = request.form.get("species_family", "").strip() + if not bool(family): + flash("The species' family MUST be selected.", "alert-danger") + error = True + + if bool(taxon_id): + cursor.execute( + "SELECT * FROM Species WHERE TaxonomyId=%s", (taxon_id,)) + res = cursor.fetchone() + if bool(res): + flash("A species already exists with the provided scientific name.", + "alert-danger") + error = True + + if error: + return redirect(url_for("species.create_species", + common_name=common_name, + scientific_name=scientific_name, + taxon_id=taxon_id)) + + species = save_species( + conn, common_name, scientific_name, family, taxon_id) + flash( + f"You have successfully added species '{species['scientific_name']} " + f"({species['common_name']})'.", + "alert-success") + + return_to = request.form.get("return_to").strip() + if return_to: + return redirect(url_for(return_to, species_id=species["species_id"])) + return redirect(url_for("species.view_species", species_id=species["species_id"])) + + +@speciesbp.route("/<int:species_id>/edit-extra", methods=["GET", "POST"]) +@require_login +@require_token +#def edit_species(species_id: int): +def edit_species_extra(token: dict, species_id: int):# pylint: disable=[unused-argument] + """Edit a species' details. + + Parameters + ---------- + token: A JWT token used for authorisation. + species_id: An identifier for the species being edited. + """ + def __failure__(res): + app.logger.debug( + "There was an error in the attempt to edit the species: %s", res) + flash(res, "alert-danger") + return redirect(url_for("species.view_species", species_id=species_id)) + + def __system_resource_uuid__(resources) -> Either: + sys_res = [ + resource for resource in resources + if resource["resource_category"]["resource_category_key"] == "system" + ] + if len(sys_res) != 1: + return Left("Could not find/identify a valid system resource.") + return Right(sys_res[0]["resource_id"]) + + def __check_privileges__(authorisations): + if len(authorisations.items()) != 1: + return Left("Got authorisations for more than a single resource!") + + auths = tuple(authorisations.items())[0][1] + authorised = "system:species:edit-extra-info" in tuple( + privilege["privilege_id"] + for role in auths["roles"] + for privilege in role["privileges"]) + if authorised: + return Right(authorised) + return Left("You are not authorised to edit species extra details.") + + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + all_the_species = all_species(conn) + families = species_families(conn) + family_order = tuple( + item[0] for item in order_by_family(all_the_species) + if item[0][1] is not None) + if bool(species) and request.method == "GET": + return oauth2_get("auth/user/resources").then( + __system_resource_uuid__ + ).then( + lambda resource_id: oauth2_post( + "auth/resource/authorisation", + json={"resource-ids": [resource_id]}) + ).then(__check_privileges__).then( + lambda authorisations: render_template( + "species/edit-species.html", + species=species, + families=families, + family_order=family_order, + max_order_id = max( + row["OrderId"] for row in all_the_species + if row["OrderId"] is not None), + activelink="edit-species") + ).either(__failure__, lambda res: res) + + if bool(species) and request.method == "POST": + update_species(conn, + species_id, + request.form["species_name"], + request.form["species_fullname"], + request.form["species_family"], + int(request.form["species_familyorderid"]), + int(request.form["species_orderid"])) + flash("Updated species successfully.", "alert-success") + return redirect(url_for("species.edit_species_extra", + species_id=species_id)) + + flash("Species with the given identifier was not found!", + "alert-danger") + return redirect(url_for("species.list_species")) diff --git a/uploader/static/css/custom-bootstrap.css b/uploader/static/css/custom-bootstrap.css new file mode 100644 index 0000000..67f1199 --- /dev/null +++ b/uploader/static/css/custom-bootstrap.css @@ -0,0 +1,23 @@ +/** Customize some bootstrap selectors **/ +.btn { + text-transform: capitalize; +} + +.navbar-inverse { + background-color: #336699; + border-color: #080808; + color: #FFFFFF; + background-image: none; +} + +.navbar-inverse .navbar-nav>li>a { + color: #FFFFFF; +} + +.navbar-nav > li > a { + padding: 5px; +} + +.navbar { + min-height: 30px; +} diff --git a/uploader/static/css/styles.css b/uploader/static/css/styles.css new file mode 100644 index 0000000..35c0627 --- /dev/null +++ b/uploader/static/css/styles.css @@ -0,0 +1,154 @@ +* { + box-sizing: border-box; +} + +body { + margin: 0.7em; + display: grid; + grid-template-columns: 1fr 9fr; + grid-gap: 20px; + + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + font-style: normal; + font-size: 20px; +} + +#header { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 8fr 2fr; + + /* Content styling */ + background-color: #336699; + color: #FFFFFF; + border-radius: 3px; + min-height: 30px; +} + +#header #header-text { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; + + /* Content styling */ + padding-left: 1em; +} + +#header #header-nav { + /* Place it in the parent element */ + grid-column-start: 2; + grid-column-end: 3; +} + +#header #header-nav .nav li a { + /* Content styling */ + color: #FFFFFF; + background: #4477AA; + border: solid 5px #336699; + border-radius: 5px; + font-size: 0.7em; + text-align: center; + padding: 1px 7px; +} + +#nav-sidebar { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; +} + +#nav-sidebar .nav li a:hover { + border-radius: 0.5em; +} + +#nav-sidebar .nav .activemenu { + border-style: solid; + border-radius: 0.5em; + border-color: #AAAAAA; + background-color: #EFEFEF; +} + +#main { + /* Place it in the parent element */ + grid-column-start: 2; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 1fr; + grid-template-rows: 4em 100%; + grid-gap: 1em; +} + +#main #pagetitle { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Content-styling */ + border-radius: 3px; + background-color: #88BBEE; +} + +#main #pagetitle .title { + font-size: 1.4em; + text-transform: capitalize; + padding-left: 0.5em; +} + +#main #all-content { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 7fr 3fr; /* For a maximum screen width of 1366 pixels */ + grid-gap: 1.5em; +} + +#main #all-content .row { + margin: 0 2px; +} + +#main #all-content #main-content { + background: #FFFFFF; + max-width: 950px; +} + +#pagetitle .breadcrumb { + background: none; + text-transform: capitalize; + font-size: 0.75em; +} + +#pagetitle .breadcrumb .active a { + color: #333333; +} + +#pagetitle .breadcrumb a { + color: #666666; +} + +.heading { + border-bottom: solid #EEBB88; + text-transform: capitalize; +} + +.subheading { + padding: 1em 0 0.1em 0.5em; + border-bottom: solid #88BBEE; + text-transform: capitalize; +} + +input[type="search"] { + border-radius: 5px; +} + +.btn { + text-transform: Capitalize; +} diff --git a/uploader/static/css/two-column-with-separator.css b/uploader/static/css/two-column-with-separator.css new file mode 100644 index 0000000..b6efd46 --- /dev/null +++ b/uploader/static/css/two-column-with-separator.css @@ -0,0 +1,27 @@ +.two-column-with-separator { + display: grid; + grid-template-columns: 9fr 1fr 9fr; +} + +.two-col-sep-col1 { + grid-column: 1 / 2; +} + +.two-col-sep-separator { + grid-column: 2 / 3; + text-align: center; + color: #FE3535; + font-weight: bolder; +} + +.two-col-sep-col2 { + grid-column: 3 / 4; +} + +.two-col-sep-col1, .two-col-sep-col2 { + border-style: solid; + border-color: #FE3535; + border-width: 1px; + border-radius: 2em; + padding: 2em 3em 2em 3em; +} diff --git a/uploader/static/images/CITGLogo.png b/uploader/static/images/CITGLogo.png Binary files differnew file mode 100644 index 0000000..ae99fed --- /dev/null +++ b/uploader/static/images/CITGLogo.png diff --git a/uploader/static/js/files.js b/uploader/static/js/files.js new file mode 100644 index 0000000..9d6bca1 --- /dev/null +++ b/uploader/static/js/files.js @@ -0,0 +1,118 @@ +var readFirstNLines = (thefile, count, process_content_fns) => { + var reader = new FileReader(); + if(typeof thefile !== "undefined" && thefile !== null) { + reader.addEventListener("load", (event) => { + var content = event + .target + .result + .split("\n") + .slice(0, count) + .map((line) => {return line.trim("\r");}); + process_content_fns.forEach((fn) => {fn(content);}); + }); + reader.readAsText(thefile); + } +}; +var read_first_n_lines = readFirstNLines; + + +var readBinaryFile = (file) => { + return new Promise((resolve, reject) => { + var _reader = new FileReader(); + _reader.onload = (event) => {resolve(_reader.result);}; + _reader.readAsArrayBuffer(file); + }); +}; + + +var Uint8ArrayToHex = (arr) => { + var toHex = (val) => { + _hex = val.toString(16); + if(_hex.length < 2) { + return "0" + val; + } + return _hex; + }; + _hexstr = "" + arr.forEach((val) => {_hexstr += toHex(val)}); + return _hexstr +}; + + +var computeFileChecksum = (file) => { + return readBinaryFile(file) + .then((content) => { + return window.crypto.subtle.digest( + "SHA-256", new Uint8Array(content)); + }).then((digest) => { + return Uint8ArrayToHex(new Uint8Array(digest)) + }); +}; + + +var defaultResumableHandler = (event) => { + throw new Error("Please provide a valid event handler!"); +}; + +var addHandler = (resumable, handlername, handler) => { + if(resumable.support) { + resumable.on(handlername, (handler || defaultResumableHandler)); + } + return resumable; +}; + + +var makeResumableHandler = (handlername) => { + return (resumable, handler) => { + return addHandler(resumable, handlername, handler); + }; +}; + + +var fileSuccessHandler = makeResumableHandler("fileSuccess"); +var fileProgressHandler = makeResumableHandler("fileProgress"); +var fileAddedHandler = makeResumableHandler("fileAdded"); +var filesAddedHandler = makeResumableHandler("filesAdded"); +var filesRetryHandler = makeResumableHandler("filesRetry"); +var filesErrorHandler = makeResumableHandler("filesError"); +var uploadStartHandler = makeResumableHandler("uploadStart"); +var completeHandler = makeResumableHandler("complete"); +var progressHandler = makeResumableHandler("progress"); +var errorHandler = makeResumableHandler("error"); + + +var markResumableDragAndDropElement = (resumable, fileinput, droparea, browsebutton) => { + if(resumable.support) { + //Hide file input element and display drag&drop UI + add_class(fileinput, "hidden"); + remove_class(droparea, "hidden"); + + // Define UI elements for browse and drag&drop + resumable.assignDrop(droparea); + resumable.assignBrowse(browsebutton); + } + + return resumable; +}; + + +var makeResumableElement = (targeturi, fileinput, droparea, uploadbutton, filetype) => { + var resumable = Resumable({ + target: targeturi, + fileType: filetype, + maxFiles: 1, + forceChunkSize: true, + generateUniqueIdentifier: (file, event) => { + return computeFileChecksum(file).then((checksum) => { + var _relativePath = (file.webkitRelativePath + || file.relativePath + || file.fileName + || file.name); + return checksum + "-" + _relativePath.replace( + /[^a-zA-Z0-9_-]/img, ""); + }); + } + }); + + return resumable; +}; diff --git a/uploader/static/js/misc.js b/uploader/static/js/misc.js new file mode 100644 index 0000000..cf7b39e --- /dev/null +++ b/uploader/static/js/misc.js @@ -0,0 +1,6 @@ +"Miscellaneous functions and event-handlers" + +$(".not-implemented").click((event) => { + event.preventDefault(); + alert("This feature is not implemented yet. Please bear with us."); +}); diff --git a/uploader/static/js/populations.js b/uploader/static/js/populations.js new file mode 100644 index 0000000..ded9b10 --- /dev/null +++ b/uploader/static/js/populations.js @@ -0,0 +1,40 @@ +var populationDataTable = (populationdata) => { + var lengthMenu = [10, 25, 50, 100, 1000]; + if(populationdata.length > 1000) { + lengthMenu.push(populationdata.length) + } + $("#tbl-select-population").DataTable({ + responsive: true, + lengthMenu: lengthMenu, + language: { + processing: "Processing… Please wait.", + loadingRecords: "Loading population — Please wait.", + lengthMenu: "Show _MENU_ populations", + info: "Showing _START_ to _END_ of _TOTAL_ populations" + }, + data: populationdata, + columns: [ + { + data: (apopulation) => { + return `<input type="radio" name="population_id"` + + `id="rdo_population_id_${apopulation.InbredSetId}" ` + + `value="${apopulation.InbredSetId}">`; + } + }, + { + data: (apopulation) => { + return `<label for="rdo_population_id_${apopulation.InbredSetId}" ` + + `class="control-label" style="font-weight: 1;">` + + `${apopulation.FullName} (${apopulation.InbredSetName})` + + `</label>`; + } + } + ] + }); +}; + + +$(() => { + populationDataTable(JSON.parse( + $("#tbl-select-population").attr("data-populations-list"))); +}); diff --git a/uploader/static/js/select_platform.js b/uploader/static/js/select_platform.js new file mode 100644 index 0000000..4fdd865 --- /dev/null +++ b/uploader/static/js/select_platform.js @@ -0,0 +1,70 @@ +function radio_column(chip) { + col = document.createElement("td"); + radio = document.createElement("input"); + radio.setAttribute("type", "radio"); + radio.setAttribute("name", "genechipid"); + radio.setAttribute("value", chip["GeneChipId"]); + radio.setAttribute("required", "required"); + col.appendChild(radio); + return col; +} + +function setup_genechips(genechip_data) { + columns = ["GeneChipId", "GeneChipName"] + submit_button = document.querySelector( + "#select-platform-form button[type='submit']"); + elt = document.getElementById( + "genechips-table").getElementsByTagName("tbody")[0]; + remove_children(elt); + if((genechip_data === undefined) || genechip_data.length === 0) { + row = document.createElement("tr"); + col = document.createElement("td"); + col.setAttribute("colspan", "3"); + text = document.createTextNode("No chips found for selected species"); + col.appendChild(text); + row.appendChild(col); + elt.appendChild(row); + submit_button.setAttribute("disabled", true); + return false; + } + + submit_button.removeAttribute("disabled") + genechip_data.forEach(chip => { + row = document.createElement("tr"); + row.appendChild(radio_column(chip)); + columns.forEach(column => { + col = document.createElement("td"); + content = document.createTextNode(chip[column]); + col.appendChild(content); + row.appendChild(col); + }); + elt.appendChild(row); + }); +} + +function genechips() { + return JSON.parse( + document.getElementById("select-platform-form").getAttribute( + "data-genechips")); +} + +function update_genechips(event) { + genec = genechips(); + + species_elt = document.getElementById("species"); + + if(event.target == species_elt) { + setup_genechips(genec[species_elt.value.toLowerCase()]); + } +} + +function select_row_radio(row) { + radio = row.getElementsByTagName( + "td")[0].getElementsByTagName( + "input")[0]; + if(radio === undefined) { + return false; + } + radio.setAttribute("checked", "checked"); + return true; +} diff --git a/uploader/static/js/species.js b/uploader/static/js/species.js new file mode 100644 index 0000000..b070725 --- /dev/null +++ b/uploader/static/js/species.js @@ -0,0 +1,39 @@ +var speciesDataTable = (speciesdata) => { + var lengthMenu = [10, 25, 50, 100, 1000]; + if(speciesdata.length > 1000) { + lengthMenu.push(speciesdata.length) + } + $("#tbl-select-species").DataTable({ + responsive: true, + lengthMenu: lengthMenu, + language: { + processing: "Processing… Please wait.", + loadingRecords: "Loading species — Please wait.", + lengthMenu: "Show _MENU_ species", + info: "Showing _START_ to _END_ of _TOTAL_ species" + }, + data: speciesdata, + columns: [ + { + data: (aspecies) => { + return `<input type="radio" name="species_id"` + + `id="rdo_species_id_${aspecies.SpeciesId}" ` + + `value="${aspecies.SpeciesId}">`; + } + }, + { + data: (aspecies) => { + return `<label for="rdo_species_id_${aspecies.SpeciesId}" ` + + `class="control-label" style="font-weight: 1;">` + + `${aspecies.FullName} (${aspecies.SpeciesName})` + + `</label>`; + } + } + ] + }); +}; + +$(() => { + speciesDataTable(JSON.parse( + $("#tbl-select-species").attr("data-species-list"))); +}); diff --git a/uploader/static/js/upload_progress.js b/uploader/static/js/upload_progress.js new file mode 100644 index 0000000..9638b36 --- /dev/null +++ b/uploader/static/js/upload_progress.js @@ -0,0 +1,97 @@ +function make_processing_indicator(elt) { + var count = 0; + return function() { + var message = "Finalising upload and saving file: " + if(count > 5) { + count = 1; + } + for(i = 0; i < count; i++) { + message = message + "."; + } + elt.innerHTML = message + count = count + 1 + }; +} + +function make_progress_updater(file, indicator_elt) { + var progress_bar = indicator_elt.querySelector("#progress-bar"); + var progress_text = indicator_elt.querySelector("#progress-text"); + var extra_text = indicator_elt.querySelector("#progress-extra-text"); + return function(event) { + if(event.loaded <= file.size) { + var percent = Math.round((event.loaded / file.size) * 100); + progress_bar.value = percent + progress_text.innerHTML = "Uploading: " + percent + "%"; + extra_text.setAttribute("class", "hidden") + } + + if(event.loaded == event.total) { + progress_bar.value = 100; + progress_text.innerHTML = "Uploaded: 100%"; + extra_text.setAttribute("class", null); + intv = setInterval(make_processing_indicator(extra_text), 400); + setTimeout(function() {clearTimeout(intv);}, 20000); + } + }; +} + +function setup_cancel_upload(request, indicator_elt) { + document.getElementById("btn-cancel-upload").addEventListener( + "click", function(event) { + event.preventDefault(); + request.abort(); + }); +} + +function setup_request(file, progress_indicator_elt) { + var request = new XMLHttpRequest(); + var updater = make_progress_updater(file, progress_indicator_elt); + request.upload.addEventListener("progress", updater); + request.onload = function(event) { + document.location.assign(request.responseURL); + }; + setup_cancel_upload(request, progress_indicator_elt) + return request; +} + +function selected_filetype(radios) { + for(idx = 0; idx < radios.length; idx++) { + if(radios[idx].checked) { + return radios[idx].value; + } + } +} + +function make_data_uploader(setup_formdata) { + return function(event) { + event.preventDefault(); + + var pindicator = document.getElementById("upload-progress-indicator"); + + var form = event.target; + var the_file = form.querySelector("input[type='file']").files[0]; + if(the_file === undefined) { + form.querySelector("input[type='file']").parentElement.setAttribute( + "class", "invalid-input"); + error_elt = form.querySelector("#no-file-error"); + if(error_elt !== undefined) { + error_elt.setAttribute("style", "display: block;"); + } + return false; + } + var formdata = setup_formdata(form); + + document.getElementById("progress-filename").innerHTML = the_file.name; + var request = setup_request(the_file, pindicator); + request.open(form.getAttribute("method"), form.getAttribute("action")); + request.send(formdata); + return false; + } +} + + +function setup_upload_handlers(formid, datauploader) { + console.info("Setting up the upload handlers.") + upload_form = document.getElementById(formid); + upload_form.addEventListener("submit", datauploader); +} diff --git a/uploader/static/js/upload_samples.js b/uploader/static/js/upload_samples.js new file mode 100644 index 0000000..aed536f --- /dev/null +++ b/uploader/static/js/upload_samples.js @@ -0,0 +1,132 @@ +/* + * Read the file content and set the `data-preview-content` attribute on the + * file element + */ +function read_first_n_lines(event, + fileelement, + numlines, + firstlineheading = true) { + var thefile = fileelement.files[0]; + var reader = new FileReader(); + reader.addEventListener("load", (event) => { + var filecontent = event.target.result.split( + "\n").slice( + 0, (numlines + (firstlineheading ? 1 : 0))).map( + (line) => {return line.trim("\r");}); + fileelement.setAttribute( + "data-preview-content", JSON.stringify(filecontent)); + display_preview(event); + }) + reader.readAsText(thefile); +} + +function remove_rows(preview_table) { + var table_body = preview_table.getElementsByTagName("tbody")[0]; + while(table_body.children.length > 0) { + table_body.removeChild(table_body.children.item(0)); + } +} + +/* + * Display error row + */ +function display_error_row(preview_table, error_message) { + remove_rows(preview_table); + row = document.createElement("tr"); + cell = document.createElement("td"); + cell.setAttribute("colspan", 4); + cell.innerHTML = error_message; + row.appendChild(cell); + preview_table.getElementsByTagName("tbody")[0].appendChild(row); +} + +function strip(str, chars) { + var end = str.length; + var start = 0 + for(var j = str.length; j > 0; j--) { + if(!chars.includes(str[j - 1])) { + break; + } + end = end - 1; + } + for(var i = 0; i < end; i++) { + if(!chars.includes(str[i])) { + break; + } + start = start + 1; + } + return str.slice(start, end); +} + +function process_preview_data(preview_data, separator, delimiter) { + return preview_data.map((line) => { + return line.split(separator).map((field) => { + return strip(field, delimiter); + }); + }); +} + +function render_preview(preview_table, preview_data) { + remove_rows(preview_table); + var table_body = preview_table.getElementsByTagName("tbody")[0]; + preview_data.forEach((line) => { + var row = document.createElement("tr"); + line.forEach((field) => { + var cell = document.createElement("td"); + cell.innerHTML = field; + row.appendChild(cell); + }); + table_body.appendChild(row); + }); +} + +/* + * Display a preview of the data, relying on the user's selection. + */ +function display_preview(event) { + var data_preview_table = document.getElementById("tbl:samples-preview"); + remove_rows(data_preview_table); + + var separator = document.getElementById("select:separator").value; + if(separator === "other") { + separator = document.getElementById("txt:separator").value; + } + if(separator == "") { + display_error_row(data_preview_table, "Please provide a separator."); + return false; + } + + var delimiter = document.getElementById("txt:delimiter").value; + + var firstlineheading = document.getElementById("chk:heading").checked; + + var fileelement = document.getElementById("file:samples"); + var preview_data = JSON.parse( + fileelement.getAttribute("data-preview-content") || "[]"); + if(preview_data.length == 0) { + display_error_row( + data_preview_table, + "No file data to preview. Check that file is provided."); + } + + render_preview(data_preview_table, process_preview_data( + preview_data.slice(0 + (firstlineheading ? 1 : 0)), + separator, + delimiter)); +} + +document.getElementById("chk:heading").addEventListener( + "change", display_preview); +document.getElementById("select:separator").addEventListener( + "change", display_preview); +document.getElementById("txt:separator").addEventListener( + "keyup", display_preview); +document.getElementById("txt:delimiter").addEventListener( + "keyup", display_preview); +document.getElementById("file:samples").addEventListener( + "change", (event) => { + read_first_n_lines(event, + document.getElementById("file:samples"), + 30, + document.getElementById("chk:heading").checked); + }); diff --git a/uploader/static/js/utils.js b/uploader/static/js/utils.js new file mode 100644 index 0000000..045dd47 --- /dev/null +++ b/uploader/static/js/utils.js @@ -0,0 +1,10 @@ +function remove_children(element) { + Array.from(element.children).forEach(child => { + element.removeChild(child); + }); +} + +function trigger_change_event(element) { + evt = new Event("change"); + element.dispatchEvent(evt); +} diff --git a/uploader/templates/base.html b/uploader/templates/base.html new file mode 100644 index 0000000..5eca445 --- /dev/null +++ b/uploader/templates/base.html @@ -0,0 +1,139 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + + <meta charset="UTF-8" /> + <meta application-name="GeneNetwork Quality-Control Application" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + {%block extrameta%}{%endblock%} + + <title>Data Upload and Quality Control: {%block title%}{%endblock%}</title> + + <link rel="stylesheet" type="text/css" + href="{{url_for('base.bootstrap', + filename='css/bootstrap.min.css')}}" /> + <link rel="stylesheet" type="text/css" + href="{{url_for('base.bootstrap', + filename='css/bootstrap-theme.min.css')}}" /> + <link rel="stylesheet" type="text/css" href="/static/css/styles.css" /> + <link rel="stylesheet" + href="{{url_for('base.datatables', filename='css/jquery.dataTables.css')}}" /> + + {%block css%}{%endblock%} + + </head> + + <body> + <header id="header"> + <span id="header-text">GeneNetwork</span> + <nav id="header-nav"> + <ul class="nav justify-content-end"> + <li> + {%if user_logged_in()%} + <a href="{{url_for('oauth2.logout')}}" + title="Log out of the system"> + <span class="glyphicon glyphicon-user"></span> + Sign Out</a> + {%else%} + <a href="{{authserver_authorise_uri()}}" + title="Log in to the system">Sign In</a> + {%endif%} + </li> + </ul> + </nav> + </header> + + <aside id="nav-sidebar"> + <ul class="nav flex-column"> + <li {%if activemenu=="home"%}class="activemenu"{%endif%}> + <a href="/" >Home</a></li> + <li {%if activemenu=="species"%}class="activemenu"{%endif%}> + <a href="{{url_for('species.list_species')}}" + title="View and manage species information.">Species</a></li> + <li {%if activemenu=="platforms"%}class="activemenu"{%endif%}> + <a href="{{url_for('species.platforms.index')}}" + title="View and manage species platforms.">Sequencing Platforms</a></li> + <li {%if activemenu=="populations"%}class="activemenu"{%endif%}> + <a href="{{url_for('species.populations.index')}}" + title="View and manage species populations.">Populations</a></li> + <li {%if activemenu=="samples"%}class="activemenu"{%endif%}> + <a href="{{url_for('species.populations.samples.index')}}" + title="Upload population samples.">Samples</a></li> + <li {%if activemenu=="genotypes"%}class="activemenu"{%endif%}> + <a href="{{url_for('species.populations.genotypes.index')}}" + title="Upload Genotype data.">Genotype Data</a></li> + <!-- + TODO: Maybe include menus here for managing studies and dataset or + maybe have the studies/datasets managed under their respective + sections, e.g. "Publish*" studies/datasets under the "Phenotypes" + section, "ProbeSet*" studies/datasets under the "Expression Data" + sections, etc. + --> + <li {%if activemenu=="phenotypes"%}class="activemenu"{%endif%}> + <a href="{{url_for('species.populations.phenotypes.index')}}" + title="Upload phenotype data.">Phenotype Data</a></li> + <!-- + <li {%if activemenu=="expression-data"%}class="activemenu"{%endif%}> + <a href="{{url_for('species.populations.expression-data.index')}}" + title="Upload expression data." + class="not-implemented">Expression Data</a></li> + <li {%if activemenu=="individuals"%}class="activemenu"{%endif%}> + <a href="#" + class="not-implemented" + title="Upload individual data.">Individual Data</a></li> + <li {%if activemenu=="rna-seq"%}class="activemenu"{%endif%}> + <a href="#" + class="not-implemented" + title="Upload RNA-Seq data.">RNA-Seq Data</a></li> + <li {%if activemenu=="async-jobs"%}class="activemenu"{%endif%}> + <a href="#" + class="not-implemented" + title="View and manage the backgroud jobs you have running"> + Background Jobs</a></li> + --> + </ul> + </aside> + + <main id="main" class="main"> + + <div id="pagetitle" class="pagetitle"> + <span class="title">Data Upload and Quality Control: {%block pagetitle%}{%endblock%}</span> + <!-- + <nav> + <ol class="breadcrumb"> + <li {%if activelink is not defined or activelink=="home"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('base.index')}}">Home</a> + </li> + {%block lvl1_breadcrumbs%}{%endblock%} + </ol> + </nav> + --> + </div> + + <div id="all-content"> + <div id="main-content"> + {%block contents%}{%endblock%} + </div> + <div id="sidebar-content"> + {%block sidebarcontents%}{%endblock%} + </div> + </div> + </main> + + + <script src="{{url_for('base.jquery', + filename='jquery.min.js')}}"></script> + <script src="{{url_for('base.bootstrap', + filename='js/bootstrap.min.js')}}"></script> + <script type="text/javascript" src="/static/js/misc.js"></script> + <script type="text/javascript" + src="{{url_for('base.datatables', + filename='js/jquery.dataTables.js')}}"></script> + {%block javascript%}{%endblock%} + </body> +</html> diff --git a/uploader/templates/cli-output.html b/uploader/templates/cli-output.html new file mode 100644 index 0000000..64b1a9a --- /dev/null +++ b/uploader/templates/cli-output.html @@ -0,0 +1,8 @@ +{%macro cli_output(job, stream)%} + +<h4 class="subheading">{{stream | upper}} Output</h4> +<div class="cli-output" style="max-height: 10em; overflow: auto;"> + <pre>{{job.get(stream, "")}}</pre> +</div> + +{%endmacro%} diff --git a/uploader/templates/continue_from_create_dataset.html b/uploader/templates/continue_from_create_dataset.html new file mode 100644 index 0000000..03bb49c --- /dev/null +++ b/uploader/templates/continue_from_create_dataset.html @@ -0,0 +1,52 @@ +{%extends "base.html"%} +{%from "dbupdate_hidden_fields.html" import hidden_fields%} + +{%block title%}Create Study{%endblock%} + +{%block css%} +<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> +{%endblock%} + +{%block contents%} +<h2 class="heading">{{filename}}: create study</h2> + +{%with messages = get_flashed_messages(with_categories=true)%} +{%if messages:%} +<ul> + {%for category, message in messages:%} + <li class="{{category}}">{{message}}</li> + {%endfor%} +</ul> +{%endif%} +{%endwith%} + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.final_confirmation')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col1"> + <legend>continue with new dataset</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, datasetid=datasetid, totallines=totallines)}} + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +<div class="row"> + <p class="two-col-sep-separator">OR</p> +</div> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.select_dataset')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col2"> + <legend>Select from existing dataset</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, datasetid=datasetid, totallines=totallines)}} + + <button type="submit" class="btn btn-primary">go back</button> + </form> +</div> +{%endblock%} diff --git a/uploader/templates/continue_from_create_study.html b/uploader/templates/continue_from_create_study.html new file mode 100644 index 0000000..34e6e5e --- /dev/null +++ b/uploader/templates/continue_from_create_study.html @@ -0,0 +1,52 @@ +{%extends "base.html"%} +{%from "dbupdate_hidden_fields.html" import hidden_fields%} + +{%block title%}Create Study{%endblock%} + +{%block css%} +<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> +{%endblock%} + +{%block contents%} +<h2 class="heading">{{filename}}: create study</h2> + +{%with messages = get_flashed_messages(with_categories=true)%} +{%if messages:%} +<ul> + {%for category, message in messages:%} + <li class="{{category}}">{{message}}</li> + {%endfor%} +</ul> +{%endif%} +{%endwith%} + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.select_dataset')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col1"> + <legend>continue with new study</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, totallines=totallines)}} + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +<div class="row"> + <p class="two-col-sep-separator">OR</p> +</div> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.select_study')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col2"> + <legend>Select from existing study</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, totallines=totallines)}} + + <button type="submit" class="btn btn-primary">go back</button> + </form> +</div> +{%endblock%} diff --git a/uploader/templates/dbupdate_error.html b/uploader/templates/dbupdate_error.html new file mode 100644 index 0000000..e1359d2 --- /dev/null +++ b/uploader/templates/dbupdate_error.html @@ -0,0 +1,12 @@ +{%extends "base.html"%} + +{%block title%}DB Update Error{%endblock%} + +{%block contents%} +<h1 class="heading">database update error</h2> + +<p class="alert-danger"> + <strong>Database Update Error</strong>: {{error_message}} +</p> + +{%endblock%} diff --git a/uploader/templates/dbupdate_hidden_fields.html b/uploader/templates/dbupdate_hidden_fields.html new file mode 100644 index 0000000..ccbc299 --- /dev/null +++ b/uploader/templates/dbupdate_hidden_fields.html @@ -0,0 +1,29 @@ +{%macro hidden_fields(filename, filetype):%} + +<!-- {{kwargs}}: mostly for accessing the kwargs in macro --> + +<input type="hidden" name="filename" value="{{filename}}" /> +<input type="hidden" name="filetype" value="{{filetype}}" /> +{%if kwargs.get("totallines")%} +<input type="hidden" name="totallines" value="{{kwargs['totallines']}}" /> +{%endif%} +{%if kwargs.get("species"):%} +<input type="hidden" name="species" value="{{kwargs['species']}}" /> +{%endif%} +{%if kwargs.get("genechipid"):%} +<input type="hidden" name="genechipid" value="{{kwargs['genechipid']}}" /> +{%endif%} +{%if kwargs.get("inbredsetid"):%} +<input type="hidden" name="inbredsetid" value="{{kwargs['inbredsetid']}}" /> +{%endif%} +{%if kwargs.get("tissueid"):%} +<input type="hidden" name="tissueid" value="{{kwargs['tissueid']}}" /> +{%endif%} +{%if kwargs.get("studyid"):%} +<input type="hidden" name="studyid" value="{{kwargs['studyid']}}" /> +{%endif%} +{%if kwargs.get("datasetid"):%} +<input type="hidden" name="datasetid" value="{{kwargs['datasetid']}}" /> +{%endif%} + +{%endmacro%} diff --git a/uploader/templates/errors_display.html b/uploader/templates/errors_display.html new file mode 100644 index 0000000..715cfcf --- /dev/null +++ b/uploader/templates/errors_display.html @@ -0,0 +1,47 @@ +{%macro errors_display(errors, no_error_msg, error_message, complete)%} + +{%if errors | length == 0 %} +<span {%if complete%}class="alert-success"{%endif%}>{{no_error_msg}}</span> +{%else %} +<p class="alert-danger">{{error_message}}</p> + +<table class="table reports-table"> + <thead> + <tr> + <th>line number</th> + <th>column(s)</th> + <th>error</th> + <th>error message</th> + </tr> + </thead> + + <tbody> + {%for error in errors%} + <tr> + <td>{{error["line"]}}</td> + <td> + {%if isinvalidvalue(error):%} + {{error.column}} + {%elif isduplicateheading(error): %} + {{error.columns}} + {%else: %} + - + {%endif %} + </td> + <td> + {%if isinvalidvalue(error):%} + Invalid Value + {%elif isduplicateheading(error): %} + Duplicate Header + {%else%} + Inconsistent Columns + {%endif %} + </td> + <td>{{error["message"]}}</td> + </tr> + {%endfor%} + </tbody> +</table> +{%endif%} + +{%endmacro%} diff --git a/uploader/templates/expression-data/base.html b/uploader/templates/expression-data/base.html new file mode 100644 index 0000000..d63fd7e --- /dev/null +++ b/uploader/templates/expression-data/base.html @@ -0,0 +1,13 @@ +{%extends "populations/base.html"%} + +{%block lvl3_breadcrumbs%} +<li {%if activelink=="expression-data"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.expression-data.index')}}"> + Expression Data</a> +</li> +{%block lvl4_breadcrumbs%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/expression-data/data-review.html b/uploader/templates/expression-data/data-review.html new file mode 100644 index 0000000..c985b03 --- /dev/null +++ b/uploader/templates/expression-data/data-review.html @@ -0,0 +1,85 @@ +{%extends "base.html"%} + +{%block title%}Data Review{%endblock%} + +{%block contents%} +<h1 class="heading">data review</h1> + +<div class="row"> + <h2 id="data-concerns">Data Concerns</h2> + <p>The following are some of the requirements that the data in your file + <strong>MUST</strong> fulfil before it is considered valid for this system: + </p> + + <ol> + <li>File headings + <ul> + <li>The first row in the file should contains the headings. The number of + headings in this first row determines the number of columns expected for + all other lines in the file.</li> + <li>Each heading value in the first row MUST appear in the first row + <strong>ONE AND ONLY ONE</strong> time</li> + <li>The sample/cases (previously 'strains') headers in your first row will be + against those in the <a href="https://genenetwork.org" + title="Link to the GeneNetwork service"> + GeneNetwork</a> database.<br /> + <small class="text-muted"> + If you encounter an error saying your sample(s)/case(s) do not exist + in the GeneNetwork database, then you will have to use the + <a href="{{url_for('species.populations.samples.index')}}" + title="Upload samples/cases feature">Upload Samples/Cases</a> + option on this system to upload them. + </small> + </ul> + </li> + + <li>Data + <ol> + <li><strong>NONE</strong> of the data cells/fields is allowed to be empty. + All fields/cells <strong>MUST</strong> contain a value.</li> + <li>The first column of the data rows will be considered a textual field, + holding the "identifier" for that row<li> + <li>Except for the first column/field for each data row, + <strong>NONE</strong> of the data columns/cells/fields should contain + spurious characters like `eeeee`, `5.555iloveguix`, etc...<br /> + All of them should be decimal values</li> + <li>decimal numbers must conform to the following criteria: + <ul> + <li>when checking an average file decimal numbers must have exactly three + decimal places to the right of the decimal point.</li> + <li>when checking a standard error file decimal numbers must have six or + greater decimal places to the right of the decimal point.</li> + <li>there must be a number to the left side of the decimal place + (e.g. 0.55555 is allowed but .55555 is not).</li> + </ul> + </li> + </ol> + </li> + </ol> +</div> + + +<div class="row"> + <h2 id="file-types">Supported File Types</h2> + We support the following file types: + + <ul> + <li>Tab-Separated value files (.tsv) + <ul> + <li>The <strong>TAB</strong> character is used to separate the fields of each + column</li> + <li>The values of each field <strong>ARE NOT</strong> quoted.</li> + <li>Here is an + <a href="https://gitlab.com/fredmanglis/gnqc_py/-/blob/main/tests/test_data/no_data_errors.tsv" + target="_blank">example file</a> with a single data row.</li> + </ul> + </li> + <li>.txt files: Content has the same format as .tsv file above</li> + <li>.zip files: each zip file should contain + <strong>ONE AND ONLY ONE</strong> file of the .tsv or .txt type above. + <br />Any zip file with more than one file is invalid, and so is an empty + zip file.</li> + </ul> + +</div> +{%endblock%} diff --git a/uploader/templates/expression-data/index.html b/uploader/templates/expression-data/index.html new file mode 100644 index 0000000..9ba3582 --- /dev/null +++ b/uploader/templates/expression-data/index.html @@ -0,0 +1,33 @@ +{%extends "expression-data/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-select-species.html" import select_species_form%} + +{%block title%}Expression Data{%endblock%} + +{%block pagetitle%}Expression Data{%endblock%} + +{%block breadcrumb%} +<li class="breadcrumb-item"> + <a href="{{url_for('base.index')}}">Home</a> +</li> +<li class="breadcrumb-item active"> + <a href="{{url_for('species.populations.expression-data.index')}}" + title="Upload expression data."> + Expression Data</a> +</li> +{%endblock%} + +{%block contents%} +<div class="row"> + <h2 class="heading">Expression Data</h2> + {{flash_all_messages()}} + + <p>This section allows you to enter the expression data for your experiment. + You will need to select the species that your data concerns below.</p> +</div> + +<div class="row"> + {{select_species_form(url_for("species.populations.expression-data.index"), + species)}} +</div> +{%endblock%} diff --git a/uploader/templates/expression-data/job-progress.html b/uploader/templates/expression-data/job-progress.html new file mode 100644 index 0000000..ef264e1 --- /dev/null +++ b/uploader/templates/expression-data/job-progress.html @@ -0,0 +1,47 @@ +{%extends "base.html"%} +{%from "errors_display.html" import errors_display%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block extrameta%} +<meta http-equiv="refresh" content="5"> +{%endblock%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">{{job_name}}</h2> + +<div class="row"> + <form action="{{url_for('species.populations.expression-data.abort', + species_id=species.SpeciesId, + population_id=population.Id)}}" method="POST"> + <legend class="heading">Status</legend> + <div class="form-group"> + <label for="job_status" class="form-label">status:</label> + <span class="form-text">{{job_status}}: {{message}}</span><br /> + </div> + + <div class="form-group"> + <label for="job_{{job_id}}" class="form-label">parsing: </label> + <progress id="job_{{job_id}}" + value="{{progress/100}}" + class="form-control"> + {{progress}}</progress> + <span class="form-text text-muted">{{"%.2f" | format(progress)}}%</span> + </div> + + <input type="hidden" name="job_id" value="{{job_id}}" /> + + <button type="submit" class="btn btn-danger">Abort</button> + </form> +</div> + +<div class="row"> + {{errors_display(errors, "No errors found so far", "We have found the following errors so far", False)}} +</div> + +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/expression-data/no-such-job.html b/uploader/templates/expression-data/no-such-job.html new file mode 100644 index 0000000..d22c429 --- /dev/null +++ b/uploader/templates/expression-data/no-such-job.html @@ -0,0 +1,15 @@ +{%extends "base.html"%} + +{%block extrameta%} +<meta http-equiv="refresh" + content="5;url={{url_for('species.populations.expression-data.index.upload_file')}}"> +{%endblock%} + +{%block title%}No Such Job{%endblock%} + +{%block contents%} +<h1 class="heading">No Such Job: {{job_id}}</h2> + +<p>No job, with the id '<em>{{job_id}}</em>' was found!</p> + +{%endblock%} diff --git a/uploader/templates/expression-data/parse-failure.html b/uploader/templates/expression-data/parse-failure.html new file mode 100644 index 0000000..31f6be8 --- /dev/null +++ b/uploader/templates/expression-data/parse-failure.html @@ -0,0 +1,26 @@ +{%extends "base.html"%} + +{%block title%}Worker Failure{%endblock%} + +{%block contents%} +<h1 class="heading">Worker Failure</h1> + +<p> + There was an error while parsing your file. +</p> + +<p> + Please notify the developers of this issue when you encounter it, + providing the information below. +</p> + +<h4>Debugging Information</h4> + +<ul> + <li><strong>job id</strong>: {{job["job_id"]}}</li> + <li><strong>filename</strong>: {{job["filename"]}}</li> + <li><strong>line number</strong>: {{job["line_number"]}}</li> + <li><strong>Progress</strong>: {{job["percent"]}} %</li> +</ul> + +{%endblock%} diff --git a/uploader/templates/expression-data/parse-results.html b/uploader/templates/expression-data/parse-results.html new file mode 100644 index 0000000..03a23e2 --- /dev/null +++ b/uploader/templates/expression-data/parse-results.html @@ -0,0 +1,39 @@ +{%extends "base.html"%} +{%from "errors_display.html" import errors_display%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Parse Results{%endblock%} + +{%block contents%} + +<div class="row"> + <h2 class="heading">{{job_name}}: parse results</h2> + + {%if user_aborted%} + <span class="alert-warning">Job aborted by the user</span> + {%endif%} + + {{errors_display(errors, "No errors found in the file", "We found the following errors", True)}} + + {%if errors | length == 0 and not user_aborted %} + <form method="post" action="{{url_for('dbinsert.select_platform')}}"> + <input type="hidden" name="job_id" value="{{job_id}}" /> + <input type="submit" value="update database" class="btn btn-primary" /> + </form> + {%endif%} + + {%if errors | length > 0 or user_aborted %} + <br /> + <a href="{{url_for('species.populations.expression-data.upload_file', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="Back to index page." + class="btn btn-primary">Go back</a> + + {%endif%} +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/expression-data/select-file.html b/uploader/templates/expression-data/select-file.html new file mode 100644 index 0000000..4ca461e --- /dev/null +++ b/uploader/templates/expression-data/select-file.html @@ -0,0 +1,115 @@ +{%extends "expression-data/base.html"%} +{%from "flash_messages.html" import flash_messages%} +{%from "upload_progress_indicator.html" import upload_progress_indicator%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Expression Data — Upload Data{%endblock%} + +{%block pagetitle%}Expression Data — Upload Data{%endblock%} + +{%block contents%} +{{upload_progress_indicator()}} + +<div class="row"> + <h2 class="heading">Upload Expression Data</h2> + + <p>This feature enables you to upload expression data. It expects the data to + be in <strong>tab-separated values (TSV)</strong> files. The data should be + a simple matrix of <em>phenotype × sample</em>, i.e. The first column is a + list of the <em>phenotypes</em> and the first row is a list of + <em>samples/cases</em>.</p> + + <p>If you haven't done so please go to this page to learn the requirements for + file formats and helpful suggestions to enter your data in a fast and easy + way.</p> + + <ol> + <li><strong>PLEASE REVIEW YOUR DATA.</strong>Make sure your data complies + with our system requirements. ( + <a href="{{url_for('species.populations.expression-data.data_review')}}#data-concerns" + title="Details for the data expectations.">Help</a> + )</li> + <li><strong>UPLOAD YOUR DATA FOR DATA VERIFICATION.</strong> We accept + <strong>.csv</strong>, <strong>.txt</strong> and <strong>.zip</strong> + files (<a href="{{url_for('species.populations.expression-data.data_review')}}#file-types" + title="Details for the data expectations.">Help</a>)</li> + </ol> +</div> + +<div class="row"> + <form action="{{url_for( + 'species.populations.expression-data.upload_file', + species_id=species.SpeciesId, + population_id=population.Id)}}" + method="POST" + enctype="multipart/form-data" + id="frm-upload-expression-data"> + {{flash_messages("error-expr-data")}} + + <div class="form-group"> + <legend class="heading">File Type</legend> + + <div class="radio"> + <label for="filetype_average" class="form-check-label"> + <input type="radio" name="filetype" value="average" id="filetype_average" + required="required" class="form-check-input" /> + Average</label> + <p class="form-text text-muted"> + <small>The averages data …</small></p> + </div> + + <div class="radio"> + <label for="filetype_standard_error" class="form-check-label"> + <input type="radio" name="filetype" value="standard-error" + id="filetype_standard_error" required="required" + class="form-check-input" /> + Standard Error + </label> + <p class="form-text text-muted"> + <small>The standard errors computed from the averages …</small></p> + </div> + </div> + + <div class="form-group"> + <span id="no-file-error" class="alert-danger" style="display: none;"> + No file selected + </span> + <label for="file_upload" class="form-label">Select File</label> + <input type="file" name="qc_text_file" id="file_upload" + accept="text/plain, text/tab-separated-values, application/zip" + class="form-control"/> + <p class="form-text text-muted"> + <small>Select the file to upload.</small></p> + </div> + + <button type="submit" + class="btn btn-primary" + data-toggle="modal" + data-target="#upload-progress-indicator">upload file</button> + </form> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/upload_progress.js"></script> +<script type="text/javascript"> + function setup_formdata(form) { + var formdata = new FormData(); + formdata.append( + "qc_text_file", + form.querySelector("input[type='file']").files[0]); + formdata.append( + "filetype", + selected_filetype( + Array.from(form.querySelectorAll("input[type='radio']")))); + return formdata; + } + + setup_upload_handlers( + "frm-upload-expression-data", make_data_uploader(setup_formdata)); +</script> +{%endblock%} diff --git a/uploader/templates/expression-data/select-population.html b/uploader/templates/expression-data/select-population.html new file mode 100644 index 0000000..8555e27 --- /dev/null +++ b/uploader/templates/expression-data/select-population.html @@ -0,0 +1,29 @@ +{%extends "expression-data/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-display-species-card.html" import display_species_card%} +{%from "populations/macro-select-population.html" import select_population_form%} + +{%block title%}Expression Data{%endblock%} + +{%block pagetitle%}Expression Data{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <p>You have selected the species. Now you need to select the population that + the expression data belongs to.</p> +</div> + +<div class="row"> + {{select_population_form(url_for( + "species.populations.expression-data.select_population", + species_id=species.SpeciesId), + populations)}} +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} diff --git a/uploader/templates/final_confirmation.html b/uploader/templates/final_confirmation.html new file mode 100644 index 0000000..0727fc8 --- /dev/null +++ b/uploader/templates/final_confirmation.html @@ -0,0 +1,47 @@ +{%extends "base.html"%} +{%from "dbupdate_hidden_fields.html" import hidden_fields%} + +{%block title%}Confirmation{%endblock%} + +{%macro display_item(item_name, item_data):%} +<li> + <strong>{{item_name}}</strong> + {%if item_data%} + <ul> + {%for term,value in item_data.items():%} + <li><strong>{{term}}:</strong> {{value}}</li> + {%endfor%} + </ul> + {%endif%} +</li> +{%endmacro%} + +{%block contents%} +<h2 class="heading">Final Confirmation</h2> + +<div class="two-col-sep-col1"> + <p><strong>Selected Data</strong></p> + <ul> + <li><strong>File</strong> + <ul> + <li><strong>Filename</strong>: {{filename}}</li> + <li><strong>File Type</strong>: {{filetype}}</li> + </ul> + </li> + {{display_item("Species", the_species)}} + {{display_item("Platform", platform)}} + {{display_item("Study", study)}} + {{display_item("Dataset", dataset)}} + </ul> +</div> + +<form method="POST" action="{{url_for('dbinsert.insert_data')}}"> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid,datasetid=datasetid, totallines=totallines)}} + <fieldset> + <input type="submit" class="btn btn-primary" value="confirm" /> + </fieldset> +</form> +</div> +{%endblock%} diff --git a/uploader/templates/flash_messages.html b/uploader/templates/flash_messages.html new file mode 100644 index 0000000..b7af178 --- /dev/null +++ b/uploader/templates/flash_messages.html @@ -0,0 +1,25 @@ +{%macro flash_all_messages()%} +{%with messages = get_flashed_messages(with_categories=true)%} +{%if messages:%} +<ul> + {%for category, message in messages:%} + <li class="{{category}}">{{message}}</li> + {%endfor%} +</ul> +{%endif%} +{%endwith%} +{%endmacro%} + +{%macro flash_messages(filter_class)%} +{%with messages = get_flashed_messages(with_categories=true)%} +{%if messages:%} +<ul> + {%for category, message in messages:%} + {%if filter_class in category%} + <li class="{{category}}">{{message}}</li> + {%endif%} + {%endfor%} +</ul> +{%endif%} +{%endwith%} +{%endmacro%} diff --git a/uploader/templates/genotypes/base.html b/uploader/templates/genotypes/base.html new file mode 100644 index 0000000..7d61312 --- /dev/null +++ b/uploader/templates/genotypes/base.html @@ -0,0 +1,23 @@ +{%extends "populations/base.html"%} + +{%block lvl3_breadcrumbs%} +<li {%if activelink=="genotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + {%if population is mapping%} + <a href="{{url_for('species.populations.genotypes.list_genotypes', + species_id=species.SpeciesId, + population_id=population.Id)}}"> + {%if dataset is defined and dataset is mapping%} + {{dataset.Name}} + {%else%} + Genotypes + {%endif%}</a> + {%else%} + <a href="{{url_for('species.populations.genotypes.index')}}">Genotypes</a> + {%endif%} +</li> +{%block lvl4_breadcrumbs%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/genotypes/create-dataset.html b/uploader/templates/genotypes/create-dataset.html new file mode 100644 index 0000000..10331c1 --- /dev/null +++ b/uploader/templates/genotypes/create-dataset.html @@ -0,0 +1,82 @@ +{%extends "genotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Genotypes — Create Dataset{%endblock%} + +{%block pagetitle%}Genotypes — Create Dataset{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="create-dataset"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.genotypes.create_dataset', + species_id=species.SpeciesId, + population_id=population.Id)}}">Create Dataset</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <form id="frm-geno-create-dataset" + method="POST" + action="{{url_for('species.populations.genotypes.create_dataset', + species_id=species.SpeciesId, + population_id=population.Id)}}"> + <legend>Create a new Genotype Dataset</legend> + + <div class="form-group"> + <label for="txt-geno-dataset-name" class="form-label">Name</label> + <input type="text" + id="txt-geno-dataset-name" + name="geno-dataset-name" + required="required" + class="form-control" /> + <small class="form-text text-muted"> + <p>This is a short representative, but constrained name for the genotype + dataset.<br /> + The field will only accept letters ('A-Za-z'), numbers (0-9), hyphens + and underscores. Any other character will cause the name to be + rejected.</p></small> + </div> + + <div class="form-group"> + <label for="txt-geno-dataset-fullname" class="form-label">Full Name</label> + <input type="text" + id="txt-geno-dataset-fullname" + name="geno-dataset-fullname" + required="required" + class="form-control" /> + <small class="form-text text-muted"> + <p>This is a longer, more descriptive name for your dataset.</p></small> + </div> + + <div class="form-group"> + <label for="txt-geno-dataset-shortname" + class="form-label">Short Name</label> + <input type="text" + id="txt-geno-dataset-shortname" + name="geno-dataset-shortname" + class="form-control" /> + <small class="form-text text-muted"> + <p>A short name for your dataset. If you leave this field blank, the + short name will be set to the same value as the + "<strong>Name</strong>" field above.</p></small> + </div> + + <div class="form-group"> + <input type="submit" + class="btn btn-primary" + value="create dataset" /> + </div> + </form> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/genotypes/index.html b/uploader/templates/genotypes/index.html new file mode 100644 index 0000000..b50ebc5 --- /dev/null +++ b/uploader/templates/genotypes/index.html @@ -0,0 +1,32 @@ +{%extends "genotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-select-species.html" import select_species_form%} + +{%block title%}Genotypes{%endblock%} + +{%block pagetitle%}Genotypes{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <p> + This section allows you to upload genotype information for your experiments, + in the case that you have not previously done so. + </p> + <p> + We'll need to link the genotypes to the species and population, so do please + go ahead and select those in the next two steps. + </p> +</div> + +<div class="row"> + {{select_species_form(url_for("species.populations.genotypes.index"), + species)}} +</div> +{%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/genotypes/list-genotypes.html b/uploader/templates/genotypes/list-genotypes.html new file mode 100644 index 0000000..0f074fd --- /dev/null +++ b/uploader/templates/genotypes/list-genotypes.html @@ -0,0 +1,149 @@ +{%extends "genotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Genotypes{%endblock%} + +{%block pagetitle%}Genotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="list-genotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.genotypes.list_genotypes', + species_id=species.SpeciesId, + population_id=population.Id)}}">List genotypes</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2>Genetic Markers</h2> + <p>There are a total of {{total_markers}} currently registered genetic markers + for the "{{species.FullName}}" species. You can click + <a href="{{url_for('species.populations.genotypes.list_markers', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="View genetic markers for species '{{species.FullName}}"> + this link to view the genetic markers + </a>. + </p> +</div> + +<div class="row"> + <h2>Genotype Encoding</h2> + <p> + The genotype encoding used for the "{{population.FullName}}" population from + the "{{species.FullName}}" species is as shown in the table below. + </p> + <table class="table"> + + <thead> + <tr> + <th>Allele Type</th> + <th>Allele Symbol</th> + <th>Allele Value</th> + </tr> + </thead> + + <tbody> + {%for row in genocode%} + <tr> + <td>{{row.AlleleType}}</td> + <td>{{row.AlleleSymbol}}</td> + <td>{{row.DatabaseValue if row.DatabaseValue is not none else "NULL"}}</td> + </tr> + {%else%} + <tr> + <td colspan="7" class="text-info"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + There is no explicit genotype encoding defined for this population. + </td> + </tr> + {%endfor%} + </tbody> + </table> + + {%if genocode | length < 1%} + <a href="#add-genotype-encoding" + title="Add a genotype encoding system for this population" + class="btn btn-primary not-implemented"> + add genotype encoding + </a> + {%endif%} +</div> + +<div class="row text-danger"> + <h3>Some Important Concepts to Consider/Remember</h3> + <ul> + <li>Reference vs. Non-reference alleles</li> + <li>In <em>GenoCode</em> table, items are ordered by <strong>InbredSet</strong></li> + </ul> + <h3>Possible references</h3> + <ul> + <li>https://mr-dictionary.mrcieu.ac.uk/term/genotype/</li> + <li>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7363099/</li> + </ul> +</div> + +<div class="row"> + <h2>Genotype Datasets</h2> + + <p>The genotype data is organised under various genotype datasets. You can + click on the link for the relevant dataset to view a little more information + about it.</p> + + {%if dataset is not none%} + <table class="table"> + <thead> + <tr> + <th>Name</th> + <th>Full Name</th> + </tr> + </thead> + + <tbody> + <tr> + <td>{{dataset.Name}}</td> + <td><a href="{{url_for('species.populations.genotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + title="View details regarding and manage dataset '{{dataset.FullName}}'"> + {{dataset.FullName}}</a></td> + </tr> + </tbody> + </table> + {%else%} + <p class="text-warning"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + There is no genotype dataset defined for this population. + </p> + <p> + <a href="{{url_for('species.populations.genotypes.create_dataset', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="Create a new genotype dataset for the '{{population.FullName}}' population for the '{{species.FullName}}' species." + class="btn btn-primary"> + create new genotype dataset</a></p> + {%endif%} +</div> +<div class="row text-warning"> + <p> + <span class="glyphicon glyphicon-exclamation-sign"></span> + <strong>NOTE</strong>: Currently the GN2 (and related) system(s) expect a + single genotype dataset. If there is more than one, the system apparently + fails in unpredictable ways. + </p> + <p>Fix this to allow multiple datasets, each with a different assembly from + all the rest.</p> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/genotypes/list-markers.html b/uploader/templates/genotypes/list-markers.html new file mode 100644 index 0000000..a705ae3 --- /dev/null +++ b/uploader/templates/genotypes/list-markers.html @@ -0,0 +1,105 @@ +{%extends "genotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-display-species-card.html" import display_species_card%} + +{%block title%}Genotypes: List Markers{%endblock%} + +{%block pagetitle%}Genotypes: List Markers{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="list-markers"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.genotypes.list_markers', + species_id=species.SpeciesId, + population_id=population.Id)}}">List markers</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +{%if markers | length > 0%} +<div class="row"> + <p> + There are a total of {{total_markers}} genotype markers for this species. + </p> + <div class="row"> + <div class="col-md-2" style="text-align: start;"> + {%if start_from > 0%} + <a href="{{url_for('species.populations.genotypes.list_markers', + species_id=species.SpeciesId, + population_id=population.Id, + start_from=start_from-count, + count=count)}}"> + <span class="glyphicon glyphicon-backward"></span> + Previous + </a> + {%endif%} + </div> + <div class="col-md-8" style="text-align: center;"> + Displaying markers {{start_from+1}} to {{start_from+count if start_from+count < total_markers else total_markers}} of + {{total_markers}} + </div> + <div class="col-md-2" style="text-align: end;"> + {%if start_from + count < total_markers%} + <a href="{{url_for('species.populations.genotypes.list_markers', + species_id=species.SpeciesId, + population_id=population.Id, + start_from=start_from+count, + count=count)}}"> + Next + <span class="glyphicon glyphicon-forward"></span> + </a> + {%endif%} + </div> + </div> + <table class="table"> + <thead> + <tr> + <th title="">#</th> + <th title="">Marker Name</th> + <th title="Chromosome">Chr</th> + <th title="Physical location of the marker in megabasepairs"> + Location (Mb)</th> + <th title="">Source</th> + <th title="">Source2</th> + </thead> + + <tbody> + {%for marker in markers%} + <tr> + <td>{{marker.sequence_number}}</td> + <td>{{marker.Marker_Name}}</td> + <td>{{marker.Chr}}</td> + <td>{{marker.Mb}}</td> + <td>{{marker.Source}}</td> + <td>{{marker.Source2}}</td> + </tr> + {%endfor%} + </tbody> + </table> +</div> +{%else%} +<div class="row"> + <p class="text-warning"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + This species does not currently have any genetic markers uploaded, therefore, + there is nothing to display here. + </p> + <p> + <a href="#add-genetic-markers-for-species-{{species.SpeciesId}}" + title="Add genetic markers for this species" + class="btn btn-primary"> + add genetic markers + </a> + </p> +</div> +{%endif%} +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} diff --git a/uploader/templates/genotypes/select-population.html b/uploader/templates/genotypes/select-population.html new file mode 100644 index 0000000..acdd063 --- /dev/null +++ b/uploader/templates/genotypes/select-population.html @@ -0,0 +1,25 @@ +{%extends "genotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-display-species-card.html" import display_species_card%} +{%from "populations/macro-select-population.html" import select_population_form%} + +{%block title%}Genotypes{%endblock%} + +{%block pagetitle%}Genotypes{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + {{select_population_form(url_for("species.populations.genotypes.select_population", species_id=species.SpeciesId), species, populations)}} +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> +{%endblock%} diff --git a/uploader/templates/genotypes/view-dataset.html b/uploader/templates/genotypes/view-dataset.html new file mode 100644 index 0000000..e7ceb36 --- /dev/null +++ b/uploader/templates/genotypes/view-dataset.html @@ -0,0 +1,61 @@ +{%extends "genotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Genotypes: View Dataset{%endblock%} + +{%block pagetitle%}Genotypes: View Dataset{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="view-dataset"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.genotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">view dataset</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2>Genotype Dataset Details</h2> + <table class="table"> + <thead> + <tr> + <th>Name</th> + <th>Full Name</th> + </tr> + </thead> + + <tbody> + <tr> + <td>{{dataset.Name}}</td> + <td>{{dataset.FullName}}</td> + </tr> + </tbody> + </table> +</div> + +<div class="row text-warning"> + <h2>Assembly Details</h2> + + <p>Maybe include the assembly details here if found to be necessary.</p> +</div> + +<div class="row"> + <h2>Genotype Data</h2> + + <p class="text-danger"> + Provide link to enable uploading of genotype data here.</p> +</div> + +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/http-error.html b/uploader/templates/http-error.html new file mode 100644 index 0000000..374fb86 --- /dev/null +++ b/uploader/templates/http-error.html @@ -0,0 +1,18 @@ +{%extends "base.html"%} + +{%block title%}HTTP Error: {{exc.code}}{%endblock%} + +{%block contents%} +<h1>{{exc.code}}: {{exc.description}}</h1> + +<div class="row"> + <p> + You attempted to access {{request_url}} which failed with the following + error: + </p> +</div> + +<div class="row"> + <pre>{{"\n".join(trace)}}</pre> +</div> +{%endblock%} diff --git a/uploader/templates/index.html b/uploader/templates/index.html new file mode 100644 index 0000000..aa1414e --- /dev/null +++ b/uploader/templates/index.html @@ -0,0 +1,107 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Home{%endblock%} + +{%block pagetitle%}Home{%endblock%} + +{%block contents%} + +<div class="row"> + {{flash_all_messages()}} + <div class="explainer"> + <p>Welcome to the <strong>GeneNetwork Data Upload and Quality Control + System</strong>.</p> + <p>This tool helps you prepare and upload research data to GeneNetwork for + analysis.</p> + + <h2 class="heading">Getting Started</h2> + <p>The sections below explain the features of the system. Review this guide + to learn how to use the system.</p> + + {%block extrapageinfo%}{%endblock%} + + <h3 class="subheading">Species</h3> + + <p>GeneNetwork supports genetic studies across multiple species (e.g. mice + [Mus musculus], human [homo sapiens], rats [Rattus norvegicus], etc.) . + Here you can:</p> + <ul> + <li>View all species that are currently supported</li> + <li>Add new species not yet in the system</li> + </ul> + + <h3 class="subheading">Populations</h3> + + <p>A "population" refers to a specific subgroup within a species that you’re + studying (e.g., BXD mice). Here you can:</p> + <ul> + <li>View the populations that exist for a selected species</li> + <li>Add new populations of study for a selected species</li> + </ul> + + <h3 class="subheading">Samples</h3> + + <p>Manage individual specimens or cases used in your experiments. These + include:</p> + + <ul> + <li>Experimental subjects</li> + <li>Data sources (e.g., tissue samples, clinical cases)</li> + <li>Strain means (instead of entering multiple BXD1 individuals, for + example, the mean would be entered for a single BXD1 strain)</li> + </ul> + + + <h3 class="subheading">Genotype Data</h3> + + <p>Upload and review genetic markers and allele encodings for your + population. Key details:</p> + + <ul> + <li>Markers are species-level (e.g., mouse SNP databases).</li> + <li>Allele data is population-specific (tied to your experimental + samples).</li> + </ul> + + <p><strong>Requirement</strong>: Samples must already have been registered + in the system before uploading genotype data.</p> + + <h3 class="subheading">Phenotype Data</h3> + + <p>Phenotypes are the visible traits or features of a living thing. For + example, phenotypes include:</p> + + <ul> + <li>Weight</li> + <li>Height</li> + <li>Color (such as the color of fur or eyes)</li> + </ul> + + <p>This part of the system will allow you to upload and manage the values + for different phenotypes from various samples in your studies.</p> + + <!-- + + <h3 class="subheading">Expression Data</h3> + + <p class="text-danger"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + <strong>TODO</strong>: Document this …</p> + + <h3 class="subheading">Individual Data</h3> + + <p class="text-danger"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + <strong>TODO</strong>: Document this …</p> + + <h3 class="subheading">RNA-Seq Data</h3> + + <p class="text-danger"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + <strong>TODO</strong>: Document this …</p> + </div> + --> +</div> + +{%endblock%} diff --git a/uploader/templates/insert_error.html b/uploader/templates/insert_error.html new file mode 100644 index 0000000..5301288 --- /dev/null +++ b/uploader/templates/insert_error.html @@ -0,0 +1,32 @@ +{%extends "base.html"%} + +{%block title%}Data Insertion Failure{%endblock%} + +{%block contents%} +<h1 class="heading">Insertion Failure</h1> + +<div class="row"> + <p> + There was an error inserting data into the database + </p> + + <p> + Please notify the developers of this issue when you encounter it, + providing the information below. + </p> + + <h4>Debugging Information</h4> + + <ul> + <li><strong>job id</strong>: {{job["jobid"]}}</li> + </ul> +</div> + +<div class="row"> + <h4>STDERR Output</h4> + <pre class="cli-output"> + {{job["stderr"]}} + </pre> +</div> + +{%endblock%} diff --git a/uploader/templates/insert_progress.html b/uploader/templates/insert_progress.html new file mode 100644 index 0000000..52177d6 --- /dev/null +++ b/uploader/templates/insert_progress.html @@ -0,0 +1,46 @@ +{%extends "base.html"%} +{%from "stdout_output.html" import stdout_output%} + +{%block extrameta%} +<meta http-equiv="refresh" content="5"> +{%endblock%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">{{job_name}}</h1> + +<div class="row"> + <form> + <div class="form-group"> + <label for="job_status" class="form-label">status:</label> + <span class="form-text">{{job_status}}: {{message}}</span> + </div> + +{%if job.get("stdout", "").split("\n\n") | length < 3 %} +{%set lines = 0%} +{%else%} +{%set lines = (job.get("stdout", "").split("\n\n") | length / 3) %} +{%endif%} +{%set totallines = job.get("totallines", lines+3) | int %} +{%if totallines > 1000 %} +{%set fraction = ((lines*1000)/totallines) %} +{%else%} +{%set fraction = (lines/totallines)%} +{%endif%} + + <div class="form-group"> + <label for="job_{{job_id}}" class="form-label">inserting: </label> + <progress id="jobs_{{job_id}}" + value="{{(fraction)}}" + class="form-control">{{fraction*100}}</progress> + <span class="form-text text-muted"> + {{"%.2f" | format(fraction * 100 | float)}}%</span> + </div> + </form> +</div> + + +{{stdout_output(job)}} + +{%endblock%} diff --git a/uploader/templates/insert_success.html b/uploader/templates/insert_success.html new file mode 100644 index 0000000..7e1fa8d --- /dev/null +++ b/uploader/templates/insert_success.html @@ -0,0 +1,19 @@ +{%extends "base.html"%} +{%from "stdout_output.html" import stdout_output%} + +{%block title%}Insertion Success{%endblock%} + +{%block contents%} +<h1 class="heading">Insertion Success</h1> + +<div class="row"> +<p>Data inserted successfully!</p> + +<p>The following queries were run:</p> +</div> + +<div class="row"> + {{stdout_output(job)}} +</div> + +{%endblock%} diff --git a/uploader/templates/login.html b/uploader/templates/login.html new file mode 100644 index 0000000..e76c644 --- /dev/null +++ b/uploader/templates/login.html @@ -0,0 +1,12 @@ +{%extends "index.html"%} + +{%block title%}Data Upload{%endblock%} + +{%block pagetitle%}log in{%endblock%} + +{%block extrapageinfo%} +<p class="text-dark"> + You <strong>need to + <a href="{{authserver_authorise_uri()}}" + title="Sign in to the system">sign in</a></strong> to use this system.</p> +{%endblock%} diff --git a/uploader/templates/macro-table-pagination.html b/uploader/templates/macro-table-pagination.html new file mode 100644 index 0000000..292c531 --- /dev/null +++ b/uploader/templates/macro-table-pagination.html @@ -0,0 +1,26 @@ +{%macro table_pagination(start_at, page_count, total_count, base_uri, name)%} +{%set ns = namespace(forward_uri=base_uri, back_uri=base_uri)%} +{%set ns.forward_uri="brr"%} + <div class="row"> + <div class="col-md-2" style="text-align: start;"> + {%if start_at > 0%} + <a href="{{base_uri + + '?start_at='+((start_at-page_count)|string) + + '&count='+(page_count|string)}}"> + <span class="glyphicon glyphicon-backward"></span> + Previous + </a> + {%endif%} + </div> + <div class="col-md-8" style="text-align: center;"> + Displaying {{name}} {{start_at+1}} to {{start_at+page_count if start_at+page_count < total_count else total_count}} of {{total_count}}</div> + <div class="col-md-2" style="text-align: end;"> + {%if start_at + page_count < total_count%} + <a href="{{base_uri + + '?start_at='+((start_at+page_count)|string) + + '&count='+(page_count|string)}}"> + Next<span class="glyphicon glyphicon-forward"></span></a> + {%endif%} + </div> + </div> +{%endmacro%} diff --git a/uploader/templates/phenotypes/add-phenotypes-base.html b/uploader/templates/phenotypes/add-phenotypes-base.html new file mode 100644 index 0000000..97b55f2 --- /dev/null +++ b/uploader/templates/phenotypes/add-phenotypes-base.html @@ -0,0 +1,331 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="add-phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">Add Phenotypes</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <form id="frm-add-phenotypes" + method="POST" + enctype="multipart/form-data" + action="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + use_bundle=use_bundle)}}" + data-resumable-target="{{url_for('files.resumable_upload_post')}}"> + <legend>Add New Phenotypes</legend> + + <div class="form-text help-block"> + {%block frm_add_phenotypes_documentation%}{%endblock%} + <p><strong class="text-warning">This will not update any existing phenotypes!</strong></p> + </div> + + {%block frm_add_phenotypes_elements%}{%endblock%} + + <div class="checkbox"> + <label> + <input id="chk-published" type="checkbox" name="published?" /> + These phenotypes are published</label> + </div> + + <fieldset id="fldset-publication-info" class="hidden"> + <legend>Publication Information</legend> + <div class="form-group"> + <label for="txt-pubmed-id" class="form-label">Pubmed ID</label> + <div class="input-group"> + <input id="txt-pubmed-id" name="pubmed-id" type="text" + class="form-control" /> + <span class="input-group-btn"> + <button id="btn-search-pubmed-id" class="btn btn-info">Search</button> + </span> + </div> + <span id="search-pubmed-id-error" + class="form-text text-muted text-danger hidden"> + </span><br /> + <span class="form-text text-muted"> + Enter your publication's PubMed ID above and click "Search" to search + for some (or all) of the publication details requested below. + </span> + </div> + + <div class="form-group"> + <label for="txt-publication-authors" class="form-label">Authors</label> + <input id="txt-publication-authors" name="publication-authors" + type="text" class="form-control" /> + <span class="form-text text-muted"> + Enter the authors in the following format …</span> + </div> + + <div class="form-group"> + <label for="txt-publication-title" class="form-label"> + Publication Title</label> + <input id="txt-publication-title" name="publication-title" type="text" + class="form-control" /> + <span class="form-text text-muted"> + Enter your publication's title.</span> + </div> + + <div class="form-group"> + <label for="txt-publication-abstract" class="form-label"> + Publication Abstract</label> + <textarea id="txt-publication-abstract" name="publication-abstract" + class="form-control" rows="10"></textarea> + <span class="form-text text-muted"> + Enter the abstract for your publication.</span> + </div> + + <div class="form-group"> + <label for="txt-publication-journal" class="form-label">Journal</label> + <input id="txt-publication-journal" name="journal" type="text" + class="form-control" /> + <span class="form-text text-muted"> + Enter the name of the journal where your work was published.</span> + </div> + + <div class="form-group"> + <label for="txt-publication-volume" class="form-label">Volume</label> + <input id="txt-publication-volume" name="publication-volume" type="text" + class="form-control" /> + <span class="form-text text-muted"> + Enter the volume in the following format …</span> + </div> + + <div class="form-group"> + <label for="txt-publication-pages" class="form-label">Pages</label> + <input id="txt-publication-pages" name="publication-pages" type="text" + class="form-control" /> + <span class="form-text text-muted"> + Enter the journal volume where your work was published.</span> + </div> + + <div class="form-group"> + <label for="select-publication-month" class="form-label"> + Publication Month</label> + <select id="select-publication-month" name="publication-month" + class="form-control"> + {%for month in monthnames%} + <option value="{{month | lower}}" + {%if current_month | lower == month | lower%} + selected="selected" + {%endif%}>{{month | capitalize}}</option> + {%endfor%} + </select> + <span class="form-text text-muted"> + Select the month when the work was published. + <span class="text-danger"> + This cannot be before, say 1600 and cannot be in the future!</span></span> + </div> + + <div class="form-group"> + <label for="txt-publication-year" class="form-label">Publication Year</label> + <input id="txt-publication-year" name="publication-year" type="text" + class="form-control" value="{{current_year}}" /> + <span class="form-text text-muted"> + Enter the year your work was published. + <span class="text-danger"> + This cannot be before, say 1600 and cannot be in the future!</span> + </span> + </div> + </fieldset> + + <div class="form-group"> + <input type="submit" + value="upload phenotypes" + class="btn btn-primary" /> + </div> + </form> +</div> + +<div class="row"> + {%block page_documentation%}{%endblock%} +</div> + +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + var remove_class = (element, classvalue) => { + new_classes = (element.attr("class") || "").split(" ").map((val) => { + return val.trim(); + }).filter((val) => { + return ((val !== classvalue) && + (val !== "")) + }).join(" "); + + if(new_classes === "") { + element.removeAttr("class"); + } else { + element.attr("class", new_classes); + } + }; + + var add_class = (element, classvalue) => { + remove_class(element, classvalue); + element.attr("class", (element.attr("class") || "") + " " + classvalue); + }; + + $("#chk-published").on("click", (event) => { + pub_details = $("#fldset-publication-info") + if(event.target.checked) { + // display the publication details + remove_class(pub_details, "hidden"); + } else { + // hide the publication details + add_class(pub_details, "hidden"); + } + }); + + var extract_details = (pubmed_id, details) => { + var months = { + "jan": "January", + "feb": "February", + "mar": "March", + "apr": "April", + "may": "May", + "jun": "June", + "jul": "July", + "aug": "August", + "sep": "September", + "oct": "October", + "nov": "November", + "dec": "December" + }; + var _date = details[pubmed_id].pubdate.split(" "); + return { + "authors": details[pubmed_id].authors.map((authobj) => { + return authobj.name; + }), + "title": details[pubmed_id].title, + "journal": details[pubmed_id].fulljournalname, + "volume": details[pubmed_id].volume, + "pages": details[pubmed_id].pages, + "month": _date.length > 1 ? months[_date[1].toLowerCase()] : "jan", + "year": _date[0], + }; + }; + + var update_publication_details = (details) => { + Object.entries(details).forEach((entry) => {; + switch(entry[0]) { + case "authors": + $("#txt-publication-authors").val(entry[1].join(", ")); + break; + case "month": + $("#select-publication-month") + .children("option") + .each((index, child) => { + child.selected = child.value == entry[1].toLowerCase(); + }); + default: + $("#txt-publication-" + entry[0]).val(entry[1]); + break; + } + }); + }; + + var fetch_publication_abstract = (pubmed_id, pub_details) => { + $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi", + { + "method": "GET", + "data": { + "db": "pubmed", + "id": pubmed_id, + "rettype": "abstract", + "retmode": "xml" + }, + "success": (data, textStatus, jqXHR) => { + update_publication_details({ + ...pub_details, + ...{ + "abstract": Array.from(data + .getElementsByTagName( + "Abstract")[0] + .children) + .map((elt) => {return elt.textContent.trim();}) + .join("\r\n") + }}); + }, + "error": (jqXHR, textStatus, errorThrown) => {}, + "complete": (jqXHR, textStatus) => {}, + "dataType": "xml" + }); + }; + + var fetch_publication_details = (pubmed_id, complete_thunks) => { + error_display = $("#search-pubmed-id-error"); + error_display.text(""); + add_class(error_display, "hidden"); + $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi", + { + "method": "GET", + "data": {"db": "pubmed", "id": pubmed_id, "format": "json"}, + "success": (data, textStatus, jqXHR) => { + // process and update publication details + hasError = ( + Object.hasOwn(data, "error") || + Object.hasOwn(data.result[pubmed_id], "error")); + if(hasError) { + error_display.text( + "There was an error fetching a publication with " + + "the given PubMed ID! The error received " + + "was: '" + ( + data.error || + data.result[pubmed_id].error) + + "'. Please check ID you provided and try " + + "again."); + remove_class(error_display, "hidden"); + } else { + fetch_publication_abstract( + pubmed_id, + extract_details(pubmed_id, data.result)); + } + }, + "error": (jqXHR, textStatus, errorThrown) => {}, + "complete": () => { + complete_thunks.forEach((thunk) => {thunk()}); + }, + "dataType": "json" + }); + }; + + $("#btn-search-pubmed-id").on("click", (event) => { + event.preventDefault(); + var search_button = event.target; + var pubmed_id = $("#txt-pubmed-id").val().trim(); + remove_class($("#txt-pubmed-id").parent(), "has-error"); + if(pubmed_id == "") { + add_class($("#txt-pubmed-id").parent(), "has-error"); + return false; + } + + search_button.disabled = true; + // Fetch publication details + fetch_publication_details(pubmed_id, + [() => {search_button.disabled = false;}]); + return false; + }); +</script> + +{%block more_javascript%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/phenotypes/add-phenotypes-raw-files.html b/uploader/templates/phenotypes/add-phenotypes-raw-files.html new file mode 100644 index 0000000..7f8d8b0 --- /dev/null +++ b/uploader/templates/phenotypes/add-phenotypes-raw-files.html @@ -0,0 +1,732 @@ +{%extends "phenotypes/add-phenotypes-base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} +{%from "phenotypes/macro-display-preview-table.html" import display_preview_table%} +{%from "phenotypes/macro-display-resumable-elements.html" import display_resumable_elements%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="add-phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">Add Phenotypes</a> +</li> +{%endblock%} + +{%block frm_add_phenotypes_documentation%} +<p>This page will allow you to upload all the separate files that make up your + phenotypes. Here, you will have to upload each separate file individually. If + you want instead to upload all your files as a single ZIP file, + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + use_bundle=true)}}" + title="">click here</a>.</p> +{%endblock%} + +{%block frm_add_phenotypes_elements%} +<fieldset id="fldset-file-metadata"> + <legend>File(s) Metadata</legend> + <div class="form-group"> + <label for="txt-file-separator" class="form-label">File Separator</label> + <div class="input-group"> + <input id="txt-file-separator" + name="file-separator" + type="text" + value="	" + class="form-control" + maxlength="1" /> + <span class="input-group-btn"> + <button id="btn-reset-file-separator" class="btn btn-info">Reset Default</button> + </span> + </div> + <span class="form-text text-muted"> + Provide the character that separates the fields in your file(s). It should + be the same character for all files (if more than one is provided).<br /> + A tab character will be assumed if you leave this field blank. See + <a href="#docs-file-separator" + title="Documentation for file-separator characters"> + documentation for more information</a>. + </span> + </div> + + <div class="form-group"> + <label for="txt-file-comment-character" class="form-label">File Comment-Character</label> + <div class="input-group"> + <input id="txt-file-comment-character" + name="file-comment-character" + type="text" + value="#" + class="form-control" + maxlength="1" /> + <span class="input-group-btn"> + <button id="btn-reset-file-comment-character" class="btn btn-info"> + Reset Default</button> + </span> + </div> + <span class="form-text text-muted"> + This specifies that lines that begin with the character provided will be + considered comment lines and ignored in their entirety. See + <a href="#docs-file-comment-character" + title="Documentation for comment characters"> + documentation for more information</a>. + </span> + </div> + + <div class="form-group"> + <label for="txt-file-na" class="form-label">File "No-Value" Indicators</label> + <div class="input-group"> + <input id="txt-file-na" + name="file-na" + type="text" + value="- NA N/A" + class="form-control" /> + <span class="input-group-btn"> + <button id="btn-reset-file-na" class="btn btn-info">Reset Default</button> + </span> + </div> + <span class="form-text text-muted"> + This specifies strings in your file indicate that there is no value for a + particular cell (a cell is where a column and row intersect). Provide a + space-separated list of strings if you have more than one way of + indicating no values. See + <a href="#docs-file-na" title="Documentation for no-value fields"> + documentation for more information</a>.</span> + </div> +</fieldset> + +<fieldset id="fldset-data-files"> + <legend>Data File(s)</legend> + + <div class="form-group non-resumable-elements"> + <label for="finput-phenotype-descriptions" class="form-label"> + Phenotype Descriptions</label> + <input id="finput-phenotype-descriptions" + name="phenotype-descriptions" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-desc" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the phenotype descriptions, + <a href="#docs-file-phenotype-description" + title="Documentation of the phenotype data file format."> + the documentation for the expected format of the file</a>.</span> + </div> + + {{display_resumable_elements( + "resumable-phenotype-descriptions", + "phenotype descriptions", + '<p>You can drop a CSV file that contains the phenotype descriptions here, + or you can click the "Browse" button (below and to the right) to select it + from your computer.</p> + <p>The CSV file must conform to some standards, as documented in the + <a href="#docs-file-phenotype-description" + title="Documentation of the phenotype data file format."> + "Phenotypes Descriptions" documentation</a> section below.</p>')}} + + + <div class="form-group non-resumable-elements"> + <label for="finput-phenotype-data" class="form-label">Phenotype Data</label> + <input id="finput-phenotype-data" + name="phenotype-data" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-data" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the phenotype data. See + <a href="#docs-file-phenotype-data" + title="Documentation of the phenotype data file format."> + the documentation for the expected format of the file</a>.</span> + </div> + + {{display_resumable_elements( + "resumable-phenotype-data", + "phenotype data", + '<p>You can drop a CSV file that contains the phenotype data here, + or you can click the "Browse" button (below and to the right) to select it + from your computer.</p> + <p>The CSV file must conform to some standards, as documented in the + <a href="#docs-file-phenotype-data" + title="Documentation of the phenotype data file format."> + "Phenotypes Data" documentation</a> section below.</p>')}} + + {%if population.Family in families_with_se_and_n%} + <div class="form-group non-resumable-elements"> + <label for="finput-phenotype-se" class="form-label">Phenotype: Standard Errors</label> + <input id="finput-phenotype-se" + name="phenotype-se" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-se" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the standard errors for the phenotypes, + computed from the data above.</span> + </div> + {{display_resumable_elements( + "resumable-phenotype-se", + "standard errors", + '<p>You can drop a CSV file that contains the computed standard-errors data + here, or you can click the "Browse" button (below and to the right) to + select it from your computer.</p> + <p>The CSV file must conform to some standards, as documented in the + <a href="#docs-file-phenotype-se" + title="Documentation of the phenotype data file format."> + "Phenotypes Data" documentation</a> section below.</p>')}} + + + <div class="form-group non-resumable-elements"> + <label for="finput-phenotype-n" class="form-label">Phenotype: Number of Samples/Individuals</label> + <input id="finput-phenotype-n" + name="phenotype-n" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-n" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the number of samples/individuals used in + the computation of the standard errors above.</span> + </div> + {{display_resumable_elements( + "resumable-phenotype-n", + "number of samples/individuals", + '<p>You can drop a CSV file that contains the number of samples/individuals + used in computation of the standard-errors here, or you can click the + "Browse" button (below and to the right) to select it from your computer. + </p> + <p>The CSV file must conform to some standards, as documented in the + <a href="#docs-file-phenotype-n" + title="Documentation of the phenotype data file format."> + "Phenotypes Data" documentation</a> section below.</p>')}} +</fieldset> +{%endif%} +{%endblock%} + + +{%block page_documentation%} +<div class="row"> + <h2 class="heading" id="docs-help">Help</h2> + <h3 class="subheading">Common Features</h3> + <p>The following are the common expectations for <strong>ALL</strong> the + files provided in the form above: + <ul> + <li>The file <strong>MUST</strong> be character-separated values (CSV) + text file</li> + <li>The first row in the file <strong>MUST</strong> be a heading row, and + will be composed of the list identifiers for all of + samples/individuals/cases involved in your study.</li> + <li>The first column of data in the file <strong>MUST</strong> be the + identifiers for all of the phenotypes you wish to upload.</li> + </ul> + </p> + + <p>If you do not specify the separator character, then we will assume a + <strong>TAB</strong> character was used as your separator.</p> + + <p>We also assume you might include comments lines in your files. In that + case, if you do not specify what character denotes that a line in your files + is a comment line, we will assume the <strong>#</strong> character.<br /> + A comment <strong>MUST ALWAYS</strong> begin at the start of the line marked + with the comment character specified.</p> + + <h3 class="subheading" id="docs-file-metadata">File Metadata</h3> + <p>We request some details about your files to help us parse and process the + files correctly. The details we collect are:</p> + <dl> + <dt id="docs-file-separator">File separator</dt> + <dd>The files you provide should be character-separated value (CSV) files. + We need to know what character you used to separate the values in your + file. Some common ones are the Tab character, the comma, etc.<br /> + Providing that information makes it possible for the system to parse and + process your files correctly.<br> + <strong>NOTE:</strong> All the files you upload MUST use the same + separator.</dd> + + <dt id="docs-file-comment-character">Comment character</dt> + <dd>We support use of comment lines in your files. We only support one type + of comment style, the <em>line comment</em>.<br /> + This mean the comment begins at the start of the line, and the end of that + line indicates the end of that comment. If you have a really long comment, + then you need to break it across multiple lines, marking each line a + comment line.<br /> + The "comment character" is the character at the start of the line that + indicates that the line is a line comment.</dd> + + <dt id="docs-file-na">No-Value indicator(s)</dt> + <dd>Data in the real world is messy, and in some cases, entirely absent. You + need to indicate, in your files, that a particular field did not have a + value, and once you do that, you then need to let the system know how you + mark such fields. Common ways of indicating "empty values" are, leaving + the field blank, using a character such as '-', or using strings like + "NA", "N/A", "NULL", etc.<br /> + Providing this information will help with parsing and processing such + no-value fields the correct way.</dd> + </dl> + + <h3 class="subheading" id="docs-file-phenotype-description"> + file: Phenotypes Descriptions</h3> + <p>The data in this file is a matrix of <em>phenotypes × metadata-fields</em>. + Please note we use the term "metadata-fields" above loosely, due to lack of + a good word for this.</p> + <p>The file <strong>MUST</strong> have columns in this order: + <dl> + <dt>Phenotype Identifiers</dt> + <dd>These are the names/identifiers for your phenotypes. These + names/identifiers are the same ones you will have in all the other files you are + uploading.</dd> + + <dt>Descriptions</dt> + <dd>Each phenotype will need a description. Good description are necessary + to inform other people of what the data is about. Good description are + hard to construct, so we provide + <a href="https://info.genenetwork.org/faq.php#q-22" + title="How to write phenotype descriptions"> + advice on describing your phenotypes.</a></dd> + + <dt>Units</dt> + <dd>Each phenotype will need units for the measurements taken. If there are + none, then indicate the field is a no-value field.</dd> + </dl></p> + <p>You can add more columns after those three if you want to, but these 3 + <strong>MUST</strong> be present.</p> + <p>The file would, for example, look like the following:</p> + <code>id,description,units,…<br /> + pheno10001|Central nervous system, behavior, cognition; …|mg|…<br /> + pheno10002|Aging, metabolism, central nervous system: …|mg|…<br /> + â‹®<br /></code> + + <p><strong>Note 01</strong>: The first usable row is the heading row.</p> + <p><strong>Note 02: </strong>This example demonstrates a subtle issue that + could make your CSV file invalid — the choice of your field separator + character.<br > + In the example above, we use the pipe character (<code>|</code>) as our + field separator. This is because, if we follow the advice on how to write + good descriptions, then we cannot use the comma as our separator – if + we did, then our CSV file would be invalid because the system would have no + way to tell the difference between the comma as a field separator, and the + comma as a way to separate the "general category and ontology terms".</p> + + <h3 class="subheading">file: Phenotype Data, Standard Errors and/or Sample Counts</h3> + <span id="docs-file-phenotype-data"></span> + <span id="docs-file-phenotype-se"></span> + <span id="docs-file-phenotype-n"></span> + <p>The data is a matrix of <em>phenotypes × individuals</em>, e.g.</p> + <code> + # num-cases: 2549 + # num-phenos: 13 + id,IND001,IND002,IND003,IND004,…<br /> + pheno10001,61.400002,54.099998,483,49.799999,…<br /> + pheno10002,49,50.099998,403,45.5,…<br /> + pheno10003,62.5,53.299999,501,62.900002,…<br /> + pheno10004,53.099998,55.099998,403,NA,…<br /> + â‹®<br /></code> + + <p>where <code>IND001,IND002,IND003,IND004,…</code> are the + samples/individuals/cases in your study, and + <code>pheno10001,pheno10002,pheno10004,pheno10004,…</code> are the + identifiers for your phenotypes.</p> + <p>The lines beginning with the "<em>#</em>" symbol (i.e. + <code># num-cases: 2549</code> and <code># num-phenos: 13</code> are comment + lines and will be ignored</p> + <p>In this example, the comma (,) is used as the file separator.</p> +</div> + +{%endblock%} + +{%block sidebarcontents%} +{{display_preview_table("tbl-preview-pheno-desc", "descriptions")}} +{{display_preview_table("tbl-preview-pheno-data", "data")}} +{%if population.Family in families_with_se_and_n%} +{{display_preview_table("tbl-preview-pheno-se", "standard errors")}} +{{display_preview_table("tbl-preview-pheno-n", "number of samples")}} +{%endif%} +{{display_pheno_dataset_card(species, population, dataset)}} +{%endblock%} + + +{%block more_javascript%} +<script src="{{url_for('base.node_modules', + filename='resumablejs/resumable.js')}}"></script> +<script type="text/javascript" src="/static/js/files.js"></script> + +<script type="text/javascript"> + $("#btn-reset-file-separator").on("click", (event) => { + event.preventDefault(); + $("#txt-file-separator").val("\t"); + $("#txt-file-separator").trigger("change"); + }); + $("#btn-reset-file-comment-character").on("click", (event) => { + event.preventDefault(); + $("#txt-file-comment-character").val("#"); + $("#txt-file-comment-character").trigger("change"); + }); + $("#btn-reset-file-na").on("click", (event) => { + event.preventDefault(); + $("#txt-file-na").val("- NA N/A"); + $("#txt-file-na").trigger("change"); + }); + + var update_preview = (table, filedata, formdata, numrows) => { + table.find("thead tr").remove() + table.find(".data-row").remove(); + var linenum = 0; + var tableheader = table.find("thead"); + var tablebody = table.find("tbody"); + var numheadings = 0; + var navalues = formdata + .na_strings + .split(" ") + .map((v) => {return v.trim();}) + .filter((v) => {return Boolean(v);}); + filedata.forEach((line) => { + if(line.startsWith(formdata.comment_char) || linenum >= numrows) { + return false; + } + var row = $("<tr></tr>"); + line.split(formdata.separator) + .map((field) => { + var value = field.trim(); + if(navalues.includes(value)) { + return "⋘NULâ‹™"; + } + return value; + }) + .filter((field) => { + return (field !== "" && field != undefined && field != null); + }) + .forEach((field) => { + if(linenum == 0) { + numheadings += 1; + var tablefield = $("<th></th>"); + tablefield.text(field); + row.append(tablefield); + } else { + add_class(row, "data-row"); + var tablefield = $("<td></td>"); + tablefield.text(field); + row.append(tablefield); + } + }); + + if(linenum == 0) { + tableheader.append(row); + } else { + tablebody.append(row); + } + linenum += 1; + }); + + if(table.find("tbody tr.data-row").length > 0) { + add_class(table.find(".data-row-template"), "hidden"); + } else { + remove_class(table.find(".data-row-template"), "hidden"); + } + }; + + var makePreviewUpdater = (preview_table) => { + return (data) => { + update_preview( + preview_table, + data, + filesMetadata(), + PREVIEW_ROWS); + }; + }; + + var preview_tables_to_elements_map = { + "#tbl-preview-pheno-desc": "#finput-phenotype-descriptions", + "#tbl-preview-pheno-data": "#finput-phenotype-data", + "#tbl-preview-pheno-se": "#finput-phenotype-se", + "#tbl-preview-pheno-n": "#finput-phenotype-n" + }; + + var filesMetadata = () => { + return { + "separator": $("#txt-file-separator").val(), + "comment_char": $( + "#txt-file-comment-character").val(), + "na_strings": $("#txt-file-na").val() + } + }; + + var PREVIEW_ROWS = 5; + + var handler_update_previews = (event) => { + Object.entries(preview_tables_to_elements_map).forEach((mapentry) => { + var preview_table = $(mapentry[0]); + var file_input = $(mapentry[1]); + if(file_input.length === 1) { + readFirstNLines( + file_input[0].files[0], + 10, + [makePreviewUpdater(preview_table)]); + } + }); + }; + + [ + "#txt-file-separator", + "#txt-file-comment-character", + "#txt-file-na" + ].forEach((elementid) => { + $(elementid).on("change", handler_update_previews); + }); + + [ + "#finput-phenotype-descriptions", + "#finput-phenotype-data", + "#finput-phenotype-se", + "#finput-phenotype-n" + ].forEach((elementid) => { + $(elementid).on("change", (event) => { + readFirstNLines( + event.target.files[0], + 10, + [makePreviewUpdater( + $("#" + event.target.getAttribute("data-preview-table")))]); + }); + }); + + + var resumableDisplayFiles = (display_area, files) => { + files.forEach((file) => { + display_area.find(".file-display").remove(); + var display_element = display_area + .find(".file-display-template") + .clone(); + remove_class(display_element, "hidden"); + remove_class(display_element, "file-display-template"); + add_class(display_element, "file-display"); + display_element.find(".filename").text(file.name + || file.fileName + || file.relativePath + || file.webkitRelativePath); + display_element.find(".filesize").text( + (file.size / (1024*1024)).toFixed(2) + "MB"); + display_element.find(".fileuniqueid").text(file.uniqueIdentifier); + display_element.find(".filemimetype").text(file.file.type); + display_area.append(display_element); + }); + }; + + + var indicateProgress = (resumable, progress_bar) => { + return () => {/*Has no event!*/ + var progress = (resumable.progress() * 100).toFixed(2); + var pbar = progress_bar.find(".progress-bar"); + remove_class(progress_bar, "hidden"); + pbar.css("width", progress+"%"); + pbar.attr("aria-valuenow", progress); + pbar.text("Uploading: " + progress + "%"); + }; + }; + + var retryUpload = (retry_button, cancel_button) => { + retry_button.on("click", (event) => { + resumable.files.forEach((file) => {file.retry();}); + add_class(retry_button, "hidden"); + remove_class(cancel_button, "hidden"); + add_class(browse_button, "hidden"); + }); + }; + + var cancelUpload = (cancel_button, retry_button) => { + cancel_button.on("click", (event) => { + resumable.files.forEach((file) => { + if(file.isUploading()) { + file.abort(); + } + }); + add_class(cancel_button, "hidden"); + remove_class(retry_button, "hidden"); + remove_class(browse_button, "hidden"); + }); + }; + + + var startUpload = (browse_button, retry_button, cancel_button) => { + return (event) => { + remove_class(cancel_button, "hidden"); + add_class(retry_button, "hidden"); + add_class(browse_button, "hidden"); + }; + }; + + var processForm = (form) => { + var formdata = new FormData(form); + uploaded_files.forEach((msg) => { + formdata.delete(msg["file-input-name"]); + formdata.append(msg["file-input-name"], JSON.stringify({ + "uploaded-file": msg["uploaded-file"], + "original-name": msg["original-name"] + })); + }); + formdata.append("resumable-upload", "true"); + return formdata; + } + + var uploaded_files = new Set(); + var submitForm = (new_file) => { + uploaded_files.add(new_file); + if(uploaded_files.size === resumables.length) { + var form = $("#frm-add-phenotypes"); + if(form.length !== 1) { + // TODO: Handle error somehow? + alert("Could not find form!!!"); + return false; + } + + $.ajax({ + "url": form.attr("action"), + "type": "POST", + "data": processForm(form[0]), + "processData": false, + "contentType": false, + "success": (data, textstatus, jqxhr) => { + // TODO: Redirect to endpoint that should come as part of the + // success/error message. + console.log("SUCCESS DATA: ", data); + console.log("SUCCESS STATUS: ", textstatus); + console.log("SUCCESS jqXHR: ", jqxhr); + window.location.assign(window.location.origin + data["redirect-to"]); + }, + }); + return false; + } + return false; + }; + + var uploadSuccess = (file_input_name) => { + return (file, message) => { + submitForm({...JSON.parse(message), "file-input-name": file_input_name}); + }; + }; + + + var uploadError = () => { + return (message, file) => { + $("#frm-add-phenotypes input[type=submit]").removeAttr("disabled"); + console.log("THE FILE:", file); + console.log("THE ERROR MESSAGE:", message); + }; + }; + + + + var makeResumableObject = (form_id, file_input_id, resumable_element_id, preview_table_id) => { + var the_form = $("#" + form_id); + var file_input = $("#" + file_input_id); + var submit_button = the_form.find("input[type=submit]"); + if(file_input.length != 1) { + return false; + } + var r = errorHandler( + fileSuccessHandler( + uploadStartHandler( + filesAddedHandler( + markResumableDragAndDropElement( + makeResumableElement( + the_form.attr("data-resumable-target"), + file_input.parent(), + $("#" + resumable_element_id), + submit_button, + ["csv", "tsv"]), + file_input.parent(), + $("#" + resumable_element_id), + $("#" + resumable_element_id + "-browse-button")), + (files) => { + // TODO: Also trigger preview! + resumableDisplayFiles( + $("#" + resumable_element_id + "-selected-files"), files); + files.forEach((file) => { + readFirstNLines( + file.file, + 10, + [makePreviewUpdater( + $("#" + preview_table_id))]) + }); + }), + startUpload($("#" + resumable_element_id + "-browse-button"), + $("#" + resumable_element_id + "-retry-button"), + $("#" + resumable_element_id + "-cancel-button"))), + uploadSuccess(file_input.attr("name"))), + uploadError()); + + /** Setup progress indicator **/ + progressHandler( + r, + indicateProgress(r, $("#" + resumable_element_id + "-progress-bar"))); + + return r; + }; + + var resumables = [ + ["frm-add-phenotypes", "finput-phenotype-descriptions", "resumable-phenotype-descriptions", "tbl-preview-pheno-desc"], + ["frm-add-phenotypes", "finput-phenotype-data", "resumable-phenotype-data", "tbl-preview-pheno-data"], + ["frm-add-phenotypes", "finput-phenotype-se", "resumable-phenotype-se", "tbl-preview-pheno-se"], + ["frm-add-phenotypes", "finput-phenotype-n", "resumable-phenotype-n", "tbl-preview-pheno-n"], + ].map((row) => { + return makeResumableObject(row[0], row[1], row[2], row[3]); + }).filter((val) => { + return Boolean(val); + }); + + $("#frm-add-phenotypes input[type=submit]").on("click", (event) => { + event.preventDefault(); + // TODO: Check all the relevant files exist + // TODO: Verify that files are not duplicated + var filenames = []; + var nondupfiles = []; + resumables.forEach((r) => { + var fname = r.files[0].file.name; + filenames.push(fname); + if(!nondupfiles.includes(fname)) { + nondupfiles.push(fname); + } + }); + + // Check that all files were provided + if(resumables.length !== filenames.length) { + window.alert("You MUST provide all the files requested."); + event.target.removeAttribute("disabled"); + return false; + } + + // Check that there are no duplicate files + var duplicates = Object.entries(filenames.reduce( + (acc, curr, idx, arr) => { + acc[curr] = (acc[curr] || 0) + 1; + return acc; + }, + {})).filter((entry) => {return entry[1] !== 1;}); + if(duplicates.length > 0) { + var msg = "The file(s):\r\n"; + msg = msg + duplicates.reduce( + (msgstr, afile) => { + return msgstr + " • " + afile[0] + "\r\n"; + }, + ""); + msg = msg + "is(are) duplicated. Please fix and try again."; + window.alert(msg); + event.target.removeAttribute("disabled"); + return false; + } + // TODO: Check all fields + // Start the uploads. + event.target.setAttribute("disabled", "disabled"); + resumables.forEach((r) => {r.upload();}); + }); +</script> +{%endblock%} diff --git a/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html b/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html new file mode 100644 index 0000000..898fc0c --- /dev/null +++ b/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html @@ -0,0 +1,207 @@ +{%extends "phenotypes/add-phenotypes-base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="add-phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">Add Phenotypes</a> +</li> +{%endblock%} + +{%block frm_add_phenotypes_documentation%} +<p>Select the zip file bundle containing information on the phenotypes you + wish to upload, then click the "Upload Phenotypes" button below to + upload the data.</p> +<p>If you wish to upload the files individually instead, + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + title="">click here</a>.</p> +<p>See the <a href="#section-file-formats">File Formats</a> section below + to get an understanding of what is expected of the bundle files you + upload.</p> +{%endblock%} + +{%block frm_add_phenotypes_elements%} +<div class="form-group"> + <label for="finput-phenotypes-bundle" class="form-label"> + Phenotypes Bundle</label> + <input type="file" + id="finput-phenotypes-bundle" + name="phenotypes-bundle" + accept="application/zip, .zip" + required="required" + class="form-control" /> +</div> +{%endblock%} + +{%block page_documentation%} +<div class="row"> + <h2 class="heading" id="section-file-formats">File Formats</h2> + <p>We accept an extended form of the + <a href="https://kbroman.org/qtl2/assets/vignettes/input_files.html#format-of-the-data-files" + title="R/qtl2 software input file format documentation"> + input files' format used with the R/qtl2 software</a> as a single ZIP + file</p> + <p>The files that are used for this feature are: + <ul> + <li>the <em>control</em> file</li> + <li><em>pheno</em> file(s)</li> + <li><em>phenocovar</em> file(s)</li> + <li><em>phenose</em> files(s)</li> + </ul> + </p> + <p>Other files within the bundle will be ignored, for this feature.</p> + <p>The following section will detail the expectations for each of the + different file types within the uploaded ZIP file bundle for phenotypes:</p> + + <h3 class="subheading">Control File</h3> + <p>There <strong>MUST be <em>one, and only one</em></strong> file that acts + as the control file. This file can be: + <ul> + <li>a <em>JSON</em> file, or</li> + <li>a <em>YAML</em> file.</li> + </ul> + </p> + + <p>The control file is useful for defining things about the bundle such as:</p> + <ul> + <li>The field separator value (default: <code>sep: ','</code>). There can + only ever be one field separator and it <strong>MUST</strong> be the same + one for <strong>ALL</strong> files in the bundle.</li> + <li>The comment character (default: <code>comment.char: '#'</code>). Any + line that starts with this character will be considered a comment line and + be ignored in its entirety.</li> + <li>Code for missing values (default: <code>na.strings: 'NA'</code>). You + can specify more than one code to indicate missing values, e.g. + <code>{…, "na.strings": ["NA", "N/A", "-"], …}</code></li> + </ul> + + <h3 class="subheading"><em>pheno</em> File(s)</h3> + <p>These files are the main data files. You must have at least one of these + files in your bundle for it to be valid for this step.</p> + <p>The data is a matrix of <em>individuals × phenotypes</em> by default, as + below:<br /> + <code> + id,10001,10002,10003,10004,…<br /> + BXD1,61.400002,54.099998,483,49.799999,…<br /> + BXD2,49,50.099998,403,45.5,…<br /> + BXD5,62.5,53.299999,501,62.900002,…<br /> + BXD6,53.099998,55.099998,403,NA,…<br /> + â‹®<br /></code> + </p> + <p>If the <code>pheno_transposed</code> value is set to <code>True</code>, + then the data will be a <em>phenotypes × individuals</em> matrix as in the + example below:<br /> + <code> + id,BXD1,BXD2,BXD5,BXD6,…<br /> + 10001,61.400002,49,62.5,53.099998,…<br /> + 10002,54.099998,50.099998,53.299999,55.099998,…<br /> + 10003,483,403,501,403,…<br /> + 10004,49.799999,45.5,62.900002,NA,…<br /> + â‹® + </code> + </p> + + + <h3 class="subheading"><em>phenocovar</em> File(s)</h3> + <p>At least one phenotypes metadata file with the metadata values such as + descriptions, PubMed Identifier, publication titles (if present), etc.</p> + <p>The data in this/these file(s) is a matrix of + <em>phenotypes × phenotypes-covariates</em>. The first column is always the + phenotype names/identifiers — same as in the R/qtl2 format.</p> + <p><em>phenocovar</em> files <strong>should never be transposed</strong>!</p> + <p>This file <strong>MUST</strong> be present in the bundle, and have data for + the bundle to be considered valid by our system for this step.<br /> + In addition to that, the following are the fields that <strong>must be + present</strong>, and + have values, in the file before the file is considered valid: + <ul> + <li><em>description</em>: A description for each phenotype. Useful + for users to know what the phenotype is about.</li> + <li><em>units</em>: The units of measurement for the phenotype, + e.g. milligrams for brain weight, centimetres/millimetres for + tail-length, etc.</li> + </ul></p> + + <p>The following <em>optional</em> fields can also be provided: + <ul> + <li><em>pubmedid</em>: A PubMed Identifier for the publication where + the phenotype is published. If this field is not provided, the system will + assume your phenotype is not published.</li> + </ul> + </p> + <p>These files will be marked up in the control file with the + <code>phenocovar</code> key, as in the examples below: + <ol> + <li>JSON: single file<br /> + <code>{<br /> + â‹®,<br /> + "phenocovar": "your_covariates_file.csv",<br /> + â‹®<br /> + } + </code> + </li> + <li>JSON: multiple files<br /> + <code>{<br /> + â‹®,<br /> + "phenocovar": [<br /> + "covariates_file_01.csv",<br /> + "covariates_file_01.csv",<br /> + â‹®<br /> + ],<br /> + â‹®<br /> + } + </code> + </li> + <li>YAML: single file or<br /> + <code> + â‹®<br /> + phenocovar: your_covariates_file.csv<br /> + â‹® + </code> + </li> + <li>YAML: multiple files<br /> + <code> + â‹®<br /> + phenocovar:<br /> + - covariates_file_01.csv<br /> + - covariates_file_02.csv<br /> + - covariates_file_03.csv<br /> + …<br /> + â‹® + </code> + </li> + </ol> + </p> + + <h3 class="subheading"><em>phenose</em> and <em>phenonum</em> File(s)</h3> + <p>These are extensions to the R/qtl2 standard, i.e. these types ofs file are + not supported by the original R/qtl2 file format</p> + <p>We use these files to upload the standard errors (<em>phenose</em>) when + the data file (<em>pheno</em>) is average data. In that case, the + <em>phenonum</em> file(s) contains the number of individuals that were + involved when computing the averages.</p> + <p>Both types of files are matrices of <em>individuals × phenotypes</em> by + default. Like the related <em>pheno</em> files, if + <code>pheno_transposed: True</code>, then the file will be a matrix of + <em>phenotypes × individuals</em>.</p> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_pheno_dataset_card(species, population, dataset)}} +{%endblock%} diff --git a/uploader/templates/phenotypes/base.html b/uploader/templates/phenotypes/base.html new file mode 100644 index 0000000..adbc012 --- /dev/null +++ b/uploader/templates/phenotypes/base.html @@ -0,0 +1,19 @@ +{%extends "populations/base.html"%} + +{%block lvl3_breadcrumbs%} +<li {%if activelink=="phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + {%if dataset is mapping%} + <a href="{{url_for('species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">{{dataset.Name}}</a> + {%else%} + <a href="{{url_for('species.populations.phenotypes.index')}}">Phenotypes</a> + {%endif%} +</li> +{%block lvl4_breadcrumbs%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/phenotypes/create-dataset.html b/uploader/templates/phenotypes/create-dataset.html new file mode 100644 index 0000000..8e45491 --- /dev/null +++ b/uploader/templates/phenotypes/create-dataset.html @@ -0,0 +1,108 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="create-dataset"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.create_dataset', + species_id=species.SpeciesId, + population_id=population.Id)}}">Create Datasets</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <p>Create a new phenotype dataset.</p> +</div> + +<div class="row"> + <form id="frm-create-pheno-dataset" + action="{{url_for('species.populations.phenotypes.create_dataset', + species_id=species.SpeciesId, + population_id=population.Id)}}" + method="POST"> + + <div class="form-group"> + <label class="form-label" for="txt-dataset-name">Name</label> + {%if errors["dataset-name"] is defined%} + <small class="form-text text-muted danger"> + <p>{{errors["dataset-name"]}}</p></small> + {%endif%} + <input type="text" + name="dataset-name" + id="txt-dataset-name" + value="{{original_formdata.get('dataset-name') or (population.InbredSetCode + 'Publish')}}" + {%if errors["dataset-name"] is defined%} + class="form-control danger" + {%else%} + class="form-control" + {%endif%} + required="required" /> + <small class="form-text text-muted"> + <p>A short representative name for the dataset.</p> + <p>Recommended: Use the population code and append "Publish" at the end. + <br />This field will only accept names composed of + letters ('A-Za-z'), numbers (0-9), hyphens and underscores.</p> + </small> + </div> + + <div class="form-group"> + <label class="form-label" for="txt-dataset-fullname">Full Name</label> + {%if errors["dataset-fullname"] is defined%} + <small class="form-text text-muted danger"> + <p>{{errors["dataset-fullname"]}}</p></small> + {%endif%} + <input id="txt-dataset-fullname" + name="dataset-fullname" + type="text" + value="{{original_formdata.get('dataset-fullname', '')}}" + {%if errors["dataset-fullname"] is defined%} + class="form-control danger" + {%else%} + class="form-control" + {%endif%} + required="required" /> + <small class="form-text text-muted"> + <p>A longer, descriptive name for the dataset. The name is meant for use + by humans, and therefore, it should be clear what the dataset contains + from the name.</p> + </small> + </div> + + <div class="form-group"> + <label class="form-label" for="txt-dataset-shortname">Short Name</label> + <input id="txt-dataset-shortname" + name="dataset-shortname" + type="text" + class="form-control" + value="{{original_formdata.get('dataset-shortname') or (population.InbredSetCode + ' Publish')}}" /> + <small class="form-text text-muted"> + <p>An optional, short name for the dataset. <br /> + If this is not provided, it will default to the value provided for the + <strong>Name</strong> field above.</p></small> + </div> + + <div class="form-group"> + <input type="submit" + class="btn btn-primary" + value="create phenotype dataset" /> + </div> + + </form> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/phenotypes/edit-phenotype.html b/uploader/templates/phenotypes/edit-phenotype.html new file mode 100644 index 0000000..32c903f --- /dev/null +++ b/uploader/templates/phenotypes/edit-phenotype.html @@ -0,0 +1,332 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="edit-phenotype"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}">View Datasets</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2 class="heading">edit phenotype data</h2> + <p>The forms provided in this page help you update the data for the + phenotypes, and the publication information for the phenotype, + respectively.</p> +</div> + +<div class="row"> + <h3 class="subheading">Basic metadata</h3> + <form name="frm-phenotype-basic-metadata" + class="form-horizontal" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}"> + <input type="hidden" name="phenotype-id" value="{{phenotype.Id}}" /> + <div class="form-group"> + <label for="txt-pre-publication-description" + class="control-label col-sm-2">Pre-Publication Description</label> + <div class="col-sm-10"> + <input type="text" + id="txt-pre-publication-description" + name="pre-publication-description" + class="form-control" + value="{{phenotype['Pre_publication_description'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-pre-publication-abbreviation" + class="control-label col-sm-2">Pre-Publication Abbreviation</label> + <div class="col-sm-10"> + <input type="text" + id="txt-pre-publication-abbreviation" + name="pre-publication-abbreviation" + class="form-control" + value="{{phenotype['Pre_publication_abbreviation'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-post-publication-description" + class="control-label col-sm-2">Post-Publication Description</label> + <div class="col-sm-10"> + <input type="text" + id="txt-post-publication-description" + name="post-publication-description" + class="form-control" + value="{{phenotype['Post_publication_description'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-post-publication-abbreviation" + class="control-label col-sm-2">Post-Publication Abbreviation</label> + <div class="col-sm-10"> + <input type="text" + id="txt-post-publication-abbreviation" + name="post-publication-abbreviation" + class="form-control" + value="{{phenotype['Post_publication_abbreviation'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-original-description" + class="control-label col-sm-2">Original Description</label> + <div class="col-sm-10"> + <input type="text" + id="txt-original-description" + name="original-description" + class="form-control" + value="{{phenotype['Original_description'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-units" + class="control-label col-sm-2">units</label> + <div class="col-sm-10"> + <input type="text" + id="txt-units" + name="units" + class="form-control" + required="required" + value="{{phenotype['Units']}}" /> + </div> + </div> + + <div class="form-group"> + <div class="col-sm-offset-2 col-sm-10"> + <input type="submit" + name="submit" + class="btn btn-primary" + value="update basic metadata"> + </div> + </div> + </form> +</div> + + +<div class="row"> + <h3 class="subheading">phenotype data</h3> + <form id="frm-edit-phenotype-data" + class="form-horizontal" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}"> + <div style="max-height: 23.37em;overflow-y: scroll;"> + <table class="table table-striped table-responsive table-form-table"> + <thead style="position: sticky; top: 0;"> + <tr> + <th>#</th> + <th>Sample</th> + <th>Value</th> + {%if population.Family in families_with_se_and_n%} + <th>Standard-Error</th> + <th>Number of Samples</th> + {%endif%} + </tr> + </thead> + + <tbody> + {%for item in phenotype.data%} + <tr> + <td>{{loop.index}}</td> + <td>{{item.StrainName}}</td> + <td> + <input type="text" + name="value-new::{{item.DataId}}::{{item.StrainId}}" + value="{{item.value}}" + class="form-control" /> + <input type="hidden" + name="value-original::{{item.DataId}}::{{item.StrainId}}" + value="{{item.value}}" /></td> + {%if population.Family in families_with_se_and_n%} + <td> + <input type="text" + name="se-new::{{item.DataId}}::{{item.StrainId}}" + value="{{item.error or ''}}" + data-original-value="{{item.error or ''}}" + class="form-control" /> + <input type="hidden" + name="se-original::{{item.DataId}}::{{item.StrainId}}" + value="{{item.error or ''}}" /></td> + <td> + <input type="text" + name="n-new::{{item.DataId}}::{{item.StrainId}}" + value="{{item.count or ''}}" + data-original-value="{{item.count or "-"}}" + class="form-control" /> + <input type="hidden" + name="n-original::{{item.DataId}}::{{item.StrainId}}" + value="{{item.count or ''}}" /></td> + {%endif%} + </tr> + {%endfor%} + </tbody> + </table> + </div> + <div class="form-group"> + <div class="col-sm-offset-2 col-sm-10"> + <input type="submit" + name="submit" + class="btn btn-primary" + value="update data" /> + </div> + </div> + </form> +</div> + + +<div class="row"> + <h3 class="subheading">publication information</h3> + <p>Use the form below to update the publication information for this + phenotype.</p> + <form id="frm-edit-phenotype-pub-data" + class="form-horizontal" + method="POST" + action="#"> + <div class="form-group"> + <label for="txt-pubmed-id" class="control-label col-sm-2">Pubmed ID</label> + <div class="col-sm-10"> + <input id="txt-pubmed-id" name="pubmed-id" type="text" + class="form-control" /> + <span class="form-text text-muted"> + Enter your publication's PubMed ID.</span> + </div> + </div> + + <div class="form-group"> + <label for="txt-publication-authors" class="control-label col-sm-2">Authors</label> + <div class="col-sm-10"> + <input id="txt-publication-authors" name="publication-authors" + type="text" class="form-control" /> + <span class="form-text text-muted"> + Enter the authors.</span> + </div> + </div> + + <div class="form-group"> + <label for="txt-publication-title" class="control-label col-sm-2"> + Publication Title</label> + <div class="col-sm-10"> + <input id="txt-publication-title" name="publication-title" type="text" + class="form-control" /> + <span class="form-text text-muted"> + Enter your publication's title.</span> + </div> + </div> + + <div class="form-group"> + <label for="txt-publication-abstract" class="control-label col-sm-2"> + Publication Abstract</label> + <div class="col-sm-10"> + <textarea id="txt-publication-abstract" name="publication-abstract" + class="form-control" rows="10"></textarea> + <span class="form-text text-muted"> + Enter the abstract for your publication.</span> + </div> + </div> + + <div class="form-group"> + <label for="txt-publication-journal" class="control-label col-sm-2">Journal</label> + <div class="col-sm-10"> + <input id="txt-publication-journal" name="journal" type="text" + class="form-control" /> + <span class="form-text text-muted"> + Enter the name of the journal where your work was published.</span> + </div> + </div> + + <div class="form-group"> + <label for="txt-publication-volume" class="control-label col-sm-2">Volume</label> + <div class="col-sm-10"> + <input id="txt-publication-volume" name="publication-volume" type="text" + class="form-control" /> + <span class="form-text text-muted"> + Enter the volume in the following format …</span> + </div> + </div> + + <div class="form-group"> + <label for="txt-publication-pages" class="control-label col-sm-2">Pages</label> + <div class="col-sm-10"> + <input id="txt-publication-pages" name="publication-pages" type="text" + class="form-control" /> + <span class="form-text text-muted"> + Enter the journal volume where your work was published.</span> + </div> + </div> + + <div class="form-group"> + <label for="select-publication-month" class="control-label col-sm-2"> + Publication Month</label> + <div class="col-sm-10"> + <select id="select-publication-month" name="publication-month" + class="form-control"> + {%for month in monthnames%} + <option value="{{month | lower}}" + {%if current_month | lower == month | lower%} + selected="selected" + {%endif%}>{{month | capitalize}}</option> + {%endfor%} + </select> + <span class="form-text text-muted"> + Select the month when the work was published. + <span class="text-danger"> + This cannot be before, say 1600 and cannot be in the future!</span></span> + </div> + </div> + + <div class="form-group"> + <label for="txt-publication-year" class="control-label col-sm-2">Publication Year</label> + <div class="col-sm-10"> + <input id="txt-publication-year" name="publication-year" type="text" + class="form-control" value="{{current_year}}" /> + <span class="form-text text-muted"> + Enter the year your work was published. + <span class="text-danger"> + This cannot be before, say 1600 and cannot be in the future!</span> + </span> + </div> + </div> + <div class="form-group"> + <div class="col-sm-offset-2 col-sm-10"> + <input type="submit" + name="submit" + class="btn btn-primary not-implemented" + value="update publication" /> + </div> + </div> + </form> +</div> + +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/phenotypes/index.html b/uploader/templates/phenotypes/index.html new file mode 100644 index 0000000..689c28e --- /dev/null +++ b/uploader/templates/phenotypes/index.html @@ -0,0 +1,21 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-select-species.html" import select_species_form%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + {{select_species_form(url_for("species.populations.phenotypes.index"), species)}} +</div> +{%endblock%} + + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/phenotypes/job-status.html b/uploader/templates/phenotypes/job-status.html new file mode 100644 index 0000000..12963c1 --- /dev/null +++ b/uploader/templates/phenotypes/job-status.html @@ -0,0 +1,155 @@ +{%extends "phenotypes/base.html"%} +{%from "cli-output.html" import cli_output%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} + +{%block extrameta%} +{%if job and job.status not in ("success", "completed:success", "error", "completed:error")%} +<meta http-equiv="refresh" content="5" /> +{%endif%} +{%endblock%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="add-phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">View Datasets</a> +</li> +{%endblock%} + +{%block contents%} + +{%if job%} +<h4 class="subheading">Progress</h4> +<div class="row" style="overflow:scroll;"> + <p><strong>Process Status:</strong> {{job.status}}</p> + {%if metadata%} + <table class="table table-responsive"> + <thead> + <tr> + <th>File</th> + <th>Status</th> + <th>Lines Processed</th> + <th>Total Errors</th> + </tr> + </thead> + + <tbody> + {%for file,meta in metadata.items()%} + <tr> + <td>{{file}}</td> + <td>{{meta.status}}</td> + <td>{{meta.linecount}}</td> + <td>{{meta["total-errors"]}}</td> + </tr> + {%endfor%} + </tbody> + </table> + {%endif%} +</div> + +<div class="row"> + {%if job.status in ("completed:success", "success")%} + <p> + {%if errors | length == 0%} + <a href="{{url_for('species.populations.phenotypes.review_job_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + job_id=job_id)}}" + class="btn btn-primary" + title="Continue to process data">Continue</a> + {%else%} + <span class="text-muted" + disabled="disabled" + style="border: solid 2px;border-radius: 5px;padding: 0.3em;"> + Cannot continue due to errors. Please fix the errors first. + </span> + {%endif%} + </p> + {%endif%} +</div> + +<h4 class="subheading">Errors</h4> +<div class="row" style="max-height: 20em; overflow: scroll;"> + {%if errors | length == 0 %} + <p class="text-info"> + <span class="glyphicon glyphicon-info-sign"></span> + No errors found so far + </p> + {%else%} + <table class="table table-responsive"> + <thead style="position: sticky; top: 0; background: white;"> + <tr> + <th>File</th> + <th>Row</th> + <th>Column</th> + <th>Value</th> + <th>Message</th> + </tr> + </thead> + + <tbody style="font-size: 0.9em;"> + {%for error in errors%} + <tr> + <td>{{error.filename}}</td> + <td>{{error.rowtitle}}</td> + <td>{{error.coltitle}}</td> + <td>{%if error.cellvalue | length > 25%} + {{error.cellvalue[0:24]}}… + {%else%} + {{error.cellvalue}} + {%endif%} + </td> + <td> + {%if error.message | length > 250 %} + {{error.message[0:249]}}… + {%else%} + {{error.message}} + {%endif%} + </td> + </tr> + {%endfor%} + </tbody> + </table> + {%endif%} +</div> + +<div class="row"> + {{cli_output(job, "stdout")}} +</div> + +<div class="row"> + {{cli_output(job, "stderr")}} +</div> + +{%else%} +<div class="row"> + <h3 class="text-danger">No Such Job</h3> + <p>Could not find a job with the ID: {{job_id}}</p> + <p> + Please go back to + <a href="{{url_for('species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + title="'{{dataset.Name}}' dataset page"> + the '{{dataset.Name}}' dataset page</a> + to upload new phenotypes or edit existing ones.</p> +</div> +{%endif%} +{%endblock%} + +{%block sidebarcontents%} +{{display_pheno_dataset_card(species, population, dataset)}} +{%endblock%} diff --git a/uploader/templates/phenotypes/list-datasets.html b/uploader/templates/phenotypes/list-datasets.html new file mode 100644 index 0000000..2cf2c7f --- /dev/null +++ b/uploader/templates/phenotypes/list-datasets.html @@ -0,0 +1,68 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="list-datasets"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.list_datasets', + species_id=species.SpeciesId, + population_id=population.Id)}}">List Datasets</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + {%if datasets | length > 0%} + <p>The dataset(s) available for this population is/are:</p> + + <table class="table"> + <thead> + <tr> + <th>Name</th> + <th>Full Name</th> + <th>Short Name</th> + </tr> + </thead> + + <tbody> + {%for dataset in datasets%} + <tr> + <td><a href="{{url_for('species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">{{dataset.Name}}</a></td> + <td>{{dataset.FullName}}</td> + <td>{{dataset.ShortName}}</td> + </tr> + {%endfor%} + </tbody> + </table> + {%else%} + <p>Phenotypes need to go into a dataset. We do not currently have a dataset + for species <strong>'{{species["FullName"]}} ({{species["Name"]}})'</strong> + phenotypes.</p> + + <p>Do, please, create a new dataset by clicking on the "Create Dataset" button + below and following the prompts/instructions.</p> + <p><a href="{{url_for('species.populations.phenotypes.create_dataset', + species_id=species.SpeciesId, + population_id=population.Id)}}" + class="btn btn-primary" + title="Create a new phenotype dataset.">create dataset</a></p> + {%endif%} +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html new file mode 100644 index 0000000..11b108b --- /dev/null +++ b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html @@ -0,0 +1,31 @@ +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%macro display_pheno_dataset_card(species, population, dataset)%} +{{display_population_card(species, population)}} + +<div class="card"> + <div class="card-body"> + <h5 class="card-title">Phenotypes' Dataset</h5> + <div class="card-text"> + <table class="table"> + <tbody> + <tr> + <td>Name</td> + <td>{{dataset.Name}}</td> + </tr> + + <tr> + <td>Full Name</td> + <td>{{dataset.FullName}}</td> + </tr> + + <tr> + <td>Short Name</td> + <td>{{dataset.ShortName}}</td> + </tr> + </tbody> + </table> + </div> + </div> +</div> +{%endmacro%} diff --git a/uploader/templates/phenotypes/macro-display-preview-table.html b/uploader/templates/phenotypes/macro-display-preview-table.html new file mode 100644 index 0000000..f54c53e --- /dev/null +++ b/uploader/templates/phenotypes/macro-display-preview-table.html @@ -0,0 +1,21 @@ +{%macro display_preview_table(tableid, filetype)%} +<div class="card" style="max-width: 676px;"> + <div class="card-body"> + <h5 class="card-title">Phenotypes '{{filetype | title}}' File Preview</h5> + <div class="card-text" style="overflow: scroll;"> + <table id="{{tableid}}" class="table table-condensed table-responsive"> + <thead> + <tr> + </tr> + <tbody> + <tr> + <td class="data-row-template text-info"> + Provide a phenotype '{{filetype | lower}}' file to preview. + </td> + </tr> + </tbody> + </table> + </div> + </div> +</div> +{%endmacro%} diff --git a/uploader/templates/phenotypes/macro-display-resumable-elements.html b/uploader/templates/phenotypes/macro-display-resumable-elements.html new file mode 100644 index 0000000..b0bf1b5 --- /dev/null +++ b/uploader/templates/phenotypes/macro-display-resumable-elements.html @@ -0,0 +1,60 @@ +{%macro display_resumable_elements(id, title, help)%} +<div id="{{id}}" + class="resumable-elements hidden" + style="background:#D4D4EE;border-radius: 5px;;padding: 1em;border-left: solid #B2B2CC 1px;border-bottom: solid #B2B2CC 2px;margin-top:0.3em;"> + <strong style="line-height: 1.2em;">{{title | title}}</strong> + + <span class="form-text text-muted">{{help | safe}}</span> + + <div id="{{id}}-selected-files" + class="resumable-selected-files" + style="display:flex;flex-direction:row;flex-wrap: wrap;justify-content:space-around;gap:10px 20px;"> + <div class="panel panel-info file-display-template hidden"> + <div class="panel-heading filename">The Filename Goes Here!</div> + <div class="panel-body"> + <ul> + <li> + <strong>Name</strong>: + <span class="filename">the file's name</span></li> + + <li><strong>Size</strong>: <span class="filesize">0 MB</span></li> + + <li> + <strong>Unique Identifier</strong>: + <span class="fileuniqueid">brrr</span></li> + + <li> + <strong>Mime</strong>: + <span class="filemimetype">text/csv</span></li> + </ul> + </div> + </div> + </div> + + <a id="{{id}}-browse-button" + class="resumable-browse-button btn btn-info" + href="#" + style="margin-left: 80%;">Browse</a> + + <div id="{{id}}-progress-bar" class="progress hidden"> + <div class="progress-bar" + role="progress-bar" + aria-valuenow="60" + aria-valuemin="0" + aria-valuemax="100" + style="width: 0%;"> + Uploading: 60% + </div> + </div> + + <div id="{{id}}-cancel-resume-buttons"> + <a id="{{id}}-resume-button" + class="resumable-resume-button btn btn-info hidden" + href="#">resume upload</a> + + <a id="{{id}}-cancel-button" + class="resumable-cancel-button btn btn-danger hidden" + href="#">cancel upload</a> + </div> +</div> +{%endmacro%} diff --git a/uploader/templates/phenotypes/review-job-data.html b/uploader/templates/phenotypes/review-job-data.html new file mode 100644 index 0000000..7bc8c62 --- /dev/null +++ b/uploader/templates/phenotypes/review-job-data.html @@ -0,0 +1,101 @@ +{%extends "phenotypes/base.html"%} +{%from "cli-output.html" import cli_output%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} + +{%block extrameta%} +{%if not job%} +<meta http-equiv="refresh" + content="20; url={{url_for('species.populations.phenotypes.view_dataset', species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" /> +{%endif%} +{%endblock%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="add-phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">View Datasets</a> +</li> +{%endblock%} + +{%block contents%} + +{%if job%} +<div class="row"> + <h3 class="heading">Data Review</h3> + <p>The “<strong>{{dataset.FullName}}</strong>” dataset from the + “<strong>{{population.FullName}}</strong>” population of the + species “<strong>{{species.SpeciesName}} ({{species.FullName}})</strong>” + will be updated as follows:</p> + + {%for ftype in ("phenocovar", "pheno", "phenose", "phenonum")%} + {%if summary.get(ftype, False)%} + <ul> + <li>A total of {{summary[ftype]["number-of-files"]}} files will be processed + adding {%if ftype == "phenocovar"%}(possibly){%endif%} + {{summary[ftype]["total-data-rows"]}} new + {%if ftype == "phenocovar"%} + phenotypes + {%else%} + {{summary[ftype]["description"]}} rows + {%endif%} + to the database. + </li> + </ul> + {%endif%} + {%endfor%} + + <a href="#" class="not-implemented btn btn-primary">continue</a> +</div> +{%else%} +<div class="row"> + <h4 class="subheading">Invalid Job</h3> + <p class="text-danger"> + Could not find a job with the ID: <strong>{{job_id}}.</p> + <p>You will be redirected in + <span id="countdown-element" class="text-info">20</span> second(s)</p> + <p class="text-muted"> + <small> + If you are not redirected, please + <a href="{{url_for( + 'species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">click here</a> to continue + </small> + </p> +</div> +{%endif%} +{%endblock%} + +{%block sidebarcontents%} +{{display_pheno_dataset_card(species, population, dataset)}} +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(document).ready(function() { + var countdown = 20; + var countdown_element = $("#countdown-element"); + if(countdown_element.length === 1) { + intv = window.setInterval(function() { + countdown = countdown - 1; + countdown_element.html(countdown); + }, 1000); + } + }); +</script> +{%endblock%} diff --git a/uploader/templates/phenotypes/select-population.html b/uploader/templates/phenotypes/select-population.html new file mode 100644 index 0000000..cea0806 --- /dev/null +++ b/uploader/templates/phenotypes/select-population.html @@ -0,0 +1,31 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-display-species-card.html" import display_species_card%} +{%from "populations/macro-select-population.html" import select_population_form%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + + +<div class="row"> + {{select_population_form(url_for("species.populations.phenotypes.select_population", species_id=species.SpeciesId), species, populations)}} +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> +<script type="text/javascript"> + $(function() { + populationDataTable(JSON.parse($("#tbl-select-population").attr("data-populations-list"))); + }); + </script> +{%endblock%} diff --git a/uploader/templates/phenotypes/view-dataset.html b/uploader/templates/phenotypes/view-dataset.html new file mode 100644 index 0000000..c896214 --- /dev/null +++ b/uploader/templates/phenotypes/view-dataset.html @@ -0,0 +1,123 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="view-dataset"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">View</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <p>The basic dataset details are:</p> + + <table class="table"> + <thead> + <tr> + <th>Name</th> + <th>Full Name</th> + <th>Short Name</th> + </tr> + </thead> + + <tbody> + <tr> + <td>{{dataset.Name}}</td> + <td>{{dataset.FullName}}</td> + <td>{{dataset.ShortName}}</td> + </tr> + </tbody> + </table> +</div> + +<div class="row"> + <p><a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + title="Add a bunch of phenotypes" + class="btn btn-primary">Add phenotypes</a></p> +</div> + +<div class="row"> + <h2>Phenotype Data</h2> + + <p>Click on any of the phenotypes in the table below to view and edit that + phenotype's data.</p> + <p>Use the search to filter through all the phenotypes and find specific + phenotypes of interest.</p> + + <table id="tbl-phenotypes-list" class="table compact stripe"> + <thead> + <tr> + <th></th> + <th>Record</th> + <th>Description</th> + </tr> + </thead> + + <tbody></tbody> + </table> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(function() { + var data = {{phenotypes | tojson}}; + $("#tbl-phenotypes-list").DataTable({ + responsive: true, + lengthMenu: [10, 25, 50, 100, 1000, data.length], + language: { + processing: "Processing results… Please wait.", + loadingRecord: "Loading phenotypes — Please wait.", + info: "_START_ to _END_ of _TOTAL_ phenotypes", + lengthMenu: "Show _MENU_ phenotypes", + }, + data: data, + columns: [ + {data: "sequence_number"}, + { + data: function(pheno) { + var spcs_id = {{species.SpeciesId}}; + var pop_id = {{population.Id}}; + var dtst_id = {{dataset.Id}}; + return `<a href="/species/${spcs_id}` + + `/populations/${pop_id}` + + `/phenotypes/datasets/${dtst_id}` + + `/phenotype/${pheno.xref_id}` + + `" target="_blank">` + + `${pheno.InbredSetCode}_${pheno.xref_id}` + + `</a>`; + } + }, + {data: function(pheno) { + return (pheno.Post_publication_description || + pheno.Original_description || + pheno.Pre_publication_description); + }} + ] + }); + }); +</script> +{%endblock%} diff --git a/uploader/templates/phenotypes/view-phenotype.html b/uploader/templates/phenotypes/view-phenotype.html new file mode 100644 index 0000000..21ac501 --- /dev/null +++ b/uploader/templates/phenotypes/view-phenotype.html @@ -0,0 +1,135 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="view-phenotype"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.view_phenotype', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}">View Phenotype</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <div class="panel panel-default"> + <div class="panel-heading"><strong>Basic Phenotype Details</strong></div> + + <table class="table"> + <tbody> + <tr> + <td><strong>Phenotype</strong></td> + <td>{{phenotype.Post_publication_description or phenotype.Pre_publication_abbreviation or phenotype.Original_description}} + </tr> + <tr> + <td><strong>Database</strong></td> + <td>{{dataset.FullName}}</td> + </tr> + <tr> + <td><strong>Units</strong></td> + <td>{{phenotype.Units}}</td> + </tr> + {%for key,value in publish_data.items()%} + <tr> + <td><strong>{{key}}</strong></td> + <td>{{value}}</td> + </tr> + {%else%} + <tr> + <td colspan="2" class="text-muted"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + No publication data found. + </td> + </tr> + {%endfor%} + </tbody> + </table> + </div> +</div> + +{%if "group:resource:edit-resource" in privileges +or "group:resource:delete-resource" in privileges%} +<div class="row"> + <div class="btn-group btn-group-justified"> + <div class="btn-group"> + {%if "group:resource:edit-resource" in privileges%} + <a href="{{url_for('species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}" + title="Edit the values for the phenotype. This is meant to be used when you need to update only a few values." + class="btn btn-primary">Edit</a> + {%endif%} + </div> + <div class="btn-group"></div> + <div class="btn-group"> + {%if "group:resource:delete-resource" in privileges%} + <a href="#" + title="Delete the entire phenotype. This is useful when you need to change data for most or all of the fields for this phenotype." + class="btn btn-danger not-implemented" + disabled="disabled">delete</a> + {%endif%} + </div> + </div> +</div> +{%endif%} + +<div class="row"> + <div class="panel panel-default"> + <div class="panel-heading"><strong>Phenotype Data</strong></div> + {%if "group:resource:view-resource" in privileges%} + <table class="table"> + <thead> + <tr> + <th>#</th> + <th>Sample</th> + <th>Value</th> + {%if has_se%} + <th>SE</th> + <th>N</th> + {%endif%} + </tr> + </thead> + + <tbody> + {%for item in phenotype.data%} + <tr> + <td>{{loop.index}}</td> + <td>{{item.StrainName}}</td> + <td>{{item.value}}</td> + {%if has_se%} + <td>{{item.error or "-"}}</td> + <td>{{item.count or "-"}}</td> + {%endif%} + </tr> + {%endfor%} + </tbody> + </table> + {%else%} + <p class="text-danger"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + You do not currently have privileges to view this phenotype in greater + detail. + </p> + {%endif%} + </div> +</div> + +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/platforms/base.html b/uploader/templates/platforms/base.html new file mode 100644 index 0000000..dac965f --- /dev/null +++ b/uploader/templates/platforms/base.html @@ -0,0 +1,13 @@ +{%extends "species/base.html"%} + +{%block lvl3_breadcrumbs%} +<li {%if activelink=="platforms"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.platforms.index')}}"> + Sequencing Platforms</a> +</li> +{%block lvl4_breadcrumbs%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/platforms/create-platform.html b/uploader/templates/platforms/create-platform.html new file mode 100644 index 0000000..0866d5e --- /dev/null +++ b/uploader/templates/platforms/create-platform.html @@ -0,0 +1,124 @@ +{%extends "platforms/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-display-species-card.html" import display_species_card%} + +{%block title%}Platforms — Create Platforms{%endblock%} + +{%block pagetitle%}Platforms — Create Platforms{%endblock%} + +{%block lvl3_breadcrumbs%} +<li {%if activelink=="create-platform"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.platforms.create_platform', + species_id=species.SpeciesId)}}">create platform</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2>Create New Platform</h2> + + <p>You can create a new genetic sequencing platform below.</p> +</div> + +<div class="row"> + <form id="frm-create-platform" + method="POST" + action="{{url_for('species.platforms.create_platform', + species_id=species.SpeciesId)}}"> + + <div class="form-group"> + <label for="txt-geo-platform" class="form-label">GEO Platform</label> + <input type="text" + id="txt-geo-platform" + name="geo-platform" + required="required" + class="form-control" /> + <small class="form-text text-muted"> + <p>This is the platform's + <a href="https://www.ncbi.nlm.nih.gov/geo/browse/?view=platforms&tax={{species.TaxonomyId}}" + title="Platforms for '{{species.FullName}}' on NCBI"> + accession value on NCBI</a>. If you do not know the value, click the + link and search on NCBI for species '{{species.FullName}}'.</p></small> + </div> + + <div class="form-group"> + <label for="txt-platform-name" class="form-label">Platform Name</label> + <input type="text" + id="txt-platform-name" + name="platform-name" + required="required" + class="form-control" /> + <small class="form-text text-muted"> + <p>This is name of the genetic sequencing platform.</p></small> + </div> + + <div class="form-group"> + <label for="txt-platform-shortname" class="form-label"> + Platform Short Name</label> + <input type="text" + id="txt-platform-shortname" + name="platform-shortname" + required="required" + class="form-control" /> + <small class="form-text text-muted"> + <p>Use the following conventions for this field: + <ol> + <li>Start with a 4-letter vendor code, e.g. "Affy" for "Affymetrix", "Illu" for "Illumina", etc.</li> + <li>Append an underscore to the 4-letter vendor code</li> + <li>Use the name of the array given by the vendor, e.g. U74AV2, MOE430A, etc.</li> + </ol> + </p> + </small> + </div> + + <div class="form-group"> + <label for="txt-platform-title" class="form-label">Platform Title</label> + <input type="text" + id="txt-platform-title" + name="platform-title" + required="required" + class="form-control" /> + <small class="form-text text-muted"> + <p>The full platform title. Sometimes, this is the same as the Platform + Name above.</p></small> + </div> + + <div class="form-group"> + <label for="txt-go-tree-value" class="form-label">GO Tree Value</label> + <input type="text" + id="txt-go-tree-value" + name="go-tree-value" + class="form-control" /> + <small class="form-text text-muted"> + <p>This is a Chip identification value useful for analysis with the + <strong> + <a href="https://www.geneweaver.org/" + title="Go to the GeneWeaver site." + target="_blank">GeneWeaver</a></strong> + and + <strong> + <a href="https://www.webgestalt.org/" + title="Go to the WEB-based GEne SeT AnaLysis Toolkit site." + target="_blank">WebGestalt</a></strong> + tools.<br /> + This can be left blank for custom platforms.</p></small> + </div> + + <div class="form-group"> + <input type="submit" + value="create new platform" + class="btn btn-primary" /> + </div> + </form> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} diff --git a/uploader/templates/platforms/index.html b/uploader/templates/platforms/index.html new file mode 100644 index 0000000..555b444 --- /dev/null +++ b/uploader/templates/platforms/index.html @@ -0,0 +1,25 @@ +{%extends "platforms/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-select-species.html" import select_species_form%} + +{%block title%}Platforms{%endblock%} + +{%block pagetitle%}Platforms{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <p>In this section, you will be able to view and manage the sequencing + platforms that are currently supported by GeneNetwork.</p> +</div> + +<div class="row"> + {{select_species_form(url_for("species.platforms.index"), species)}} +</div> +{%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/platforms/list-platforms.html b/uploader/templates/platforms/list-platforms.html new file mode 100644 index 0000000..a6bcfdc --- /dev/null +++ b/uploader/templates/platforms/list-platforms.html @@ -0,0 +1,93 @@ +{%extends "platforms/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-display-species-card.html" import display_species_card%} + +{%block title%}Platforms — List Platforms{%endblock%} + +{%block pagetitle%}Platforms — List Platforms{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <p>View the list of the genetic sequencing platforms that are currently + supported by GeneNetwork.</p> + <p>If you cannot find the platform you wish to use, you can add it by clicking + the "New Platform" button below.</p> + <p><a href="{{url_for('species.platforms.create_platform', + species_id=species.SpeciesId)}}" + title="Create a new genetic sequencing platform for species {{species.FullName}}" + class="btn btn-primary">Create Platform</a></p> +</div> + +<div class="row"> + <h2>Supported Platforms</h2> + {%if platforms is defined and platforms | length > 0%} + <p>There are {{total_platforms}} platforms supported by GeneNetwork</p> + + <div class="row"> + <div class="col-md-2" style="text-align: start;"> + {%if start_from > 0%} + <a href="{{url_for('species.platforms.list_platforms', + species_id=species.SpeciesId, + start_from=start_from-count, + count=count)}}"> + <span class="glyphicon glyphicon-backward"></span> + Previous + </a> + {%endif%} + </div> + <div class="col-md-8" style="text-align: center;"> + Displaying platforms {{start_from+1}} to {{start_from+count if start_from+count < total_platforms else total_platforms}} of + {{total_platforms}} + </div> + <div class="col-md-2" style="text-align: end;"> + {%if start_from + count < total_platforms%} + <a href="{{url_for('species.platforms.list_platforms', + species_id=species.SpeciesId, + start_from=start_from+count, + count=count)}}"> + Next + <span class="glyphicon glyphicon-forward"></span> + </a> + {%endif%} + </div> + </div> + + <table class="table"> + <thead> + <tr> + <th></th> + <th>Platform Name</th> + <th><a href="https://www.ncbi.nlm.nih.gov/geo/browse/?view=platforms&tax={{species.TaxonomyId}}" + title="Gene Expression Omnibus: Platforms section" + target="_blank">GEO Platform</a></th> + <th>Title</th> + </tr> + </thead> + + <tbody> + {%for platform in platforms%} + <tr> + <td>{{platform.sequence_number}}</td> + <td>{{platform.GeneChipName}}</td> + <td><a href="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc={{platform.GeoPlatform}}" + title="View platform on the Gene Expression Omnibus" + target="_blank">{{platform.GeoPlatform}}</a></td> + <td>{{platform.Title}}</td> + </tr> + {%endfor%} + </tbody> + </table> + {%else%} + <p class="text-warning"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + There are no platforms supported at this time!</p> + {%endif%} +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} diff --git a/uploader/templates/populations/base.html b/uploader/templates/populations/base.html new file mode 100644 index 0000000..9db8083 --- /dev/null +++ b/uploader/templates/populations/base.html @@ -0,0 +1,18 @@ +{%extends "species/base.html"%} + +{%block lvl2_breadcrumbs%} +<li {%if activelink=="populations"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + {%if population is mapping%} + <a href="{{url_for('species.populations.view_population', + species_id=species.SpeciesId, + population_id=population.Id)}}">{{population.Name}}</a> + {%else%} + <a href="{{url_for('species.populations.index')}}">Populations</a> + {%endif%} +</li> +{%block lvl3_breadcrumbs%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/populations/create-population.html b/uploader/templates/populations/create-population.html new file mode 100644 index 0000000..c0c4f45 --- /dev/null +++ b/uploader/templates/populations/create-population.html @@ -0,0 +1,258 @@ +{%extends "populations/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-select-species.html" import select_species_form%} +{%from "species/macro-display-species-card.html" import display_species_card%} + +{%block title%}Create Population{%endblock%} + +{%block pagetitle%}Create Population{%endblock%} + +{%block lvl3_breadcrumbs%} +<li {%if activelink=="create-population"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.create_population', + species_id=species.SpeciesId)}}">create population</a> +</li> +{%endblock%} + + +{%block contents%} +<div class="row"> + <p>The population is the next hierarchical node under Species. Data is grouped under a specific population, under a particular species.</p> + <p> + This page enables you to create a new population, in the case that you + cannot find the population you want in the + <a + href="{{url_for('species.populations.list_species_populations', + species_id=species.SpeciesId)}}" + title="Population for species '{{species.FullName}}'."> + list of species populations + </a> + </p> +</div> + +<div class="row"> + <form method="POST" + action="{{url_for('species.populations.create_population', + species_id=species.SpeciesId, + return_to=return_to)}}"> + + <legend>Create Population</legend> + + {{flash_all_messages()}} + + <input type="hidden" name="return_to" value="{{return_to}}"> + + <div {%if errors.population_fullname%} + class="form-group has-error" + {%else%} + class="form-group" + {%endif%}> + <label for="txt-population-fullname" class="form-label">Full Name</label> + {%if errors.population_fullname%} + <small class="form-text text-danger">{{errors.population_fullname}}</small> + {%endif%} + <input type="text" + id="txt-population-fullname" + name="population_fullname" + required="required" + minLength="3" + maxLength="100" + value="{{error_values.population_fullname or ''}}" + class="form-control" /> + <small class="form-text text-muted"> + <p> + This is a descriptive name for your population — useful for + humans. + </p> + </small> + </div> + + <div {%if errors.population_name%} + class="form-group has-error" + {%else%} + class="form-group" + {%endif%}> + <label for="txt-population-name" class="form-label">Name</label> + {%if errors.population_name%} + <small class="form-text text-danger">{{errors.population_name}}</small> + {%endif%} + <input type="text" + id="txt-population-name" + name="population_name" + required="required" + minLength="3" + maxLength="30" + value="{{error_values.population_name or ''}}" + class="form-control" /> + <small class="form-text text-muted"> + <p> + This is a short representative, but constrained name for your + population. + <br /> + The field will only accept letters ('A-Za-z'), numbers (0-9), hyphens + and underscores. Any other character will cause the name to be + rejected. + </p> + </small> + </div> + + <div class="form-group"> + <label for="txt-population-code" class="form-label">Population Code</label> + <input type="text" + id="txt-population-code" + name="population_code" + maxLength="5" + minLength="3" + value="{{error_values.population_code or ''}}" + class="form-control" /> + <small class="form-text text-muted"> + <p class="form-text text-muted"> + This is a 3-character code for your population, that is prepended to + the phenotype identifiers. e.g. For the "BXD Family" population, the + code is "BXD" and therefore, the phenotype identifiers for the + population look like the following examples: <em>BXD_10148</em>, + <em>BXD_10180</em>, <em>BXD_10197</em>, etc. + </p> + </small> + </div> + + <div {%if errors.population_description%} + class="form-group has-error" + {%else%} + class="form-group" + {%endif%}> + <label for="txt-population-description" class="form-label"> + Description + </label> + {%if errors.population_description%} + <small class="form-text text-danger">{{errors.population_description}}</small> + {%endif%} + <textarea + id="txt-population-description" + name="population_description" + required="required" + class="form-control" + rows="5">{{error_values.population_description or ''}}</textarea> + <small class="form-text text-muted"> + <p> + This is a more detailed description for your population. This is + useful to communicate with other researchers some details regarding + your population, and what its about. + <br /> + Put, here, anything that describes your population but does not go + cleanly under metadata. + </p> + </small> + </div> + + <div {%if errors.population_family%} + class="form-group has-error" + {%else%} + class="form-group" + {%endif%}> + <label for="select-population-family" class="form-label">Family</label> + <select id="select-population-family" + name="population_family" + class="form-control" + required="required"> + <option value="">Please select a family</option> + {%for family in families%} + <option value="{{family}}" + {%if error_values.population_family == family%} + selected="selected" + {%endif%}>{{family}}</option> + {%endfor%} + </select> + <small class="form-text text-muted"> + <p> + This is a rough grouping of the populations in GeneNetwork into lists + of common types of populations. + </p> + </small> + </div> + + <div {%if errors.population_mapping_method_id%} + class="form-group has-error" + {%else%} + class="form-group" + {%endif%}> + <label for="select-population-mapping-methods" + class="form-label">Mapping Methods</label> + + <select id="select-population-mapping-methods" + name="population_mapping_method_id" + class="form-control" + required="required"> + <option value="">Select appropriate mapping methods</option> + {%for mmethod in mapping_methods%} + <option value="{{mmethod.id}}" + {%if error_values.population_mapping_method_id == mmethod.id%} + selected="selected" + {%endif%}>{{mmethod.value}}</option> + {%endfor%} + </select> + + <small class="form-text text-muted"> + <p>Select the mapping methods that your population will support.</p> + </small> + </div> + + <div {%if errors.population_genetic_type%} + class="form-group has-error" + {%else%} + class="form-group" + {%endif%}> + <label for="select-population-genetic-type" + class="form-label">Genetic Type</label> + <select id="select-population-genetic-type" + name="population_genetic_type" + class="form-control"> + <option value="">Select proper genetic type</option> + {%for gtype in genetic_types%} + <option value="{{gtype}}" + {%if error_values.population_genetic_type == gtype%} + selected="selected" + {%endif%}>{{gtype}}</option> + {%endfor%} + </select> + <small class="form-text text-muted text-danger"> + <p> + <span class="glyphicon glyphicon-exclamation-sign"></span> + This might be a poorly named field. + </p> + <p> + It probably has more to do with the mating crosses/crossings used to + produce the individuals in the population. I am no biologist, however, + and I'm leaving this here to remind myself to confirm this. + </p> + <p> + I still don't know what riset is.<br /> + … probably something to do with Recombinant Inbred Strains + </p> + <p> + Possible resources for this: + <ul> + <li>https://www.informatics.jax.org/silver/chapters/3-2.shtml</li> + <li>https://www.informatics.jax.org/silver/chapters/9-2.shtml</li> + </ul> + </p> + </small> + </div> + + <div class="form-group"> + <input type="submit" + value="create population" + class="btn btn-primary" /> + </div> + + </form> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} diff --git a/uploader/templates/populations/index.html b/uploader/templates/populations/index.html new file mode 100644 index 0000000..d2bee77 --- /dev/null +++ b/uploader/templates/populations/index.html @@ -0,0 +1,28 @@ +{%extends "populations/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-select-species.html" import select_species_form%} + +{%block title%}Populations{%endblock%} + +{%block pagetitle%}Populations{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <p> + Your experiment data will relate to a particular population from a + particular species. Let us know what species it is you want to work with + below. + </p> +</div> + +<div class="row"> + {{select_species_form(url_for("species.populations.index"), species)}} +</div> +{%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/populations/list-populations.html b/uploader/templates/populations/list-populations.html new file mode 100644 index 0000000..f780e94 --- /dev/null +++ b/uploader/templates/populations/list-populations.html @@ -0,0 +1,93 @@ +{%extends "populations/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-select-species.html" import select_species_form%} +{%from "species/macro-display-species-card.html" import display_species_card%} + +{%block title%}Populations{%endblock%} + +{%block pagetitle%}Populations{%endblock%} + +{%block lvl3_breadcrumbs%} +<li {%if activelink=="list-populations"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.list_species_populations', + species_id=species.SpeciesId)}}">List populations</a> +</li> +{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} +<div class="row"> + <p> + The following populations/groups exist for the '{{species.FullName}}' + species. + </p> + <p> + Click on the population's name to select and continue using the population. + </p> +</div> + +<div class="row"> + <p> + If the population you need for the species '{{species.FullName}}' does not + exist, click on the "Create Population" button below to create a new one. + </p> + <p> + <a href="{{url_for('species.populations.create_population', + species_id=species.SpeciesId)}}" + title="Create a new population for species '{{species.FullName}}'." + class="btn btn-danger"> + Create Population + </a> + </p> +</div> + +<div class="row"> + <table class="table"> + <caption>Populations for {{species.FullName}}</caption> + <thead> + <tr> + <th></th> + <th>Name</th> + <th>Full Name</th> + <th>Description</th> + </tr> + </thead> + + <tbody> + {%for population in populations%} + <tr> + <td>{{population["sequence_number"]}}</td> + <td> + <a href="{{url_for('species.populations.view_population', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + title="Population '{{population.FullName}}' for species '{{species.FullName}}'."> + {{population.Name}} + </a> + </td> + <td>{{population.FullName}}</td> + <td>{{population.Description}}</td> + </tr> + {%else%} + <tr> + <td colspan="3"> + <p class="text-danger"> + <span class="glyphicon glyphicon-exclamation-mark"></span> + There were no populations found for {{species.FullName}}! + </p> + </td> + </tr> + {%endfor%} + </tbody> + </table> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} diff --git a/uploader/templates/populations/macro-display-population-card.html b/uploader/templates/populations/macro-display-population-card.html new file mode 100644 index 0000000..16b477f --- /dev/null +++ b/uploader/templates/populations/macro-display-population-card.html @@ -0,0 +1,41 @@ +{%from "species/macro-display-species-card.html" import display_species_card%} + +{%macro display_population_card(species, population)%} +{{display_species_card(species)}} + +<div class="card"> + <div class="card-body"> + <h5 class="card-title">Population</h5> + <div class="card-text"> + <table class="table"> + <tbody> + <tr> + <td>Name</td> + <td>{{population.Name}}</td> + </tr> + + <tr> + <td>Full Name</td> + <td>{{population.FullName}}</td> + </tr> + + <tr> + <td>Code</td> + <td>{{population.InbredSetCode}}</td> + </tr> + + <tr> + <td>Genetic Type</td> + <td>{{population.GeneticType}}</td> + </tr> + + <tr> + <td>Family</td> + <td>{{population.Family}}</td> + </tr> + </tbody> + </table> + </div> + </div> +</div> +{%endmacro%} diff --git a/uploader/templates/populations/macro-select-population.html b/uploader/templates/populations/macro-select-population.html new file mode 100644 index 0000000..14b0510 --- /dev/null +++ b/uploader/templates/populations/macro-select-population.html @@ -0,0 +1,52 @@ +{%from "macro-step-indicator.html" import step_indicator%} + +{%macro select_population_form(form_action, species, populations)%} +<form method="GET" action="{{form_action}}" class="form-horizontal"> + + <h2>{{step_indicator("2")}} What population do you want to work with?</h2> + + {%if populations | length != 0%} + + <p class="form-text">Search for, and select the population from the table + below and click "Continue"</p> + + <div class="radio"> + <label class="control-label" for="rdo-cant-find-population"> + <input type="radio" id="rdo-cant-find-population" + name="population_id" value="CREATE-POPULATION" /> + I cannot find the population I want — create it! + </label> + </div> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" value="continue" class="btn btn-primary" /> + </div> + + <div style="margin-top:3em;"> + <table id="tbl-select-population" class="table compact stripe" + data-populations-list='{{populations | tojson}}'> + <thead> + <tr> + <th></th> + <th>Population</th> + </tr> + </thead> + + <tbody></tbody> + </table> + </div> + + {%else%} + <p class="form-text"> + There are no populations currently defined for {{species['FullName']}} + ({{species['SpeciesName']}}).<br /> + Click "Continue" to create the first!</p> + <input type="hidden" name="population_id" value="CREATE-POPULATION" /> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" value="continue" class="btn btn-primary" /> + </div> + {%endif%} + +</form> +{%endmacro%} diff --git a/uploader/templates/populations/rqtl2/create-tissue-success.html b/uploader/templates/populations/rqtl2/create-tissue-success.html new file mode 100644 index 0000000..d6fe154 --- /dev/null +++ b/uploader/templates/populations/rqtl2/create-tissue-success.html @@ -0,0 +1,106 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Select Tissue</h2> + +<div class="row"> + <p>You have successfully added a new tissue, organ or biological material with + the following details:</p> +</div> + +<div class="row"> + {{flash_all_messages()}} + + <form id="frm-create-tissue-display" + method="POST" + action="#"> + <legend class="heading">Create Tissue</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + + <div class="form-group"> + <label>Name</label> + <label>{{tissue.TissueName}}</label> + </div> + + <div class="form-group"> + <label>Short Name</label> + <label>{{tissue.Short_Name}}</label> + </div> + + {%if tissue.BIRN_lex_ID%} + <div class="form-group"> + <label>BIRN Lex ID</label> + <label>{{tissue.BIRN_lex_ID}}</label> + </div> + {%endif%} + + {%if tissue.BIRN_lex_Name%} + <div class="form-group"> + <label>BIRN Lex Name</label> + <label>{{tissue.BIRN_lex_Name}}</label> + </div> + {%endif%} + </form> + + <div id="action-buttons" + style="width:65ch;display:inline-grid;column-gap:5px;"> + + <form id="frm-create-tissue-success-continue" + method="POST" + action="{{url_for('expression-data.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + style="display: inline; width: 100%; grid-column: 1 / 2; + padding-top: 0.5em; text-align: center; border: none; + background-color: inherit;"> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + + <button type="submit" class="btn btn-primary">continue</button> + </form> + </div> +</div> + +<div class="row"> + <p style="display:inline;width:100%;grid-column:2/3;text-align:center; + color:#336699;font-weight:bold;"> + OR + </p> +</div> + +<div class="row"> + <form id="frm-create-tissue-success-select-existing" + method="POST" + action="{{url_for('expression-data.rqtl2.select_tissue', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + style="display: inline; width: 100%; grid-column: 3 / 4; + padding-top: 0.5em; text-align: center; border: none; + background-color: inherit;"> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + + <button type="submit" class="btn btn-primary"> + select from existing tissues</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/index.html b/uploader/templates/populations/rqtl2/index.html new file mode 100644 index 0000000..ec6ffb8 --- /dev/null +++ b/uploader/templates/populations/rqtl2/index.html @@ -0,0 +1,54 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Data Upload{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 data upload</h1> + +<h2>R/qtl2 Upload</h2> + +<div class="row"> + <form method="POST" action="{{url_for('expression-data.rqtl2.select_species')}}" + id="frm-rqtl2-upload"> + <legend class="heading">upload R/qtl2 bundle</legend> + {{flash_messages("error-rqtl2")}} + + <div class="form-group"> + <label for="select:species" class="form-label">Species</label> + <select id="select:species" + name="species_id" + required="required" + class="form-control"> + <option value="">Select species</option> + {%for spec in species%} + <option value="{{spec.SpeciesId}}">{{spec.MenuName}}</option> + {%endfor%} + </select> + <small class="form-text text-muted"> + Data that you upload to the system should belong to a know species. + Here you can select the species that you wish to upload data for. + </small> + </div> + + <input type="submit" class="btn btn-primary" value="submit" /> + </form> +</div> + +<div class="row"> + <h2 class="heading">R/qtl2 Bundles</h2> + + <div class="explainer"> + <p>This feature combines and extends the two upload methods below. Instead of + uploading one item at a time, the R/qtl2 bundle you upload can contain both + the genotypes data (samples/individuals/cases and their data) and the + expression data.</p> + <p>The R/qtl2 bundle, additionally, can contain extra metadata, that neither + of the methods below can handle.</p> + + <a href="{{url_for('expression-data.rqtl2.select_species')}}" + title="Upload a zip bundle of R/qtl2 files"> + <button class="btn btn-primary">upload R/qtl2 bundle</button></a> + </div> +</div> +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/no-such-job.html b/uploader/templates/populations/rqtl2/no-such-job.html new file mode 100644 index 0000000..b17004f --- /dev/null +++ b/uploader/templates/populations/rqtl2/no-such-job.html @@ -0,0 +1,13 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 job status</h1> + +<h2>R/qtl2 Upload: No Such Job</h2> + +<p class="alert-danger">No job with ID {{jobid}} was found.</p> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/rqtl2-job-error.html b/uploader/templates/populations/rqtl2/rqtl2-job-error.html new file mode 100644 index 0000000..9817518 --- /dev/null +++ b/uploader/templates/populations/rqtl2/rqtl2-job-error.html @@ -0,0 +1,39 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 job status</h1> + +<h2>R/qtl2 Upload: Job Status</h2> + +<div class="explainer"> + <p>The processing of the R/qtl2 bundle you uploaded has failed. We have + provided some information below to help you figure out what the problem + could be.</p> + <p>If you find that you cannot figure out what the problem is on your own, + please contact the team running the system for assistance, providing the + following details: + <ul> + <li>R/qtl2 bundle you uploaded</li> + <li>This URL: <strong>{{request_url()}}</strong></li> + <li>(maybe) a screenshot of this page</li> + </ul> + </p> +</div> + +<h4>stdout</h4> +{{cli_output(job, "stdout")}} + +<h4>stderr</h4> +{{cli_output(job, "stderr")}} + +<h4>Log</h4> +<div class="cli-output"> + {%for msg in messages%} + {{msg}}<br /> + {%endfor%} +</div> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/rqtl2-job-results.html b/uploader/templates/populations/rqtl2/rqtl2-job-results.html new file mode 100644 index 0000000..4ecd415 --- /dev/null +++ b/uploader/templates/populations/rqtl2/rqtl2-job-results.html @@ -0,0 +1,24 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 job status</h1> + +<h2>R/qtl2 Upload: Job Status</h2> + +<div class="explainer"> + <p>The processing of the R/qtl2 bundle you uploaded has completed + successfully.</p> + <p>You should now be able to use GeneNetwork to run analyses on your data.</p> +</div> + +<h4>Log</h4> +<div class="cli-output"> + {%for msg in messages%} + {{msg}}<br /> + {%endfor%} +</div> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/rqtl2-job-status.html b/uploader/templates/populations/rqtl2/rqtl2-job-status.html new file mode 100644 index 0000000..e896f88 --- /dev/null +++ b/uploader/templates/populations/rqtl2/rqtl2-job-status.html @@ -0,0 +1,20 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Job Status{%endblock%} + +{%block extrameta%} +<meta http-equiv="refresh" content="3"> +{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 job status</h1> + +<h2>R/qtl2 Upload: Job Status</h2> + +<h4>Log</h4> +<div class="cli-output"> + <pre>{{"\n".join(messages)}}</pre> +</div> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/rqtl2-qc-job-error.html b/uploader/templates/populations/rqtl2/rqtl2-qc-job-error.html new file mode 100644 index 0000000..90e8887 --- /dev/null +++ b/uploader/templates/populations/rqtl2/rqtl2-qc-job-error.html @@ -0,0 +1,120 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block title%}R/qtl2 bundle: QC Job Error{%endblock%} + +{%macro errors_table(tableid, errors)%} +<table id="{{tableid}}" class="table error-table"> + <caption>{{caption}}</caption> + <thead> + <tr> + <th>Line</th> + <th>Column</th> + <th>Value</th> + <th>Message</th> + </tr> + </thead> + <tbody> + {%for error in errors%} + <tr> + <td>{{error.line}}</td> + <td>{{error.column}}</td> + <td>{{error.value}}</td> + <td>{{error.message}}</td> + </tr> + {%else%} + <tr> + <td colspan="4">No errors to display here.</td> + </tr> + {%endfor%} + </tbody> +</table> +{%endmacro%} + +{%block contents%} +<h1 class="heading">R/qtl2 bundle: QC job Error</h1> + +<div class="explainer"> + <p>The R/qtl2 bundle has failed some <emph>Quality Control</emph> checks.</p> + <p>We list below some of the errors that need to be fixed before the data can + be uploaded onto GeneNetwork.</p> +</div> + +{%if errorsgeneric | length > 0%} +<h2 class="heading">Generic Errors ({{errorsgeneric | length}})</h3> +<div class="explainer"> + We found the following generic errors in your R/qtl2 bundle: +</div> + +<h3>Missing Files</h3> +<div class="explainer"> + <p>These files are listed in the bundle's control file, but do not actually + exist in the bundle</p> +</div> +<table id="tbl-errors-missing-files" class="table error-table"> + <thead> + <tr> + <th>Control File Key</th> + <th>Bundle File Name</th> + <th>Message</th> + </tr> + </thead> + <tbody> + {%for error in (errorsgeneric | selectattr("type", "equalto", "MissingFile"))%} + <tr> + <td>{{error.controlfilekey}}</td> + <td>{{error.filename}}</td> + <td>{{error.message}}</td> + </tr> + {%endfor%} + </tbody> +</table> + +<h3>Other Generic Errors</h3> +{{errors_table("tbl-errors-generic", errorsgeneric| selectattr("type", "ne", "MissingFile"))}} +{%endif%} + +{%if errorsgeno | length > 0%} +<h2 class="heading">Geno Errors ({{errorsgeno | length}})</h3> +<div class="explainer"> + We found the following errors in the 'geno' file in your R/qtl2 bundle: +</div> +{{errors_table("tbl-errors-geno", errorsgeno[0:50])}} +{%endif%} + +{%if errorspheno | length > 0%} +<h2 class="heading">Pheno Errors ({{errorspheno | length}})</h3> +<div class="explainer"> + We found the following errors in the 'pheno' file in your R/qtl2 bundle: +</div> +{{errors_table("tbl-errors-pheno", errorspheno[0:50])}} +{%endif%} + +{%if errorsphenose | length > 0%} +<h2 class="heading">Phenose Errors ({{errorsphenose | length}})</h3> +<div class="explainer"> + We found the following errors in the 'phenose' file in your R/qtl2 bundle: +</div> +{{errors_table("tbl-errors-phenose", errorsphenose[0:50])}} +{%endif%} + +{%if errorsphenocovar | length > 0%} +<h2 class="heading">Phenocovar Errors ({{errorsphenocovar | length}})</h3> +<div class="explainer"> + We found the following errors in the 'phenocovar' file in your R/qtl2 bundle: +</div> +{{errorsphenocovar}} +{%endif%} + +<h4>stdout</h4> +{{cli_output(job, "stdout")}} + +<h4>stderr</h4> +{{cli_output(job, "stderr")}} + +<h4>Log</h4> +<div class="cli-output"> + <pre>{{"\n".join(messages)}}</pre> +</div> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/rqtl2-qc-job-results.html b/uploader/templates/populations/rqtl2/rqtl2-qc-job-results.html new file mode 100644 index 0000000..b3c3a8f --- /dev/null +++ b/uploader/templates/populations/rqtl2/rqtl2-qc-job-results.html @@ -0,0 +1,66 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block title%}R/qtl2 bundle: QC job results{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 bundle: QC job results</h1> + +<div class="row"> + <p>The R/qtl2 bundle you uploaded has passed all automated quality-control + checks successfully.</p> + <p>You may now continue to load the data into GeneNetwork for the bundle, with + the following details:</p> +</div> + +<div class="row"> + <form id="form-qc-job-results" + action="{{url_for('expression-data.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.Id)}}" + method="POST"> + <div class="form-group"> + <legend>Species</legend> + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + + <span class="form-label">Name</span> + <span class="form-text">{{species.Name | capitalize}}</span> + + <span class="form-label">Scientific</span> + <span class="form-text">{{species.FullName | capitalize}}</span> + </div> + + <div class="form-group"> + <legend>population</legend> + <input type="hidden" name="population_id" value="{{population.Id}}" /> + + <span class="form-label">Name</span> + <span class="form-text">{{population.InbredSetName}}</span> + + <span class="form-label">Full Name</span> + <span class="form-text">{{population.FullName}}</span> + + <span class="form-label">Genetic Type</span> + <span class="form-text">{{population.GeneticType}}</span> + + <span class="form-label">Description</span> + <span class="form-text">{{population.Description or "-"}}</span> + </div> + + <div class="form-group"> + <legend>R/qtl2 Bundle File</legend> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2bundle}}" /> + <input type="hidden" name="original-filename" value="{{rqtl2bundleorig}}" /> + + <span class="form-label">Original Name</span> + <span class="form-text">{{rqtl2bundleorig}}</span> + + <span class="form-label">Internal Name</span> + <span class="form-text">{{rqtl2bundle[0:25]}}…</span> + </div> + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/rqtl2-qc-job-status.html b/uploader/templates/populations/rqtl2/rqtl2-qc-job-status.html new file mode 100644 index 0000000..f4a6266 --- /dev/null +++ b/uploader/templates/populations/rqtl2/rqtl2-qc-job-status.html @@ -0,0 +1,41 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Job Status{%endblock%} + +{%block extrameta%} +<meta http-equiv="refresh" content="3"> +{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 bundle: QC job status</h1> + +{%if geno_percent%} +<p> + <h2>Checking 'geno' file:</h2> + <progress id="prg-geno-checking" value="{{geno_percent}}" max="100"> + {{geno_percent}}%</progress> + {{geno_percent}}%</p> +{%endif%} + +{%if pheno_percent%} +<p> + <h2>Checking 'pheno' file:</h2> + <progress id="prg-pheno-checking" value="{{pheno_percent}}" max="100"> + {{pheno_percent}}%</progress> + {{pheno_percent}}%</p> +{%endif%} + +{%if phenose_percent%} +<p> + <h2>Checking 'phenose' file:</h2> + <progress id="prg-phenose-checking" value="{{phenose_percent}}" max="100"> + {{phenose_percent}}%</progress> + {{phenose_percent}}%</p> +{%endif%} + +<h4>Log</h4> +<div class="cli-output"> + <pre>{{"\n".join(messages)}}</pre> +</div> +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/rqtl2-qc-job-success.html b/uploader/templates/populations/rqtl2/rqtl2-qc-job-success.html new file mode 100644 index 0000000..f126835 --- /dev/null +++ b/uploader/templates/populations/rqtl2/rqtl2-qc-job-success.html @@ -0,0 +1,37 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}R/qtl2 Bundle: Quality Control Successful{%endblock%} + +{%block contents%} +<h2 class="heading">R/qtl2 Bundle: Quality Control Successful</h2> + +<div class="row"> + <p>The R/qtl2 bundle you uploaded has passed <emph>all</emph> quality control + checks successfully, and is now ready for uploading into the database.</p> + <p>Click "Continue" below to proceed.</p> +</div> + +<!-- + The "action" on this form takes us to the next step, where we can + select all the other data necessary to enter the data into the database. + --> +<div class="row"> + <form id="frm-upload-rqtl2-bundle" + action="{{url_for('expression-data.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + {{flash_all_messages()}} + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/select-geno-dataset.html b/uploader/templates/populations/rqtl2/select-geno-dataset.html new file mode 100644 index 0000000..3233abc --- /dev/null +++ b/uploader/templates/populations/rqtl2/select-geno-dataset.html @@ -0,0 +1,69 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Select Genotypes Dataset</h2> + +<div class="row"> + <p>Your R/qtl2 files bundle could contain a "geno" specification. You will + therefore need to select from one of the existing Genotype datasets or + create a new one.</p> + <p>This is the dataset where your data will be organised under.</p> +</div> + +<div class="row"> + <form id="frm-upload-rqtl2-bundle" + action="{{url_for('expression-data.rqtl2.select_geno_dataset', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + <legend class="heading">select from existing genotype datasets</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + + {{flash_messages("error-rqtl2-select-geno-dataset")}} + + <div class="form-group"> + <legend>Datasets</legend> + <label for="select:geno-datasets" class="form-label">Dataset</label> + <select id="select:geno-datasets" + name="geno-dataset-id" + required="required" + {%if datasets | length == 0%} + disabled="disabled" + {%endif%} + class="form-control" + aria-describedby="help-geno-dataset-select-dataset"> + <option value="">Select dataset</option> + {%for dset in datasets%} + <option value="{{dset['Id']}}">{{dset["Name"]}} ({{dset["FullName"]}})</option> + {%endfor%} + </select> + <span id="help-geno-dataset-select-dataset" class="form-text text-muted"> + Select from the existing genotype datasets for species + {{species.SpeciesName}} ({{species.FullName}}). + </span> + </div> + + <button type="submit" class="btn btn-primary">select dataset</button> + </form> +</div> + +<div class="row"> + <p>If the genotype dataset you need does not currently exist for your dataset, + go the <a href="{{url_for( + 'species.populations.genotypes.create_dataset', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="Create a new genotypes dataset for {{species.FullName}}"> + genotypes page to create the genotype dataset</a></p> +</div> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/select-population.html b/uploader/templates/populations/rqtl2/select-population.html new file mode 100644 index 0000000..ded425f --- /dev/null +++ b/uploader/templates/populations/rqtl2/select-population.html @@ -0,0 +1,57 @@ +{%extends "expression-data/index.html"%} +{%from "flash_messages.html" import flash_messages%} +{%from "species/macro-display-species-card.html" import display_species_card%} + +{%block title%}Select Grouping/Population{%endblock%} + +{%block contents%} +<h1 class="heading">Select grouping/population</h1> + +<div class="row"> + <p>The data is organised in a hierarchical form, beginning with + <em>species</em> at the very top. Under <em>species</em> the data is + organised by <em>population</em>, sometimes referred to as <em>grouping</em>. + (In some really old documents/systems, you might see this referred to as + <em>InbredSet</em>.)</p> + <p>In this section, you get to define what population your data is to be + organised by.</p> +</div> + +<div class="row"> + <form method="POST" + action="{{url_for('expression-data.rqtl2.select_population', + species_id=species.SpeciesId)}}"> + <legend class="heading">select grouping/population</legend> + {{flash_messages("error-select-population")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + + <div class="form-group"> + <label for="select:inbredset" class="form-label">population</label> + <select id="select:inbredset" + name="inbredset_id" + required="required" + class="form-control"> + <option value="">Select a grouping/population</option> + {%for pop in populations%} + <option value="{{pop.InbredSetId}}"> + {{pop.InbredSetName}} ({{pop.FullName}})</option> + {%endfor%} + </select> + <span class="form-text text-muted">Select the population for your data from + the list below.</span> + </div> + + <button type="submit" class="btn btn-primary" />select population</button> +</form> +</div> + +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} + + +{%block javascript%} +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/select-probeset-dataset.html b/uploader/templates/populations/rqtl2/select-probeset-dataset.html new file mode 100644 index 0000000..74f8f69 --- /dev/null +++ b/uploader/templates/populations/rqtl2/select-probeset-dataset.html @@ -0,0 +1,191 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Phenotype(ProbeSet) Dataset</h2> + +<div class="row"> + <p>The R/qtl2 bundle you uploaded contains (a) "<strong>pheno</strong>" + file(s). This data needs to be organised under a dataset.</p> + <p>This page gives you the ability to do that.</p> +</div> + +{%if datasets | length > 0%} +<div class="row"> + <form method="POST" + action="{{url_for('expression-data.rqtl2.select_probeset_dataset', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:select-probeset-dataset"> + <legend class="heading">Select from existing ProbeSet datasets</legend> + {{flash_messages("error-rqtl2")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" /> + + <div class="form-group"> + <label class="form-label" for="select:probe-dataset">Dataset</label> + <select id="select:probe-dataset" + name="probe-dataset-id" + required="required" + {%if datasets | length == 0%}disabled="disabled"{%endif%} + class="form-control" + aria-describedby="help-probe-dataset"> + <option value="">Select a dataset</option> + {%for dataset in datasets%} + <option value={{dataset.Id}}> + {{dataset.Name}} + {%if dataset.FullName%} + -- ({{dataset.FullName}}) + {%endif%} + </option> + {%endfor%} + </select> + + <span id="help-probe-dataset" class="form-text text-muted"> + Select from existing ProbeSet datasets.</span> + </div> + + <button type="submit" class="btn btn-primary" />select dataset</button> +</form> +</div> + +<div class="row"> + <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> +</div> +{%endif%} + +<div class="row"> + <p>Create an entirely new ProbeSet dataset for your data.</p> +</div> + +<div class="row"> + <form method="POST" + action="{{url_for('expression-data.rqtl2.create_probeset_dataset', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:create-probeset-dataset"> + <legend class="heading">Create a new ProbeSet dataset</legend> + {{flash_messages("error-rqtl2-create-probeset-dataset")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" /> + + <div class="form-group"> + <label class="form-label" for="select:average">averaging method</label> + <select id="select:average" + name="averageid" + required="required" + class="form-control" + aria-describedby="help-average"> + <option value="">Select averaging method</option> + {%for avgmethod in avgmethods%} + <option value="{{avgmethod.Id}}"> + {{avgmethod.Name}} + {%if avgmethod.Normalization%} + ({{avgmethod.Normalization}}) + {%endif%} + </option> + {%endfor%} + </select> + + <span id="help-average" class="form-text text-muted"> + Select the averaging method used for your data. + </span> + </div> + + <div class="form-group"> + <label class="form-label" for="txt:datasetname">Name</label> + <input type="text" id="txt:datasetname" name="datasetname" + required="required" + maxlength="40" + title="Name of the dataset, e.g 'BXDMicroArray_ProbeSet_June03'. (Required)" + class="form-control" + aria-describedby="help-dataset-name" /> + + <span id="help-dataset-name" class="form-text text-muted"> + Provide a name for the dataset e.g. "BXDMicroArray_ProbeSet_June03". This + is mandatory <strong>MUST</strong> be provided. + </span> + </div> + + <div class="form-group"> + <label class="form-label" for="txt:datasetfullname">Full Name</label> + <input type="text" id="txt:datasetfullname" name="datasetfullname" + required="required" + maxlength="100" + title="A longer name for the dataset, e.g. 'UTHSC Brain mRNA U74Av2 (Jun03) MAS5'. (Required)" + class="form-control" + aria-describedby="help-dataset-fullname" /> + + <span id="help-dataset-fullname" class="form-text text-muted"> + Provide a longer, more descriptive name for the dataset e.g. + "UTHSC Brain mRNA U74Av2 (Jun03) MAS5". This is mandatory and + <strong>MUST</strong> be provided. + </span> + </div> + + <div class="form-group"> + <label class="form-label" for="txt:datasetshortname">Short Name</label> + <input type="text" id="txt:datasetshortname" name="datasetshortname" + maxlength="100" + title="An abbreviated name for the dataset, e.g 'Br_U_0603_M'. (Optional)" + class="form-control" + aria-describedby="help-dataset-shortname" /> + + <span id="help-dataset-shortname" class="form-text text-muted"> + Provide a longer, more descriptive name for the dataset e.g. "Br_U_0603_M". + This is optional. + </span> + </div> + + <div class="form-check"> + <input type="checkbox" id="chk:public" name="datasetpublic" + checked="checked" + title="Whether or not the dataset is accessible by the general public." + class="form-check-input" + aria-describedby="help-public" /> + <label class="form-check-label" for="chk:datasetpublic">Public?</label> + + <span id="help-public" class="form-text text-muted"> + Check to specify that the dataset will be publicly available. Uncheck to + limit access to the dataset. + </span> + </div> + + <div class="form-group"> + <label class="form-label" for="select:datasetdatascale">Data Scale</label> + <select id="select:datasetdatascale" + name="datasetdatascale" + required="required" + class="form-control" + aria-describedby="help-dataset-datascale"> + <option value="log2" selected="selected">log2</option> + <option value="z_score">z_score</option> + <option value="log2_ratio">log2_ratio</option> + <option value="linear">linear</option> + <option value="linear_positive">linear_positive</option> + </select> + + <span id="help-dataset-datascale" class="form-text text-muted"> + Select from a list of scaling methods. + </span> + </div> + + <button type="submit" class="btn btn-primary">create dataset</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/select-probeset-study-id.html b/uploader/templates/populations/rqtl2/select-probeset-study-id.html new file mode 100644 index 0000000..e3fd9cc --- /dev/null +++ b/uploader/templates/populations/rqtl2/select-probeset-study-id.html @@ -0,0 +1,143 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages %} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Phenotype(ProbeSet) Study</h2> + +<div class="row"> + <p>The R/qtl2 bundle you uploaded contains (a) "<strong>pheno</strong>" + file(s). This data needs to be organised under a study.</p> + <p>In this page, you can either select from a existing dataset:</p> + + <form method="POST" + action="{{url_for('expression-data.rqtl2.select_probeset_study', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:select-probeset-study"> + <legend class="heading">Select from existing ProbeSet studies</legend> + {{flash_messages("error-rqtl2-select-probeset-study")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + + <div> + <label for="select:probe-study" class="form-label">Study</label> + <select id="select:probe-study" + name="probe-study-id" + required="required" + aria-describedby="help-select-probeset-study" + {%if studies | length == 0%}disabled="disabled"{%endif%} + class="form-control"> + <option value="">Select a study</option> + {%for study in studies%} + <option value={{study.Id}}> + {{study.Name}} + {%if study.FullName%} + -- ({{study.FullName}}) + {%endif%} + </option> + {%endfor%} + </select> + <small id="help-select-probeset-study" class="form-text text-muted"> + Select from existing ProbeSet studies. + </small> + </div> + + <button type="submit" class="btn btn-primary">select study</button> + </form> +</div> + +<div class="row"> + <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> +</div> + +<div class="row"> + + <p>Create a new ProbeSet dataset below:</p> + + <form method="POST" + action="{{url_for('expression-data.rqtl2.create_probeset_study', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:create-probeset-study"> + <legend class="heading">Create new ProbeSet study</legend> + + {{flash_messages("error-rqtl2-create-probeset-study")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + + <div> + <label for="select:platform" class="form-label">Platform</label> + <select id="select:platform" + name="platformid" + required="required" + aria-describedby="help-select-platform" + {%if platforms | length == 0%}disabled="disabled"{%endif%} + class="form-control"> + <option value="">Select a platform</option> + {%for platform in platforms%} + <option value="{{platform.GeneChipId}}"> + {{platform.GeneChipName}} ({{platform.Name}}) + </option> + {%endfor%} + </select> + <small id="help-select-platform" class="form-text text-muted"> + Select from a list of known genomics platforms. + </small> + </div> + + <div class="form-group"> + <label for="txt:studyname" class="form-label">Study name</label> + <input type="text" id="txt:studyname" name="studyname" + placeholder="Name of the study. (Required)" + required="required" + maxlength="100" + class="form-control" /> + <span class="form-text text-muted" id="help-study-name"> + Provide a name for the study.</span> + </div> + + <div class="form-group"> + <label for="txt:studyfullname" class="form-label">Full Study Name</label> + <input type="text" + id="txt:studyfullname" + name="studyfullname" + placeholder="Longer name of the study. (Optional)" + maxlength="100" + class="form-control" /> + <span class="form-text text-muted" id="help-study-full-name"> + Provide a longer, more descriptive name for the study. This is optional + and you can leave it blank. + </span> + </div> + + <div class="form-group"> + <label for="txt:studyshortname" class="form-label">Short Study Name</label> + <input type="text" + id="txt:studyshortname" + name="studyshortname" + placeholder="Shorter name of the study. (Optional)" + maxlength="100" + class="form-control" /> + <span class="form-text text-muted" id="help-study-short-name"> + Provide a shorter name for the study. This is optional and you can leave + it blank. + </span> + </div> + + <button type="submit" class="btn btn-primary">create study</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/select-tissue.html b/uploader/templates/populations/rqtl2/select-tissue.html new file mode 100644 index 0000000..fe3080a --- /dev/null +++ b/uploader/templates/populations/rqtl2/select-tissue.html @@ -0,0 +1,115 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Tissue</h2> + +<div class="row"> + <p>The data you are uploading concerns a tissue, cell, organ, or other + biological material used in an experiment.</p> + <p>Select the appropriate biological material below</p> +</div> + +{%if tissues | length > 0%} +<div class="row"> + <form method="POST" + action="{{url_for('expression-data.rqtl2.select_tissue', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:select-probeset-dataset"> + <legend class="heading">Select from existing ProbeSet datasets</legend> + {{flash_messages("error-select-tissue")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + + <div class="form-group"> + <label class="form-label" for="select-tissue">Tissue</label> + <select id="select-tissue" + name="tissueid" + required="required" + {%if tissues | length == 0%}disabled="disabled"{%endif%} + class="form-control" + aria-describedby="help-select-tissue"> + <option value="">Select a tissue</option> + {%for tissue in tissues%} + <option value={{tissue.Id}}> + {{tissue.Name}} + {%if tissue.Short_Name%} + -- ({{tissue.Short_Name}}) + {%endif%} + </option> + {%endfor%} + </select> + + <span id="help-select-tissue" class="form-text text-muted"> + Select from existing biological material.</span> + </div> + + <button type="submit" class="btn btn-primary">use selected</button> + </form> +</div> + +<div class="row"> + <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> +</div> +{%endif%} + +<div class="row"> + <p>If you cannot find the biological material in the drop-down above, add it + to the system below.</p> + + <form method="POST" + action="{{url_for('expression-data.rqtl2.create_tissue', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:create-probeset-dataset"> + <legend class="heading">Add new tissue, organ or biological material</legend> + {{flash_messages("error-create-tissue")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + + <div class="form-group"> + <label class="form-label" for="tissue-name">name</label> + <input type="text" + id="txt-tissuename" + name="tissuename" + required="required" + title = "A name to identify the tissue, organ or biological material." + class="form-control" + aria-describedby="help-tissue-name" /> + + <span class="form-text text-muted" id="help-tissue-name"> + A name to identify the tissue, organ or biological material. + </span> + </div> + + <div class="form-group"> + <label for="txt-shortname" class="form-label">short name</label> + <input type="text" id="txt-tissueshortname" name="tissueshortname" + required="required" + maxlength="7" + title="A short name (e.g. 'Mam') for the biological material." + class="form-control" + aria-describedby="help-tissue-short-name" /> + + <span class="form-text text-muted" id="help-tissue-short-name"> + Provide a short name for the tissue, organ or biological material used in + the experiment. + </span> + </div> + + <button type="submit" class="btn btn-primary" />add new material</button> +</form> +</div> + +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/summary-info.html b/uploader/templates/populations/rqtl2/summary-info.html new file mode 100644 index 0000000..0adba2e --- /dev/null +++ b/uploader/templates/populations/rqtl2/summary-info.html @@ -0,0 +1,65 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Summary</h2> + +<div class="row"> + <p>This is the information you have provided to accompany the R/qtl2 bundle + you have uploaded. Please verify the information is correct before + proceeding.</p> +</div> + +<div class="row"> + <dl> + <dt>Species</dt> + <dd>{{species.SpeciesName}} ({{species.FullName}})</dd> + + <dt>Population</dt> + <dd>{{population.InbredSetName}}</dd> + + {%if geno_dataset%} + <dt>Genotype Dataset</dt> + <dd>{{geno_dataset.Name}} ({{geno_dataset.FullName}})</dd> + {%endif%} + + {%if tissue%} + <dt>Tissue</dt> + <dd>{{tissue.TissueName}} ({{tissue.Name}}, {{tissue.Short_Name}})</dd> + {%endif%} + + {%if probe_study%} + <dt>ProbeSet Study</dt> + <dd>{{probe_study.Name}} ({{probe_study.FullName}})</dd> + {%endif%} + + {%if probe_dataset%} + <dt>ProbeSet Dataset</dt> + <dd>{{probe_dataset.Name2}} ({{probe_dataset.FullName}})</dd> + {%endif%} + </dl> +</div> + +<div class="row"> + <form id="frm:confirm-rqtl2bundle-details" + action="{{url_for('expression-data.rqtl2.confirm_bundle_details', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + <legend class="heading">Create ProbeSet dataset</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" /> + <input type="hidden" name="probe-dataset-id" value="{{probe_dataset.Id}}" /> + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-01.html b/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-01.html new file mode 100644 index 0000000..9d45c5f --- /dev/null +++ b/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-01.html @@ -0,0 +1,276 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "upload_progress_indicator.html" import upload_progress_indicator%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +{%macro rqtl2_file_help()%} +<span class="form-text text-muted"> + <p> + Provide a valid R/qtl2 zip file here. In particular, ensure your zip bundle + contains exactly one control file and the corresponding files mentioned in + the control file. + </p> + <p> + The control file can be either a YAML or JSON file. <em>ALL</em> other data + files in the zip bundle should be CSV files. + </p> + <p>See the + <a href="https://kbroman.org/qtl2/assets/vignettes/input_files.html" + target="_blank"> + R/qtl2 file format specifications + </a> + for more details. + </p> +</span> +{%endmacro%} +{{upload_progress_indicator()}} + +<div id="resumable-file-display-template" + class="panel panel-info" + style="display: none"> + <div class="panel-heading"></div> + <div class="panel-body"></div> +</div> + + +<h2 class="heading">Upload R/qtl2 Bundle</h2> + +<div id="resumable-drop-area" + style="display:none;background:#eeeeee;min-height:12em;border-radius:0.5em;padding:1em;"> + <p> + <a id="resumable-browse-button" href="#" + class="btn btn-info">Browse</a> + </p> + <p class="form-text text-muted"> + You can drag and drop your file here, or click the browse button. + Click on the file to remove it. + </p> + {{rqtl2_file_help()}} + <div id="resumable-selected-files" + style="display:flex;flex-direction:row;flex-wrap: wrap;justify-content:space-around;gap:10px 20px;"></div> + <div id="resumable-class-buttons" style="text-align: right;"> + <button id="resumable-upload-button" + class="btn btn-primary" + style="display: none">start upload</button> + <button id="resumable-cancel-upload-button" + class="btn btn-danger" + style="display: none">cancel upload</button> + </div> + <div id="resumable-progress-bar" class="progress" style="display: none"> + <div class="progress-bar" + role="progress-bar" + aria-valuenow="60" + aria-valuemin="0" + aria-valuemax="100" + style="width: 0%;"> + Uploading: 60% + </div> + </div> +</div> + +<form id="frm-upload-rqtl2-bundle" + action="{{url_for('expression-data.rqtl2.upload_rqtl2_bundle', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data" + data-resumable-target="{{url_for( + 'expression-data.rqtl2.upload_rqtl2_bundle_chunked_post', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}"> + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + + {{flash_all_messages()}} + + <div class="form-group"> + <legend class="heading">file upload</legend> + <label for="file-rqtl2-bundle" class="form-label">R/qtl2 bundle</label> + <input type="file" id="file-rqtl2-bundle" name="rqtl2_bundle_file" + accept="application/zip, .zip" + required="required" + class="form-control" /> + {{rqtl2_file_help()}} + </div> + + <button type="submit" + class="btn btn-primary" + data-toggle="modal" + data-target="#upload-progress-indicator">upload R/qtl2 bundle</button> +</form> + +{%endblock%} + +{%block javascript%} +<script src="{{url_for('base.node_modules', + filename='resumablejs/resumable.js')}}"></script> +<script type="text/javascript" src="/static/js/upload_progress.js"></script> +<script type="text/javascript"> + function readBinaryFile(file) { + return new Promise((resolve, reject) => { + var _reader = new FileReader(); + _reader.onload = (event) => {resolve(_reader.result);}; + _reader.readAsArrayBuffer(file); + }); + } + + function computeFileChecksum(file) { + return readBinaryFile(file) + .then((content) => { + return window.crypto.subtle.digest( + "SHA-256", new Uint8Array(content)); + }).then((digest) => { + return Uint8ArrayToHex(new Uint8Array(digest)) + }); + } + + function Uint8ArrayToHex(arr) { + var toHex = (val) => { + _hex = val.toString(16); + if(_hex.length < 2) { + return "0" + val; + } + return _hex; + }; + _hexstr = "" + arr.forEach((val) => {_hexstr += toHex(val)}); + return _hexstr + } + + var r = Resumable({ + target: $("#frm-upload-rqtl2-bundle").attr("data-resumable-target"), + fileType: ["zip"], + maxFiles: 1, + forceChunkSize: true, + generateUniqueIdentifier: (file, event) => { + return computeFileChecksum(file).then((checksum) => { + var _relativePath = (file.webkitRelativePath + || file.relativePath + || file.fileName + || file.name); + return checksum + "-" + _relativePath.replace( + /[^a-zA-Z0-9_-]/img, ""); + }); + } + }); + + if(r.support) { + //Hide form and display drag&drop UI + $("#frm-upload-rqtl2-bundle").css("display", "none"); + $("#resumable-drop-area").css("display", "block"); + + // Define UI elements for browse and drag&drop + r.assignDrop(document.getElementById("resumable-drop-area")); + r.assignBrowse(document.getElementById("resumable-browse-button")); + + // Event handlers + + function display_files(files) { + displayArea = $("#resumable-selected-files") + displayArea.empty(); + files.forEach((file) => { + var displayElement = $( + "#resumable-file-display-template").clone(); + displayElement.removeAttr("id"); + displayElement.css("display", ""); + displayElement.find(".panel-heading").text(file.fileName); + list = $("<ul></ul>"); + list.append($("<li><strong>Name</strong>: " + + (file.name + || file.fileName + || file.relativePath + || file.webkitRelativePath) + + "</li>")); + list.append($("<li><strong>Size</strong>: " + + (file.size / (1024*1024)).toFixed(2) + + " MB</li>")); + list.append($("<li><strong>Unique Identifier</strong>: " + + file.uniqueIdentifier + "</li>")); + list.append($("<li><strong>Mime</strong>: " + + file.file.type + + "</li>")); + displayElement.find(".panel-body").append(list); + displayElement.appendTo("#resumable-selected-files"); + }); + } + + r.on("filesAdded", function(files) { + display_files(files); + $("#resumable-upload-button").css("display", ""); + $("#resumable-upload-button").on("click", (event) => { + r.upload(); + }); + }); + + r.on("uploadStart", (event) => { + $("#resumable-upload-button").css("display", "none"); + $("#resumable-cancel-upload-button").css("display", ""); + $("#resumable-cancel-upload-button").on("click", (event) => { + r.files.forEach((file) => { + if(file.isUploading()) { + file.abort(); + } + }); + $("#resumable-cancel-upload-button").css("display", "none"); + $("#resumable-upload-button").on("click", (event) => { + r.files.forEach((file) => {file.retry();}); + }); + $("#resumable-upload-button").css("display", ""); + }); + }); + + r.on("progress", () => { + var progress = (r.progress() * 100).toFixed(2); + var pbar = $("#resumable-progress-bar > .progress-bar"); + $("#resumable-progress-bar").css("display", ""); + pbar.css("width", progress+"%"); + pbar.attr("aria-valuenow", progress); + pbar.text("Uploading: " + progress + "%"); + }) + + r.on("fileSuccess", (file, message) => { + if(message != "OK") { + var uri = (window.location.protocol + + "//" + + window.location.host + + message); + window.location.replace(uri); + } + }); + + r.on("error", (message, file) => { + filename = (file.webkitRelativePath + || file.relativePath + || file.fileName + || file.name); + jsonmsg = JSON.parse(message); + alert("There was an error while uploading your file '" + + filename + + "'. The error message was:\n\n\t" + + jsonmsg.error + + " (" + + jsonmsg.statuscode + + "): " + jsonmsg.message); + }) + } else { + setup_upload_handlers( + "frm-upload-rqtl2-bundle", make_data_uploader( + function (form) { + var formdata = new FormData(); + formdata.append( + "species_id", + form.querySelector('input[name="species_id"]').value); + formdata.append( + "population_id", + form.querySelector('input[name="population_id"]').value); + formdata.append( + "rqtl2_bundle_file", + form.querySelector("#file-rqtl2-bundle").files[0]); + return formdata; + })); + } +</script> +{%endblock%} diff --git a/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-02.html b/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-02.html new file mode 100644 index 0000000..8210ed0 --- /dev/null +++ b/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-02.html @@ -0,0 +1,33 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Upload R/qtl2 Bundle</h2> + +<div class="row"> + <p>You have successfully uploaded the zipped bundle of R/qtl2 files.</p> + <p>The next step is to select the various extra information we need to figure + out what to do with the data. You will select/create the relevant studies + and/or datasets to organise the data in the steps that follow.</p> + <p>Click "Continue" below to proceed.</p> + + <form id="frm-upload-rqtl2-bundle" + action="{{url_for('expression-data.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + {{flash_all_messages()}} + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/populations/view-population.html b/uploader/templates/populations/view-population.html new file mode 100644 index 0000000..b23caeb --- /dev/null +++ b/uploader/templates/populations/view-population.html @@ -0,0 +1,102 @@ +{%extends "populations/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-select-species.html" import select_species_form%} +{%from "species/macro-display-species-card.html" import display_species_card%} + +{%block title%}Populations{%endblock%} + +{%block pagetitle%}Populations{%endblock%} + +{%block lvl3_breadcrumbs%} +<li {%if activelink=="view-population"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.view_population', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}">view</a> +</li> +{%endblock%} + + +{%block contents%} +<div class="row"> + <h2>Population Details</h2> + + {{flash_all_messages()}} + + <dl> + <dt>Name</dt> + <dd>{{population.Name}}</dd> + + <dt>FullName</dt> + <dd>{{population.FullName}}</dd> + + <dt>Code</dt> + <dd>{{population.InbredSetCode}}</dd> + + <dt>Genetic Type</dt> + <dd>{{population.GeneticType}}</dd> + + <dt>Family</dt> + <dd>{{population.Family}}</dd> + + <dt>Description</dt> + <dd><pre>{{population.Description or "-"}}</pre></dd> + </dl> +</div> + +<div class="row"> + … maybe provide a way to organise populations in the same family here … +</div> + +<div class="row"> + <h3>Actions</h3> + + <p> + Click any of the following links to use this population in performing the + subsequent operations. + </p> + + <nav class="nav"> + <ul> + <li> + <a href="{{url_for('species.populations.samples.list_samples', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="Manage samples: Add new or delete existing."> + manage samples</a> + </li> + <li> + <a href="{{url_for('species.populations.genotypes.list_genotypes', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="Manage genotypes for {{species.FullName}}">Manage Genotypes</a> + </li> + <li> + <a href="{{url_for('species.populations.phenotypes.list_datasets', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="Manage phenotype data.">manage phenotype data</a> + </li> + <li> + <a href="#" title="Manage expression data" + class="not-implemented">manage expression data</a> + </li> + <li> + <a href="#" title="Manage individual data" + class="not-implemented">manage individual data</a> + </li> + <li> + <a href="#" title="Manage RNA-Seq data" + class="not-implemented">manage RNA-Seq data</a> + </li> + </ul> + </nav> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} diff --git a/uploader/templates/samples/base.html b/uploader/templates/samples/base.html new file mode 100644 index 0000000..291782b --- /dev/null +++ b/uploader/templates/samples/base.html @@ -0,0 +1,12 @@ +{%extends "populations/base.html"%} + +{%block lvl3_breadcrumbs%} +<li {%if activelink=="samples"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.samples.index')}}">Samples</a> +</li> +{%block lvl4_breadcrumbs%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/samples/index.html b/uploader/templates/samples/index.html new file mode 100644 index 0000000..ee98734 --- /dev/null +++ b/uploader/templates/samples/index.html @@ -0,0 +1,23 @@ +{%extends "samples/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-select-species.html" import select_species_form%} + +{%block title%}Populations{%endblock%} + +{%block pagetitle%}Populations{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <p>GeneNetwork has a selection of different species of organisms to choose from. Within those species, there are the populations of interest for a variety of experiments, from which you, the researcher, picked your samples (or individuals or cases) from. Here you can provide some basic details about your samples.</p> + <p>To start off, we will need to know what species and population your samples belong to. Please provide that information in the next sections.</p> + + {{select_species_form(url_for("species.populations.samples.index"), species)}} +</div> +{%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/samples/list-samples.html b/uploader/templates/samples/list-samples.html new file mode 100644 index 0000000..185e784 --- /dev/null +++ b/uploader/templates/samples/list-samples.html @@ -0,0 +1,132 @@ +{%extends "samples/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "populations/macro-select-population.html" import select_population_form%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Samples — List Samples{%endblock%} + +{%block pagetitle%}Samples — List Samples{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="list-samples"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.samples.list_samples', + species_id=species.SpeciesId, + population_id=population.Id)}}">List</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <p> + You selected the population "{{population.FullName}}" from the + "{{species.FullName}}" species. + </p> +</div> + +{%if samples | length > 0%} +<div class="row"> + <p> + This population already has <strong>{{total_samples}}</strong> + samples/individuals entered. You can explore the list of samples in this + population in the table below. + </p> +</div> + +<div class="row"> + <div class="col-md-2"> + {%if offset > 0:%} + <a href="{{url_for('species.populations.samples.list_samples', + species_id=species.SpeciesId, + population_id=population.Id, + from=offset-count, + count=count)}}"> + <span class="glyphicon glyphicon-backward"></span> + Previous + </a> + {%endif%} + </div> + + <div class="col-md-8" style="text-align: center;"> + Samples {{offset}} — {{offset+(count if offset + count < total_samples else total_samples - offset)}} / {{total_samples}} + </div> + + <div class="col-md-2"> + {%if offset + count < total_samples:%} + <a href="{{url_for('species.populations.samples.list_samples', + species_id=species.SpeciesId, + population_id=population.Id, + from=offset+count, + count=count)}}"> + Next + <span class="glyphicon glyphicon-forward"></span> + </a> + {%endif%} + </div> +</div> +<div class="row"> + <table class="table"> + <thead> + <tr> + <th></th> + <th>Name</th> + <th>Auxilliary Name</th> + <th>Symbol</th> + <th>Alias</th> + </tr> + </thead> + + <tbody> + {%for sample in samples%} + <tr> + <td>{{sample.sequence_number}}</td> + <td>{{sample.Name}}</td> + <td>{{sample.Name2}}</td> + <td>{{sample.Symbol or "-"}}</td> + <td>{{sample.Alias or "-"}}</td> + </tr> + {%endfor%} + </tbody> + </table> + + <p> + <a href="#" + title="Add samples for population '{{population.FullName}}' from species + '{{species.FullName}}'." + class="btn btn-danger"> + delete all samples + </a> + </p> +</div> + +{%else%} + +<div class="row"> + <p> + There are no samples entered for this population. Do please go ahead and add + the samples for this population by clicking on the button below. + </p> + + <p> + <a href="{{url_for('species.populations.samples.upload_samples', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="Add samples for population '{{population.FullName}}' from species + '{{species.FullName}}'." + class="btn btn-primary"> + add samples + </a> + </p> +</div> +{%endif%} + +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/samples/select-population.html b/uploader/templates/samples/select-population.html new file mode 100644 index 0000000..1cc7573 --- /dev/null +++ b/uploader/templates/samples/select-population.html @@ -0,0 +1,26 @@ +{%extends "samples/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "populations/macro-select-population.html" import select_population_form%} +{%from "species/macro-display-species-card.html" import display_species_card%} + +{%block title%}Samples — Select Population{%endblock%} + +{%block pagetitle%}Samples — Select Population{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + {{select_population_form( + url_for("species.populations.samples.select_population", species_id=species.SpeciesId), species, populations)}} +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> +{%endblock%} diff --git a/uploader/templates/samples/upload-failure.html b/uploader/templates/samples/upload-failure.html new file mode 100644 index 0000000..458ab55 --- /dev/null +++ b/uploader/templates/samples/upload-failure.html @@ -0,0 +1,37 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Samples Upload Failure{%endblock%} + +{%block contents%} +<div class="row"> +<h2 class="heading">{{job.job_name[0:50]}}…</h2> + +<p>There was a failure attempting to upload the samples.</p> + +<p>Here is some information to help with debugging the issue. Provide this + information to the developer/maintainer.</p> + +<h3>Debugging Information</h3> +<ul> + <li><strong>job id</strong>: {{job.job_id}}</li> + <li><strong>status</strong>: {{job.status}}</li> + <li><strong>job type</strong>: {{job["job-type"]}}</li> +</ul> +</div> + +<div class="row"> +<h4>stdout</h4> +{{cli_output(job, "stdout")}} +</div> + +<div class="row"> +<h4>stderr</h4> +{{cli_output(job, "stderr")}} +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/samples/upload-progress.html b/uploader/templates/samples/upload-progress.html new file mode 100644 index 0000000..677d457 --- /dev/null +++ b/uploader/templates/samples/upload-progress.html @@ -0,0 +1,31 @@ +{%extends "samples/base.html"%} +{%from "cli-output.html" import cli_output%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block extrameta%} +<meta http-equiv="refresh" content="5"> +{%endblock%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<div class="row" style="overflow-x: clip;"> +<h2 class="heading">{{job.job_name[0:50]}}…</h2> + +<p> +<strong>status</strong>: +<span>{{job["status"]}} ({{job.get("message", "-")}})</span><br /> +</p> + +<p>saving to database...</p> +</div> + +<div class="row"> + {{cli_output(job, "stdout")}} +</div> + +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/samples/upload-samples.html b/uploader/templates/samples/upload-samples.html new file mode 100644 index 0000000..25d3290 --- /dev/null +++ b/uploader/templates/samples/upload-samples.html @@ -0,0 +1,160 @@ +{%extends "samples/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "populations/macro-select-population.html" import select_population_form%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Samples — Upload Samples{%endblock%} + +{%block pagetitle%}Samples — Upload Samples{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="uploade-samples"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.samples.upload_samples', + species_id=species.SpeciesId, + population_id=population.Id)}}">List</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <p> + You can now upload the samples for the "{{population.FullName}}" population + from the "{{species.FullName}}" species here. + </p> + <p> + Upload a <strong>character-separated value (CSV)</strong> file that contains + details about your samples. The CSV file should have the following fields: + <dl> + <dt>Name</dt> + <dd>The primary name/identifier for the sample/individual.</dd> + + <dt>Name2</dt> + <dd>A secondary name for the sample. This can simply be the same as + <strong>Name</strong> above. This field <strong>MUST</strong> contain a + value.</dd> + + <dt>Symbol</dt> + <dd>A symbol for the sample. This can be a strain name, e.g. 'BXD60' for + species that have strains. This field can be left empty for species like + Humans that do not have strains..</dd> + + <dt>Alias</dt> + <dd>An alias for the sample. Can be an empty field, or take on the same + value as that of the Symbol.</dd> + </dl> + </p> +</div> + +<div class="row"> + <form id="form-samples" + method="POST" + action="{{url_for('species.populations.samples.upload_samples', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + enctype="multipart/form-data"> + <legend class="heading">upload samples</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" value="{{population.Id}}" /> + + <div class="form-group"> + <label for="file-samples" class="form-label">select file</label> + <input type="file" name="samples_file" id="file:samples" + accept="text/csv, text/tab-separated-values" + class="form-control" /> + </div> + + <div class="form-group"> + <label for="select:separator" class="form-label">field separator</label> + <select id="select:separator" + name="separator" + required="required" + class="form-control"> + <option value="">Select separator for your file: (default is comma)</option> + <option value="	">TAB</option> + <option value=" ">Space</option> + <option value=",">Comma</option> + <option value=";">Semicolon</option> + <option value="other">Other</option> + </select> + <input id="txt:separator" + type="text" + name="other_separator" + class="form-control" /> + <small class="form-text text-muted"> + If you select '<strong>Other</strong>' for the field separator value, + enter the character that separates the fields in your CSV file in the form + field below. + </small> + </div> + + <div class="form-group form-check"> + <input id="chk:heading" + type="checkbox" + name="first_line_heading" + class="form-check-input" /> + <label for="chk:heading" class="form-check-label"> + first line is a heading?</label> + <small class="form-text text-muted"> + Select this if the first line in your file contains headings for the + columns. + </small> + </div> + + <div class="form-group"> + <label for="txt:delimiter" class="form-label">field delimiter</label> + <input id="txt:delimiter" + type="text" + name="field_delimiter" + maxlength="1" + class="form-control" /> + <small class="form-text text-muted"> + If there is a character delimiting the string texts within particular + fields in your CSV, provide the character here. This can be left blank if + no such delimiters exist in your file. + </small> + </div> + + <button type="submit" + class="btn btn-primary">upload samples file</button> + </form> +</div> + +<div class="row"> + <h3>Preview File Content</h3> + + <table id="tbl:samples-preview" class="table"> + <caption class="heading">preview content</caption> + + <thead> + <tr> + <th>Name</th> + <th>Name2</th> + <th>Symbol</th> + <th>Alias</th> + </tr> + </thead> + + <tbody> + <tr id="default-row"> + <td colspan="4"> + Please make some selections in the form above to preview the data.</td> + </tr> + </tbody> + </table> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} + +{%block javascript%} +<script src="/static/js/upload_samples.js" type="text/javascript"></script> +{%endblock%} diff --git a/uploader/templates/samples/upload-success.html b/uploader/templates/samples/upload-success.html new file mode 100644 index 0000000..881d466 --- /dev/null +++ b/uploader/templates/samples/upload-success.html @@ -0,0 +1,36 @@ +{%extends "samples/base.html"%} +{%from "cli-output.html" import cli_output%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} + +<div class="row" style="overflow-x: clip;"> + <h2 class="heading">{{job.job_name[0:50]}}…</h2> + + <p> + <strong>status</strong>: + <span>{{job["status"]}} ({{job.get("message", "-")}})</span><br /> + </p> + + <p>Successfully uploaded the samples.</p> + <p> + <a href="{{url_for('species.populations.samples.list_samples', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="View population samples"> + View samples + </a> + </p> +</div> + +<div class="row"> + {{cli_output(job, "stdout")}} +</div> + +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/select_dataset.html b/uploader/templates/select_dataset.html new file mode 100644 index 0000000..2f07de8 --- /dev/null +++ b/uploader/templates/select_dataset.html @@ -0,0 +1,161 @@ +{%extends "base.html"%} +{%from "dbupdate_hidden_fields.html" import hidden_fields%} + +{%block title%}Select Dataset{%endblock%} + +{%block css%} +<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> +{%endblock%} + +{%block contents%} +<h2 class="heading">{{filename}}: select dataset</h2> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.final_confirmation')}}" + id="select-dataset-form" class="two-col-sep-col1"> + <legend class="heading">choose existing dataset</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, totallines=totallines)}} + + <div class="form-group"> + <label for="datasetid" class="form-label">dataset:</label> + <select id="datasetid" name="datasetid" class="form-control" + {%if datasets | length == 0:%} + disabled="disabled" + {%endif%}> + {%for dataset in datasets:%} + <option value="{{dataset['Id']}}"> + [{{dataset["Name"]}}] - {{dataset["FullName"]}} + </option> + {%endfor%} + </select> + </div> + + <button type="submit" class="btn btn-primary" + {%if datasets | length == 0:%} + disabled="disabled" + {%endif%} />update database</button> +</form> +</div> + +<div class="row"> + <p class="two-col-sep-separator">OR</p> +</div> + +<div class="row"> + <form method="POST" id="create-dataset-form" + action="{{url_for('dbinsert.create_dataset')}}" + class="two-col-sep-col2"> + <legend class="heading">create new dataset</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, totallines=totallines)}} + + {%with messages = get_flashed_messages(with_categories=true)%} + {%if messages:%} + <ul> + {%for category, message in messages:%} + <li class="{{category}}">{{message}}</li> + {%endfor%} + </ul> + {%endif%} + {%endwith%} + + <div class="form-group"> + <label for="avgid" class="form-label">average:</label> + <select id="avgid" name="avgid" required="required" class="form-control"> + <option value="">Select averaging method</option> + {%for method in avgmethods:%} + <option value="{{method['AvgMethodId']}}" + {%if avgid is defined and method['AvgMethodId'] | int == avgid | int%} + selected="selected" + {%endif%}> + {{method["Name"]}} + </option> + {%endfor%} + </select> + </div> + + <div class="form-group"> + <label for="datasetname" class="form-label">name:</label> + <input id="datasetname" name="datasetname" type="text" + class="form-control" + {%if datasetname is defined %} + value="{{datasetname}}" + {%endif%} /> + </div> + + <div class="form-group"> + <label for="datasetname2" class="form-label">name 2:</label> + <input id="datasetname2" name="datasetname2" type="text" + required="required" class="form-control" + {%if datasetname2 is defined %} + value="{{datasetname2}}" + {%endif%} /> + </div> + + <div class="form-group"> + <label for="datasetfullname" class="form-label">full name:</label> + <input id="datasetfullname" name="datasetfullname" type="text" + required="required" class="form-control" + {%if datasetfullname is defined %} + value="{{datasetfullname}}" + {%endif%} /> + </div> + + <div class="form-group"> + <label for="datasetshortname" class="form-label">short name:</label> + <input id="datasetshortname" name="datasetshortname" type="text" + required="required" class="form-control" + {%if datasetshortname is defined %} + value="{{datasetshortname}}" + {%endif%} /> + </div> + + <div class="form-group"> + <label for="datasetpublic" class="form-label">public:</label> + <input id="datasetpublic" name="datasetpublic" type="number" + required="required" min="0" max="2" + {%if datasetpublic is defined %} + value="{{datasetpublic | int}}" + {%else%} + value="0" + {%endif%} + class="form-control" /> + </div> + + <div class="form-group"> + <label for="datasetconfidentiality">confidentiality:</label> + <input id="datasetconfidentiality" name="datasetconfidentiality" + type="number" required="required" min="0" max="2" + {%if datasetconfidentiality is defined %} + value="{{datasetconfidentiality | int}}" + {%else%} + value="0" + {%endif%} + class="form-control" /> + </div> + + <div class="form-group"> + <label for="datasetdatascale" class="form-label">data scale:</label> + <select id="datasetdatascale" name="datasetdatascale" class="form-control"> + <option value="">None</option> + {%for dscale in datascales:%} + <option value="{{dscale}}" + {%if datasetdatascale is defined and dscale == datasetdatascale%} + selected="selected" + {%elif dscale == "log2":%} + selected="selected" + {%endif%}> + {{dscale}} + </option> + {%endfor%} + </select> + </div> + + <button type="submit" class="btn btn-primary">create new dataset</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/select_platform.html b/uploader/templates/select_platform.html new file mode 100644 index 0000000..d9bc68f --- /dev/null +++ b/uploader/templates/select_platform.html @@ -0,0 +1,82 @@ +{%extends "base.html"%} + +{%block title%}Select Dataset{%endblock%} + +{%block contents%} +<h2 class="heading">{{filename}}: select platform</h2> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.select_study')}}" + id="select-platform-form" data-genechips="{{genechips_data}}"> + <input type="hidden" name="filename" value="{{filename}}" /> + <input type="hidden" name="filetype" value="{{filetype}}" /> + <input type="hidden" name="totallines" value="{{totallines}}" /> + + <div class="form-group"> + <label for="species" class="form-label">species</label> + <select id="species" name="species" class="form-control"> + {%for row in species:%} + <option value="{{row['SpeciesId']}}" + {%if row["Name"] == default_species:%} + selected="selected" + {%endif%}> + {{row["MenuName"]}} + </option> + {%endfor%} + </select> + </div> + + <table id="genechips-table" class="table"> + <caption>select platform</caption> + <thead> + <tr> + <th>Select</th> + <th>GeneChip ID</th> + <th>GeneChip Name</th> + </tr> + </thead> + + <tbody> + {%for chip in genechips:%} + <tr> + <td> + <input type="radio" name="genechipid" value="{{chip['GeneChipId']}}" + required="required" /> + </td> + <td>{{chip["GeneChipId"]}}</td> + <td>{{chip["GeneChipName"]}}</td> + </tr> + {%else%} + <tr> + <td colspan="5">No chips found for selected species</td> + </tr> + {%endfor%} + </tbody> + </table> + + <button type="submit" class="btn btn-primary">submit platform</button> + </form> +</div> +{%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/utils.js"></script> +<script type="text/javascript" src="/static/js/select_platform.js"></script> +<script type="text/javascript"> + document.getElementById( + "species").addEventListener("change", update_genechips); + document.getElementById( + "genechips-table").getElementsByTagName( + "tbody")[0].addEventListener( + "click", + function(event) { + if(event.target.tagName.toLowerCase() == "td") { + return select_row_radio(event.target.parentElement); + } + if(event.target.tagName.toLowerCase() == "td") { + return select_row_radio(event.target); + } + return false; + }); +</script> +{%endblock%} diff --git a/uploader/templates/select_study.html b/uploader/templates/select_study.html new file mode 100644 index 0000000..648ad4c --- /dev/null +++ b/uploader/templates/select_study.html @@ -0,0 +1,108 @@ +{%extends "base.html"%} +{%from "dbupdate_hidden_fields.html" import hidden_fields%} + +{%block title%}Select Dataset{%endblock%} + +{%block css%} +<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> +{%endblock%} + +{%block contents%} +<h2 class="heading">{{filename}}: select study</h2> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.select_dataset')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col1"> + <legend class="heading">Select from existing study</legend> + {{hidden_fields(filename, filetype, species=species, genechipid=genechipid, + totallines=totallines)}} + + <div class="form-group"> + <label class="form-label" for="study">study:</label> + <select id="study" name="studyid" class="form-control"> + {%for study in studies:%} + <option value="{{study['Id']}}">{{study["Name"]}}</option> + {%endfor%} + </select> + </div> + + <button type="submit" + class="btn btn-primary" + {%if studies | length == 0:%} + disabled="disabled" + {%endif%} />submit selected study</button> +</form> +</div> + +<div class="row"> + <p class="two-col-sep-separator">OR</p> +</div> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.create_study')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col2"> + {%with messages = get_flashed_messages(with_categories=true)%} + {%if messages:%} + <ul> + {%for category, message in messages:%} + <li class="{{category}}">{{message}}</li> + {%endfor%} + </ul> + {%endif%} + {%endwith%} + <legend class="heading">Create new study</legend> + {{hidden_fields(filename, filetype, species=species, genechipid=genechipid, + totallines=totallines)}} + + <div class="form-group"> + <label class="form-label" for="studyname">name:</label> + <input type="text" id="studyname" name="studyname" class="form-control" + required="required" + {%if studyname:%} + value="{{studyname}}" + {%endif%} /> + </div> + + <div class="form-group"> + <label class="form-label" for="group">group:</label> + <select id="group" name="inbredsetid" class="form-control" + required="required"> + <option value="">Select group</option> + {%for family in groups:%} + <optgroup label="{{family}}"> + {%for group in groups[family]:%} + <option value="{{group['InbredSetId']}}" + {%if group["InbredSetId"] == selected_group:%} + selected="selected" + {%endif%}> + {{group["FullName"]}} + </option> + {%endfor%} + </optgroup> + {%endfor%} + </select> + </div> + + <div class="form-group"> + <label class="form-label" for="tissue">tissue:</label> + <select id="tissue" name="tissueid" class="form-control" + required="required"> + <option value="">Select type</option> + {%for tissue in tissues:%} + <option value="{{tissue['TissueId']}}" + {%if tissue["TissueId"] == selected_tissue:%} + selected="selected" + {%endif%}> + {{tissue["Name"]}} + </option> + {%endfor%} + </select> + </div> + + <button type="submit" class="btn btn-primary">create study</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/species/base.html b/uploader/templates/species/base.html new file mode 100644 index 0000000..f64f72b --- /dev/null +++ b/uploader/templates/species/base.html @@ -0,0 +1,17 @@ +{%extends "base.html"%} + +{%block lvl1_breadcrumbs%} +<li {%if activelink=="species"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + {%if species is mapping%} + <a href="{{url_for('species.view_species', species_id=species.SpeciesId)}}"> + {{species.Name}}</a> + {%else%} + <a href="{{url_for('species.list_species')}}">Species</a> + {%endif%} +</li> +{%block lvl2_breadcrumbs%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/species/create-species.html b/uploader/templates/species/create-species.html new file mode 100644 index 0000000..138dbaa --- /dev/null +++ b/uploader/templates/species/create-species.html @@ -0,0 +1,148 @@ +{%extends "species/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Create Species{%endblock%} + +{%block pagetitle%}Create Species{%endblock%} + +{%block lvl2_breadcrumbs%} +<li {%if activelink=="create-species"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.create_species')}}">Create</a> +</li> +{%endblock%} + +{%block contents%} +<div class="row"> + <form id="frm-create-species" + method="POST" + action="{{url_for('species.create_species', return_to=return_to)}}" + class="form-horizontal"> + <legend>Create Species</legend> + + {{flash_all_messages()}} + + <input type="hidden" name="return_to" value="{{return_to}}"> + + <div class="form-group"> + <label for="txt-taxonomy-id" class="control-label col-sm-2"> + Taxonomy ID</label> + <div class="col-sm-10"> + <div class="input-group"> + <input id="txt-taxonomy-id" + name="species_taxonomy_id" + type="text" + class="form-control" /> + <span class="input-group-btn"> + <button id="btn-search-taxonid" class="btn btn-info">Search</button> + </span> + </div> + <small class="form-text text-small text-muted"> + Use + <a href="https://www.ncbi.nlm.nih.gov/Taxonomy/taxonomyhome.html/" + title="NCBI's Taxonomy Browser homepage" + target="_blank"> + NCBI's Taxonomy Browser homepage</a> to search for the species you + want. If the species exists on NCBI, they will have a Taxonomy ID. Copy + that Taxonomy ID to this field, and click "Search" to auto-fill the + details.<br /> + This field is optional.</small> + </div> + </div> + + <div class="form-group"> + <label for="txt-species-name" class="control-label col-sm-2">Common Name</label> + <div class="col-sm-10"> + <input id="txt-species-name" + name="common_name" + type="text" + class="form-control" + required="required" /> + <small class="form-text text-muted">This is the day-to-day term used by + laymen, e.g. Mouse (instead of Mus musculus), round worm (instead of + Ascaris lumbricoides), etc.<br /> + For species without this, just enter the scientific name. + </small> + </div> + </div> + + <div class="form-group"> + <label for="txt-species-scientific" class="control-label col-sm-2"> + Scientific Name</label> + <div class="col-sm-10"> + <input id="txt-species-scientific" + name="scientific_name" + type="text" + class="form-control" + required="required" /> + <small class="form-text text-muted">This is the scientific name for the + species e.g. Homo sapiens, Mus musculus, etc.</small> + </div> + </div> + + <div class="form-group"> + <label for="select-species-family" class="control-label col-sm-2">Family</label> + <div class="col-sm-10"> + <select id="select-species-family" + name="species_family" + required="required" + class="form-control"> + <option value="ungrouped">I do not know what to pick</option> + {%for family in families%} + <option value="{{family}}">{{family}}</option> + {%endfor%} + </select> + <small class="form-text text-muted"> + This is a rough grouping of the species.</small> + </div> + </div> + + <div class="col-sm-offset-2 col-sm-10"> + <input type="submit" + value="create new species" + class="btn btn-primary" /> + </div> + + </form> +</div> +{%endblock%} + +{%block javascript%} +<script> + var lastTaxonId = null; + + var fetch_taxonomy = (taxonId) => { + var uri = ( + "https://rest.uniprot.org/taxonomy/" + encodeURIComponent(taxonId)); + $.get( + uri, + {}, + (data, textStatus, jqXHR) => { + if(textStatus == "success") { + lastTaxonId = taxonId; + $("#txt-species-scientific").val(data.scientificName); + $("#txt-species-name").val(data.commonName); + return false; + } + msg = ( + "Request to '${uri}' failed with message '${textStatus}'. " + + "Please try again later, or fill the details manually."); + alert(msg); + console.error(msg, data, textStatus); + return false; + }, + "json"); + }; + + $("#btn-search-taxonid").on("click", (event) => { + event.preventDefault(); + taxonId = $("#txt-taxonomy-id").val(); + if((taxonId !== "") && (taxonId !== lastTaxonId)) { + fetch_taxonomy(taxonId); + } + }); +</script> +{%endblock%} diff --git a/uploader/templates/species/edit-species.html b/uploader/templates/species/edit-species.html new file mode 100644 index 0000000..5a26455 --- /dev/null +++ b/uploader/templates/species/edit-species.html @@ -0,0 +1,177 @@ +{%extends "species/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Edit Species{%endblock%} + +{%block pagetitle%}Edit Species{%endblock%} + +{%block css%} +<style type="text/css"> + .card { + margin-top: 0.3em; + border-width: 1px; + border-style: solid; + border-radius: 0.3em; + border-color: #AAAAAA; + padding: 0.5em; + } +</style> +{%endblock%} + +{%block lvl2_breadcrumbs%} +<li {%if activelink=="edit-species"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.edit_species_extra', + species_id=species.SpeciesId)}}">Edit</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} +<div class="row"> + <form id="frm-edit-species" + method="POST" + action="{{url_for('species.edit_species_extra', + species_id=species.SpeciesId)}}"> + + <legend>Edit Extra Detail for Species '{{species.FullName}}'</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + + <div class="form-group"> + <label for="lbl-species-taxonid" class="form-label"> + Taxonomy Id + </label> + <label id="lbl-species-taxonid" + disabled="disabled" + class="form-control">{{species.TaxonomyId}}</label> + </div> + + <div class="form-group"> + <label for="txt-species-name" class="form-label"> + Common Name + </label> + <input type="text" + id="txt-species-name" + name="species_name" + required="required" + value="{{species.SpeciesName}}" + class="form-control" /> + <small class="form-text text-muted"> + This is the layman's name for the species, e.g. mouse</mall> + </div> + + <div class="form-group"> + <label for="txt-species-fullname" class="form-label"> + Scientific Name + </label> + <input type="text" + id="txt-species-fullname" + name="species_fullname" + required="required" + value="{{species.FullName}}" + class="form-control" /> + <small class="form-text text-muted"> + A scientific name for the species that mostly adheres to the biological + binomial nomenclature system.</small> + </div> + + <div class="form-group"> + <label for="select-species-family" class="form-label"> + Family + </label> + <select id="select-species-family" + name="species_family" + class="form-control"> + <option value="">Select the family</option> + {%for family in families%} + <option value="{{family}}" + {%if species.Family == family%} + selected="selected" + {%endif%}>{{family}}</option> + {%endfor%} + </select> + <small class="form-text text-muted"> + A general classification for the species. This is mostly for use in + GeneNetwork's menus.</small> + </div> + + <div class="form-group"> + <label for="txt-species-familyorderid" class="form-label"> + Family Order Id + </label> + <input type="number" + id="txt-species-familyorderid" + name="species_familyorderid" + value="{{species.FamilyOrderId}}" + required="required" + class="form-control" /> + <small class="form-text text-muted"> + This is a number that determines the order of the "Family" groupings + above in the GeneNetwork menus. This is an integer value that's manually + assigned.</small> + </div> + + <div class="form-group"> + <label for="txt-species-orderid" class="form-label"> + Order Id + </label> + <input type="number" + id="txt-species-orderid" + name="species_orderid" + value="{{species.OrderId or (max_order_id + 5)}}" + class="form-control" /> + <small class="form-text text-muted"> + This integer value determines the order of the species in relation to + each other, but also within the respective "Family" groups.</small> + </div> + + <div class="form-group"> + <input type="submit" value="Submit Changes" class="btn btn-primary" /> + </div> + + </form> +</div> +{%endblock%} + +{%block sidebarcontents%} + +<div class="card"> + <div class="card-body"> + <h5 class="card-title">Family Order</h5> + <div class="card-text"> + <p>The current family order is as follows</p> + <table class="table"> + <thead> + <tr> + <th>Family Order Id</th> + <th>Family</th> + </tr> + </thead> + <tbody> + {%for item in family_order%} + <tr> + <td>{{item[0]}}</td> + <td>{{item[1]}}</td> + </tr> + {%endfor%} + </tbody> + </table> + </div> + </div> +</div> + +<div class="card"> + <div class="card-body"> + <h5 class="card-title">Order ID</h5> + <div class="card-text"> + <p>The current largest OrderID is: {{max_order_id}}</p> + <p>We recommend giving a new species an order ID that is five more than + the current highest i.e. {{max_order_id + 5}}.</p> + </div> + </div> +</div> +{%endblock%} diff --git a/uploader/templates/species/list-species.html b/uploader/templates/species/list-species.html new file mode 100644 index 0000000..64084b0 --- /dev/null +++ b/uploader/templates/species/list-species.html @@ -0,0 +1,75 @@ +{%extends "species/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}List Species{%endblock%} + +{%block pagetitle%}List Species{%endblock%} + +{%block contents%} +{{flash_all_messages()}} +<div class="row"> + <p> + All data in GeneNetwork revolves around species. This is the core of the + system.</p> + <p>Here you can see a list of all the species available in GeneNetwork. + Click on the link besides each species to view greater detail on the species, + and access further operations that are possible for said species.</p> +</div> + +<div class="row"> + <p>If you cannot find the species you are looking for below, click the button + below to create it</p> + <p><a href="{{url_for('species.create_species')}}" + title="Add a new species to GeneNetwork" + class="btn btn-danger">Create Species</a></p> +</div> + +<div class="row"> + <table class="table"> + <caption>Available Species</caption> + <thead> + <tr> + <th></td> + <th title="A common, layman's name for the species.">Common Name</th> + <th title="The scientific name for the species">Organism Name</th> + <th title="An identifier for the species in the NCBI taxonomy database"> + Taxonomy ID + </th> + <th title="A generic grouping used internally by GeneNetwork for organising species."> + Family + </th> + </tr> + </thead> + <tbody> + {%for species in allspecies%} + <tr> + <td>{{species["sequence_number"]}}</td> + <td>{{species["SpeciesName"]}}</td> + <td> + <a href="{{url_for('species.view_species', + species_id=species['SpeciesId'])}}" + title="View details in GeneNetwork on {{species['FullName']}}"> + {{species["FullName"]}} + </a> + </td> + <td> + <a href="https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id={{species['TaxonomyId']}}" + title="View species details on NCBI" + target="_blank">{{species["TaxonomyId"]}}</a> + </td> + <td>{{species.Family}}</td> + </tr> + {%else%} + <tr> + <td colspan="3"> + <p class="text-danger"> + <span class="glyphicon glyphicon-exclamation-mark"></span> + There were no species found! + </p> + </td> + </tr> + {%endfor%} + </tbody> + </table> +</div> +{%endblock%} diff --git a/uploader/templates/species/macro-display-species-card.html b/uploader/templates/species/macro-display-species-card.html new file mode 100644 index 0000000..166c7b9 --- /dev/null +++ b/uploader/templates/species/macro-display-species-card.html @@ -0,0 +1,22 @@ +{%macro display_species_card(species)%} +<div class="card"> + <div class="card-body"> + <h5 class="card-title">Species</h5> + <div class="card-text"> + <table class="table"> + <tbody> + <tr> + <td>Common Name</td> + <td>{{species.SpeciesName}}</td> + </tr> + + <tr> + <td>Scientific Name</td> + <td>{{species.FullName}}</td> + </tr> + </tbody> + </table> + </div> + </div> +</div> +{%endmacro%} diff --git a/uploader/templates/species/macro-select-species.html b/uploader/templates/species/macro-select-species.html new file mode 100644 index 0000000..3714ae4 --- /dev/null +++ b/uploader/templates/species/macro-select-species.html @@ -0,0 +1,59 @@ +{%from "macro-step-indicator.html" import step_indicator%} + +{%macro select_species_form(form_action, species)%} +<form method="GET" action="{{form_action}}" class="form-horizontal"> + + <h2>{{step_indicator("1")}} What species do you want to work with?</h2> + + {%if species | length != 0%} + + <p class="form-text">Search for, and select the species from the table below + and click "Continue"</p> + + <div class="radio"> + <label for="rdo-cant-find-species" + style="font-weight: 1;"> + <input id="rdo-cant-find-species" type="radio" name="species_id" + value="CREATE-SPECIES" /> + I could not find the species I want (create it). + </label> + </div> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" + class="btn btn-primary" + value="continue" /> + </div> + + <div style="margin-top:3em;"> + <table id="tbl-select-species" class="table compact stripe" + data-species-list='{{species | tojson}}'> + <div class=""> + <thead> + <tr> + <th></th> + <th>Species Name</th> + </tr> + </thead> + + <tbody></tbody> + </table> + </div> + + {%else%} + + <label class="control-label" for="rdo-cant-find-species"> + <input id="rdo-cant-find-species" type="radio" name="species_id" + value="CREATE-SPECIES" /> + There are no species to select from. Create the first one.</label> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" + class="btn btn-primary col-sm-offset-1" + value="continue" /> + </div> + + {%endif%} + +</form> +{%endmacro%} diff --git a/uploader/templates/species/view-species.html b/uploader/templates/species/view-species.html new file mode 100644 index 0000000..2d02f7e --- /dev/null +++ b/uploader/templates/species/view-species.html @@ -0,0 +1,90 @@ +{%extends "species/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}View Species{%endblock%} + +{%block pagetitle%}View Species{%endblock%} + +{%block lvl2_breadcrumbs%} +<li {%if activelink=="view-species"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.view_species', species_id=species.SpeciesId)}}">View</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} +<div class="row"> + <h2>Details on species {{species.FullName}}</h2> + + <dl> + <dt>Common Name</dt> + <dd>{{species.SpeciesName}}</dd> + + <dt>Scientific Name</dt> + <dd>{{species.FullName}}</dd> + + <dt>Taxonomy ID</dt> + <dd>{{species.TaxonomyId}}</dd> + </dl> + + <h3>Actions</h3> + + <p> + You can proceed to perform any of the following actions for species + {{species.FullName}} + </p> + + <ol> + <li> + <a href="{{url_for('species.populations.list_species_populations', + species_id=species.SpeciesId)}}" + title="Create/Edit populations for {{species.FullName}}"> + Manage populations</a> + </li> + <li> + <a href="{{url_for('species.platforms.list_platforms', + species_id=species.SpeciesId)}}" + title="Create/Edit sequencing platforms for {{species.FullName}}"> + Manage sequencing platforms</a> + </li> + </ol> + + +</div> +{%endblock%} + +{%block sidebarcontents%} +<div class="card"> + <div class="card-body"> + <h5 class="card-title">Species Extras</h5> + <div class="card-text"> + <p>Some extra internal-use details (mostly for UI concerns on GeneNetwork)</p> + <p> + <small> + If you do not understand what the following are about, simply ignore them + — + They have no bearing whatsoever on your data, or its analysis. + </small> + </p> + <dl> + <dt>Family</dt> + <dd>{{species.Family}}</dd> + + <dt>FamilyOrderId</dt> + <dd>{{species.FamilyOrderId}}</dd> + + <dt>OrderId</dt> + <dd>{{species.OrderId}}</dd> + </dl> + </div> + <a href="{{url_for('species.edit_species_extra', + species_id=species.SpeciesId)}}" + class="card-link" + title="Edit the species' internal-use details.">Edit</a> + </div> +</div> +{%endblock%} diff --git a/uploader/templates/stdout_output.html b/uploader/templates/stdout_output.html new file mode 100644 index 0000000..85345a9 --- /dev/null +++ b/uploader/templates/stdout_output.html @@ -0,0 +1,8 @@ +{%macro stdout_output(job)%} + +<h4>STDOUT Output</h4> +<div class="cli-output"> + <pre>{{job.get("stdout", "")}}</pre> +</div> + +{%endmacro%} diff --git a/uploader/templates/unhandled_exception.html b/uploader/templates/unhandled_exception.html new file mode 100644 index 0000000..cfb0c0b --- /dev/null +++ b/uploader/templates/unhandled_exception.html @@ -0,0 +1,24 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}System Error{%endblock%} + +{%block css%} +<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> +{%endblock%} + +{%block contents%} +<div class="row"> + {{flash_all_messages()}} + <h1>Exception!</h1> + + <p>An error has occured, and your request has been aborted. Please notify the + administrator to try and get this fixed.</p> + <p>The system has failed with the following error:</p> +</div> +<div class="row"> + <pre> + {{trace}} + </pre> +</div> +{%endblock%} diff --git a/uploader/templates/upload_progress_indicator.html b/uploader/templates/upload_progress_indicator.html new file mode 100644 index 0000000..e274e83 --- /dev/null +++ b/uploader/templates/upload_progress_indicator.html @@ -0,0 +1,35 @@ +{%macro upload_progress_indicator()%} +<div id="upload-progress-indicator" class="modal fade" tabindex="-1" role="dialog"> + <div class="modal-dialog" role="document"> + <div class="modal-content"> + <div class="modal-header"> + <h3 class="modal-title">Uploading file</h3> + </div> + + <div class="modal-body"> + <form id="frm-cancel-upload" style="border-style: none;"> + <div class="form-group"> + <span id="progress-filename" class="form-text">No file selected!</span> + <progress id="progress-bar" value="0" max="100" class="form-control"> + 0</progress> + </div> + + <div class="form-group"> + <span class="form-text text-muted" id="progress-text"> + Uploading 0%</span> + <span class="form-text text-muted" id="progress-extra-text"> + Processing</span> + </div> + </form> + </div> + + <div class="modal-footer"> + <button id="btn-cancel-upload" + type="button" + class="btn btn-danger" + data-dismiss="modal">Cancel</button> + </div> + </div> + </div> +</div> +{%endmacro%} diff --git a/uploader/templates/worker_failure.html b/uploader/templates/worker_failure.html new file mode 100644 index 0000000..b65b140 --- /dev/null +++ b/uploader/templates/worker_failure.html @@ -0,0 +1,24 @@ +{%extends "base.html"%} + +{%block title%}Worker Failure{%endblock%} + +{%block contents%} +<h1 class="heading">Worker Failure</h1> + +<p> + There was a critical failure launching the job to parse your file. + This is our fault and (probably) has nothing to do with the file you uploaded. +</p> + +<p> + Please notify the developers of this issue when you encounter it, + providing the link to this page, or the information below. +</p> + +<h4>Debugging Information</h4> + +<ul> + <li><strong>job id</strong>: {{job_id}}</li> +</ul> + +{%endblock%} diff --git a/uploader/ui.py b/uploader/ui.py new file mode 100644 index 0000000..1994056 --- /dev/null +++ b/uploader/ui.py @@ -0,0 +1,14 @@ +"""Utilities to handle the UI""" +from flask import render_template as flask_render_template + +def make_template_renderer(default): + """Render template for species.""" + def render_template(template, **kwargs): + return flask_render_template( + template, + **{ + **kwargs, + "activemenu": default, + "activelink": kwargs.get("activelink", default) + }) + return render_template |
