aboutsummaryrefslogtreecommitdiff
path: root/uploader
diff options
context:
space:
mode:
Diffstat (limited to 'uploader')
-rw-r--r--uploader/__init__.py91
-rw-r--r--uploader/authorisation.py67
-rw-r--r--uploader/base_routes.py60
-rw-r--r--uploader/check_connections.py27
-rw-r--r--uploader/datautils.py38
-rw-r--r--uploader/db/__init__.py2
-rw-r--r--uploader/db/averaging.py23
-rw-r--r--uploader/db/datasets.py133
-rw-r--r--uploader/db/tissues.py50
-rw-r--r--uploader/db_utils.py20
-rw-r--r--uploader/default_settings.py20
-rw-r--r--uploader/errors.py29
-rw-r--r--uploader/expression_data/__init__.py2
-rw-r--r--uploader/expression_data/dbinsert.py400
-rw-r--r--uploader/expression_data/views.py385
-rw-r--r--uploader/files/__init__.py5
-rw-r--r--uploader/files/chunks.py32
-rw-r--r--uploader/files/functions.py42
-rw-r--r--uploader/files/views.py116
-rw-r--r--uploader/genotypes/__init__.py1
-rw-r--r--uploader/genotypes/models.py102
-rw-r--r--uploader/genotypes/views.py225
-rw-r--r--uploader/input_validation.py71
-rw-r--r--uploader/jobs.py164
-rw-r--r--uploader/monadic_requests.py104
-rw-r--r--uploader/oauth2/__init__.py1
-rw-r--r--uploader/oauth2/client.py231
-rw-r--r--uploader/oauth2/jwks.py86
-rw-r--r--uploader/oauth2/views.py138
-rw-r--r--uploader/phenotypes/__init__.py2
-rw-r--r--uploader/phenotypes/models.py256
-rw-r--r--uploader/phenotypes/views.py864
-rw-r--r--uploader/platforms/__init__.py2
-rw-r--r--uploader/platforms/models.py95
-rw-r--r--uploader/platforms/views.py118
-rw-r--r--uploader/population/__init__.py3
-rw-r--r--uploader/population/models.py87
-rw-r--r--uploader/population/rqtl2.py953
-rw-r--r--uploader/population/views.py217
-rw-r--r--uploader/request_checks.py75
-rw-r--r--uploader/samples/__init__.py1
-rw-r--r--uploader/samples/models.py104
-rw-r--r--uploader/samples/views.py292
-rw-r--r--uploader/session.py118
-rw-r--r--uploader/species/__init__.py2
-rw-r--r--uploader/species/models.py154
-rw-r--r--uploader/species/views.py210
-rw-r--r--uploader/static/css/custom-bootstrap.css23
-rw-r--r--uploader/static/css/styles.css154
-rw-r--r--uploader/static/css/two-column-with-separator.css27
-rw-r--r--uploader/static/images/CITGLogo.pngbin0 -> 11962 bytes
-rw-r--r--uploader/static/js/files.js118
-rw-r--r--uploader/static/js/misc.js6
-rw-r--r--uploader/static/js/populations.js40
-rw-r--r--uploader/static/js/select_platform.js70
-rw-r--r--uploader/static/js/species.js39
-rw-r--r--uploader/static/js/upload_progress.js97
-rw-r--r--uploader/static/js/upload_samples.js132
-rw-r--r--uploader/static/js/utils.js10
-rw-r--r--uploader/templates/base.html139
-rw-r--r--uploader/templates/cli-output.html8
-rw-r--r--uploader/templates/continue_from_create_dataset.html52
-rw-r--r--uploader/templates/continue_from_create_study.html52
-rw-r--r--uploader/templates/dbupdate_error.html12
-rw-r--r--uploader/templates/dbupdate_hidden_fields.html29
-rw-r--r--uploader/templates/errors_display.html47
-rw-r--r--uploader/templates/expression-data/base.html13
-rw-r--r--uploader/templates/expression-data/data-review.html85
-rw-r--r--uploader/templates/expression-data/index.html33
-rw-r--r--uploader/templates/expression-data/job-progress.html47
-rw-r--r--uploader/templates/expression-data/no-such-job.html15
-rw-r--r--uploader/templates/expression-data/parse-failure.html26
-rw-r--r--uploader/templates/expression-data/parse-results.html39
-rw-r--r--uploader/templates/expression-data/select-file.html115
-rw-r--r--uploader/templates/expression-data/select-population.html29
-rw-r--r--uploader/templates/final_confirmation.html47
-rw-r--r--uploader/templates/flash_messages.html25
-rw-r--r--uploader/templates/genotypes/base.html23
-rw-r--r--uploader/templates/genotypes/create-dataset.html82
-rw-r--r--uploader/templates/genotypes/index.html32
-rw-r--r--uploader/templates/genotypes/list-genotypes.html149
-rw-r--r--uploader/templates/genotypes/list-markers.html105
-rw-r--r--uploader/templates/genotypes/select-population.html25
-rw-r--r--uploader/templates/genotypes/view-dataset.html61
-rw-r--r--uploader/templates/http-error.html18
-rw-r--r--uploader/templates/index.html107
-rw-r--r--uploader/templates/insert_error.html32
-rw-r--r--uploader/templates/insert_progress.html46
-rw-r--r--uploader/templates/insert_success.html19
-rw-r--r--uploader/templates/login.html12
-rw-r--r--uploader/templates/macro-table-pagination.html26
-rw-r--r--uploader/templates/phenotypes/add-phenotypes-base.html331
-rw-r--r--uploader/templates/phenotypes/add-phenotypes-raw-files.html732
-rw-r--r--uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html207
-rw-r--r--uploader/templates/phenotypes/base.html19
-rw-r--r--uploader/templates/phenotypes/create-dataset.html108
-rw-r--r--uploader/templates/phenotypes/edit-phenotype.html332
-rw-r--r--uploader/templates/phenotypes/index.html21
-rw-r--r--uploader/templates/phenotypes/job-status.html155
-rw-r--r--uploader/templates/phenotypes/list-datasets.html68
-rw-r--r--uploader/templates/phenotypes/macro-display-pheno-dataset-card.html31
-rw-r--r--uploader/templates/phenotypes/macro-display-preview-table.html21
-rw-r--r--uploader/templates/phenotypes/macro-display-resumable-elements.html60
-rw-r--r--uploader/templates/phenotypes/review-job-data.html101
-rw-r--r--uploader/templates/phenotypes/select-population.html31
-rw-r--r--uploader/templates/phenotypes/view-dataset.html123
-rw-r--r--uploader/templates/phenotypes/view-phenotype.html135
-rw-r--r--uploader/templates/platforms/base.html13
-rw-r--r--uploader/templates/platforms/create-platform.html124
-rw-r--r--uploader/templates/platforms/index.html25
-rw-r--r--uploader/templates/platforms/list-platforms.html93
-rw-r--r--uploader/templates/populations/base.html18
-rw-r--r--uploader/templates/populations/create-population.html258
-rw-r--r--uploader/templates/populations/index.html28
-rw-r--r--uploader/templates/populations/list-populations.html93
-rw-r--r--uploader/templates/populations/macro-display-population-card.html41
-rw-r--r--uploader/templates/populations/macro-select-population.html52
-rw-r--r--uploader/templates/populations/rqtl2/create-tissue-success.html106
-rw-r--r--uploader/templates/populations/rqtl2/index.html54
-rw-r--r--uploader/templates/populations/rqtl2/no-such-job.html13
-rw-r--r--uploader/templates/populations/rqtl2/rqtl2-job-error.html39
-rw-r--r--uploader/templates/populations/rqtl2/rqtl2-job-results.html24
-rw-r--r--uploader/templates/populations/rqtl2/rqtl2-job-status.html20
-rw-r--r--uploader/templates/populations/rqtl2/rqtl2-qc-job-error.html120
-rw-r--r--uploader/templates/populations/rqtl2/rqtl2-qc-job-results.html66
-rw-r--r--uploader/templates/populations/rqtl2/rqtl2-qc-job-status.html41
-rw-r--r--uploader/templates/populations/rqtl2/rqtl2-qc-job-success.html37
-rw-r--r--uploader/templates/populations/rqtl2/select-geno-dataset.html69
-rw-r--r--uploader/templates/populations/rqtl2/select-population.html57
-rw-r--r--uploader/templates/populations/rqtl2/select-probeset-dataset.html191
-rw-r--r--uploader/templates/populations/rqtl2/select-probeset-study-id.html143
-rw-r--r--uploader/templates/populations/rqtl2/select-tissue.html115
-rw-r--r--uploader/templates/populations/rqtl2/summary-info.html65
-rw-r--r--uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-01.html276
-rw-r--r--uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-02.html33
-rw-r--r--uploader/templates/populations/view-population.html102
-rw-r--r--uploader/templates/samples/base.html12
-rw-r--r--uploader/templates/samples/index.html23
-rw-r--r--uploader/templates/samples/list-samples.html132
-rw-r--r--uploader/templates/samples/select-population.html26
-rw-r--r--uploader/templates/samples/upload-failure.html37
-rw-r--r--uploader/templates/samples/upload-progress.html31
-rw-r--r--uploader/templates/samples/upload-samples.html160
-rw-r--r--uploader/templates/samples/upload-success.html36
-rw-r--r--uploader/templates/select_dataset.html161
-rw-r--r--uploader/templates/select_platform.html82
-rw-r--r--uploader/templates/select_study.html108
-rw-r--r--uploader/templates/species/base.html17
-rw-r--r--uploader/templates/species/create-species.html148
-rw-r--r--uploader/templates/species/edit-species.html177
-rw-r--r--uploader/templates/species/list-species.html75
-rw-r--r--uploader/templates/species/macro-display-species-card.html22
-rw-r--r--uploader/templates/species/macro-select-species.html59
-rw-r--r--uploader/templates/species/view-species.html90
-rw-r--r--uploader/templates/stdout_output.html8
-rw-r--r--uploader/templates/unhandled_exception.html24
-rw-r--r--uploader/templates/upload_progress_indicator.html35
-rw-r--r--uploader/templates/worker_failure.html24
-rw-r--r--uploader/ui.py14
159 files changed, 14776 insertions, 0 deletions
diff --git a/uploader/__init__.py b/uploader/__init__.py
new file mode 100644
index 0000000..cae531b
--- /dev/null
+++ b/uploader/__init__.py
@@ -0,0 +1,91 @@
+"""The Quality-Control Web Application entry point"""
+import os
+import sys
+import logging
+from pathlib import Path
+
+from flask import Flask, request
+from flask_session import Session
+
+from uploader.oauth2.client import user_logged_in, authserver_authorise_uri
+
+from . import session
+from .base_routes import base
+from .files.views import files
+from .species import speciesbp
+from .oauth2.views import oauth2
+from .expression_data import exprdatabp
+from .errors import register_error_handlers
+
+def override_settings_with_envvars(
+ app: Flask, ignore: tuple[str, ...]=tuple()) -> None:
+ """Override settings in `app` with those in ENVVARS"""
+ for setting in (key for key in app.config if key not in ignore):
+ app.config[setting] = os.environ.get(setting) or app.config[setting]
+
+
+def __log_gunicorn__(app: Flask) -> Flask:
+ """Set up logging for the WSGI environment with GUnicorn"""
+ logger = logging.getLogger("gunicorn.error")
+ app.logger.handlers = logger.handlers
+ app.logger.setLevel(logger.level)
+ return app
+
+
+def __log_dev__(app: Flask) -> Flask:
+ """Set up logging for the development environment."""
+ stderr_handler = logging.StreamHandler(stream=sys.stderr)
+ app.logger.addHandler(stderr_handler)
+
+ root_logger = logging.getLogger()
+ root_logger.addHandler(stderr_handler)
+ root_logger.setLevel(app.config["LOG_LEVEL"])
+
+ return app
+
+
+def setup_logging(app: Flask) -> Flask:
+ """Set up logging for the application."""
+ software, *_version_and_comments = os.environ.get(
+ "SERVER_SOFTWARE", "").split('/')
+ return __log_gunicorn__(app) if bool(software) else __log_dev__(app)
+
+
+def create_app():
+ """The application factory"""
+ app = Flask(__name__)
+ app.config.from_pyfile(
+ Path(__file__).parent.joinpath("default_settings.py"))
+ if "UPLOADER_CONF" in os.environ:
+ app.config.from_envvar("UPLOADER_CONF") # Override defaults with instance path
+
+ override_settings_with_envvars(app, ignore=tuple())
+
+ secretsfile = app.config.get("UPLOADER_SECRETS", "").strip()
+ if bool(secretsfile):
+ secretsfile = Path(secretsfile).absolute()
+ app.config["UPLOADER_SECRETS"] = secretsfile
+ if secretsfile.exists():
+ # Silently ignore secrets if the file does not exist.
+ app.config.from_pyfile(secretsfile)
+
+ setup_logging(app)
+
+ # setup jinja2 symbols
+ app.add_template_global(lambda : request.url, name="request_url")
+ app.add_template_global(authserver_authorise_uri)
+ app.add_template_global(lambda: app.config["GN2_SERVER_URL"],
+ name="gn2server_uri")
+ app.add_template_global(user_logged_in)
+ app.add_template_global(lambda : session.user_details()["email"], name="user_email")
+
+ Session(app)
+
+ # setup blueprints
+ app.register_blueprint(base, url_prefix="/")
+ app.register_blueprint(files, url_prefix="/files")
+ app.register_blueprint(oauth2, url_prefix="/oauth2")
+ app.register_blueprint(speciesbp, url_prefix="/species")
+
+ register_error_handlers(app)
+ return app
diff --git a/uploader/authorisation.py b/uploader/authorisation.py
new file mode 100644
index 0000000..a283980
--- /dev/null
+++ b/uploader/authorisation.py
@@ -0,0 +1,67 @@
+"""Authorisation utilities."""
+import logging
+from functools import wraps
+
+from typing import Callable
+from flask import flash, redirect
+from pymonad.either import Left, Right, Either
+from authlib.jose import KeySet, JsonWebToken
+from authlib.jose.errors import BadSignatureError
+
+from uploader import session
+from uploader.oauth2.client import auth_server_jwks
+
+def require_login(function):
+ """Check that the user is logged in before executing `func`."""
+ @wraps(function)
+ def __is_session_valid__(*args, **kwargs):
+ """Check that the user is logged in and their token is valid."""
+ def __clear_session__(_no_token):
+ session.clear_session_info()
+ flash("You need to be signed in.", "alert-danger big-alert")
+ return redirect("/")
+
+ return session.user_token().either(
+ __clear_session__,
+ lambda token: function(*args, **kwargs))
+ return __is_session_valid__
+
+
+def __validate_token__(jwks: KeySet, token: dict) -> Either:
+ """Check that a token is signed by a key from the authorisation server."""
+ for key in jwks.keys:
+ try:
+ # Fixes CVE-2016-10555. See
+ # https://docs.authlib.org/en/latest/jose/jwt.html
+ jwt = JsonWebToken(["RS256"])
+ jwt.decode(token["access_token"], key)
+ return Right(token)
+ except BadSignatureError:
+ pass
+
+ return Left({"token": token})
+
+
+def require_token(func: Callable) -> Callable:
+ """
+ Wrap functions that require the user be authorised to perform the operations
+ that the functions in question provide.
+ """
+ def __invalid_token__(_whatever):
+ logging.debug("==========> Failure log: %s", _whatever)
+ raise Exception(
+ "You attempted to access a feature of the system that requires "
+ "authorisation. Unfortunately, we could not verify you have the "
+ "appropriate authorisation to perform the action you requested. "
+ "You might need to log in, or if you already are logged in, you "
+ "need to log out, then log back in to get a newer token/session.")
+ @wraps(func)
+ def __wrapper__(*args, **kwargs):
+ return session.user_token().then(lambda tok: {
+ "jwks": auth_server_jwks(),
+ "token": tok
+ }).then(lambda vals: __validate_token__(**vals)).either(
+ __invalid_token__,
+ lambda tok: func(*args, **{**kwargs, "token": tok}))
+
+ return __wrapper__
diff --git a/uploader/base_routes.py b/uploader/base_routes.py
new file mode 100644
index 0000000..326086f
--- /dev/null
+++ b/uploader/base_routes.py
@@ -0,0 +1,60 @@
+"""Basic routes required for all pages"""
+import os
+from urllib.parse import urljoin
+
+from flask import (Blueprint,
+ current_app as app,
+ send_from_directory)
+
+from uploader.ui import make_template_renderer
+from uploader.oauth2.client import user_logged_in
+
+base = Blueprint("base", __name__)
+render_template = make_template_renderer("home")
+
+
+@base.route("/favicon.ico", methods=["GET"])
+def favicon():
+ """Return the favicon."""
+ return send_from_directory(os.path.join(app.root_path, "static"),
+ "images/CITGLogo.png",
+ mimetype="image/png")
+
+
+@base.route("/", methods=["GET"])
+def index():
+ """Load the landing page"""
+ return render_template("index.html" if user_logged_in() else "login.html",
+ gn2server_intro=urljoin(app.config["GN2_SERVER_URL"],
+ "/intro"))
+
+def appenv():
+ """Get app's guix environment path."""
+ return os.environ.get("GN_UPLOADER_ENVIRONMENT")
+
+@base.route("/bootstrap/<path:filename>")
+def bootstrap(filename):
+ """Fetch bootstrap files."""
+ return send_from_directory(
+ appenv(), f"share/genenetwork2/javascript/bootstrap/{filename}")
+
+
+@base.route("/jquery/<path:filename>")
+def jquery(filename):
+ """Fetch jquery files."""
+ return send_from_directory(
+ appenv(), f"share/genenetwork2/javascript/jquery/{filename}")
+
+
+@base.route("/datatables/<path:filename>")
+def datatables(filename):
+ """Fetch DataTables files."""
+ return send_from_directory(
+ appenv(), f"share/genenetwork2/javascript/DataTables/{filename}")
+
+
+@base.route("/node-modules/<path:filename>")
+def node_modules(filename):
+ """Fetch node-js modules."""
+ return send_from_directory(
+ appenv(), f"lib/node_modules/{filename}")
diff --git a/uploader/check_connections.py b/uploader/check_connections.py
new file mode 100644
index 0000000..c9b9aa3
--- /dev/null
+++ b/uploader/check_connections.py
@@ -0,0 +1,27 @@
+"""Check the various connection used in the application"""
+import sys
+import traceback
+
+import redis
+import MySQLdb
+from gn_libs.mysqldb import database_connection
+
+def check_redis(uri: str):
+ "Check the redis connection"
+ try:
+ with redis.Redis.from_url(uri) as rconn:
+ rconn.ping()
+ except redis.exceptions.ConnectionError as conn_err:
+ print(conn_err, file=sys.stderr)
+ print(traceback.format_exc(), file=sys.stderr)
+ sys.exit(1)
+
+def check_db(uri: str):
+ "Check the mysql connection"
+ try:
+ with database_connection(uri) as dbconn: # pylint: disable=[unused-variable]
+ pass
+ except MySQLdb.OperationalError as op_err:
+ print(op_err, file=sys.stderr)
+ print(traceback.format_exc(), file=sys.stderr)
+ sys.exit(1)
diff --git a/uploader/datautils.py b/uploader/datautils.py
new file mode 100644
index 0000000..46a55c4
--- /dev/null
+++ b/uploader/datautils.py
@@ -0,0 +1,38 @@
+"""Generic data utilities: Rename module."""
+import math
+from functools import reduce
+from typing import Union, Sequence
+
+def enumerate_sequence(seq: Sequence[dict], start:int = 1) -> Sequence[dict]:
+ """Enumerate sequence beginning at 1"""
+ return tuple({**item, "sequence_number": seqno}
+ for seqno, item in enumerate(seq, start=start))
+
+
+def order_by_family(items: tuple[dict, ...],
+ family_key: str = "Family",
+ order_key: str = "FamilyOrderId") -> list:
+ """Order the populations by their families."""
+ def __family_order__(item):
+ orderval = item[order_key]
+ return math.inf if orderval is None else orderval
+
+ def __order__(ordered, current):
+ _key = (__family_order__(current), current[family_key])
+ return {
+ **ordered,
+ _key: ordered.get(_key, tuple()) + (current,)
+ }
+
+ return sorted(tuple(reduce(__order__, items, {}).items()),
+ key=lambda item: item[0][0])
+
+
+def safe_int(val: Union[str, int, float]) -> int:
+ """
+ Convert val into an integer: if val cannot be converted, return a zero.
+ """
+ try:
+ return int(val)
+ except ValueError:
+ return 0
diff --git a/uploader/db/__init__.py b/uploader/db/__init__.py
new file mode 100644
index 0000000..d2b1d9d
--- /dev/null
+++ b/uploader/db/__init__.py
@@ -0,0 +1,2 @@
+"""Database functions"""
+from .datasets import geno_datasets_by_species_and_population
diff --git a/uploader/db/averaging.py b/uploader/db/averaging.py
new file mode 100644
index 0000000..62bbe67
--- /dev/null
+++ b/uploader/db/averaging.py
@@ -0,0 +1,23 @@
+"""Functions for db interactions for averaging methods"""
+from typing import Optional
+
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+def averaging_methods(conn: mdb.Connection) -> tuple[dict, ...]:
+ """Fetch all available averaging methods"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM AvgMethod")
+ return tuple(dict(row) for row in cursor.fetchall())
+
+def averaging_method_by_id(
+ conn: mdb.Connection, averageid: int) -> Optional[dict]:
+ """Fetch the averaging method by its ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM AvgMethod WHERE Id=%s",
+ (averageid,))
+ result = cursor.fetchone()
+ if bool(result):
+ return dict(result)
+
+ return None
diff --git a/uploader/db/datasets.py b/uploader/db/datasets.py
new file mode 100644
index 0000000..767ec41
--- /dev/null
+++ b/uploader/db/datasets.py
@@ -0,0 +1,133 @@
+"""Functions for accessing the database relating to datasets."""
+from datetime import date
+from typing import Optional
+
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+def geno_datasets_by_species_and_population(
+ conn: mdb.Connection,
+ speciesid: int,
+ populationid: int) -> tuple[dict, ...]:
+ """Retrieve all genotypes datasets by species and population"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT gf.* FROM InbredSet AS iset INNER JOIN GenoFreeze AS gf "
+ "ON iset.InbredSetId=gf.InbredSetId "
+ "WHERE iset.SpeciesId=%(sid)s AND iset.InbredSetId=%(pid)s",
+ {"sid": speciesid, "pid": populationid})
+ return tuple(dict(row) for row in cursor.fetchall())
+
+def geno_dataset_by_id(conn: mdb.Connection, dataset_id) -> Optional[dict]:
+ """Retrieve genotype dataset by ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s", (dataset_id,))
+ _dataset = cursor.fetchone()
+ return dict(_dataset) if bool(_dataset) else None
+
+def probeset_studies_by_species_and_population(
+ conn: mdb.Connection,
+ speciesid: int,
+ populationid: int) -> tuple[dict, ...]:
+ """Retrieve all probesets"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT pf.* FROM InbredSet AS iset INNER JOIN ProbeFreeze AS pf "
+ "ON iset.InbredSetId=pf.InbredSetId "
+ "WHERE iset.SpeciesId=%(sid)s AND iset.InbredSetId=%(pid)s",
+ {"sid": speciesid, "pid": populationid})
+ return tuple(dict(row) for row in cursor.fetchall())
+
+def probeset_datasets_by_study(conn: mdb.Connection,
+ studyid: int) -> tuple[dict, ...]:
+ """Retrieve all probeset databases by study."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s",
+ (studyid,))
+ return tuple(dict(row) for row in cursor.fetchall())
+
+def probeset_study_by_id(conn: mdb.Connection, studyid) -> Optional[dict]:
+ """Retrieve ProbeSet study by ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM ProbeFreeze WHERE Id=%s", (studyid,))
+ _study = cursor.fetchone()
+ return dict(_study) if bool(_study) else None
+
+def probeset_create_study(conn: mdb.Connection,#pylint: disable=[too-many-arguments]
+ populationid: int,
+ platformid: int,
+ tissueid: int,
+ studyname: str,
+ studyfullname: str = "",
+ studyshortname: str = ""):
+ """Create a new ProbeSet study."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ studydata = {
+ "platid": platformid,
+ "tissueid": tissueid,
+ "name": studyname,
+ "fname": studyfullname or studyname,
+ "sname": studyshortname,
+ "today": date.today().isoformat(),
+ "popid": populationid
+ }
+ cursor.execute(
+ """
+ INSERT INTO ProbeFreeze(
+ ChipId, TissueId, Name, FullName, ShortName, CreateTime,
+ InbredSetId
+ ) VALUES (
+ %(platid)s, %(tissueid)s, %(name)s, %(fname)s, %(sname)s,
+ %(today)s, %(popid)s
+ )
+ """,
+ studydata)
+ studyid = cursor.lastrowid
+ cursor.execute("UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s",
+ (studyid, studyid))
+ return {**studydata, "studyid": studyid}
+
+def probeset_create_dataset(conn: mdb.Connection,#pylint: disable=[too-many-arguments]
+ studyid: int,
+ averageid: int,
+ datasetname: str,
+ datasetfullname: str,
+ datasetshortname: str="",
+ public: bool = True,
+ datascale="log2") -> dict:
+ """Create a new ProbeSet dataset."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ dataset = {
+ "studyid": studyid,
+ "averageid": averageid,
+ "name2": datasetname,
+ "fname": datasetfullname,
+ "name": datasetshortname,
+ "sname": datasetshortname,
+ "today": date.today().isoformat(),
+ "public": 2 if public else 0,
+ "authorisedusers": "williamslab",
+ "datascale": datascale
+ }
+ cursor.execute(
+ """
+ INSERT INTO ProbeSetFreeze(
+ ProbeFreezeId, AvgId, Name, Name2, FullName, ShortName,
+ CreateTime, public, AuthorisedUsers, DataScale)
+ VALUES(
+ %(studyid)s, %(averageid)s, %(name)s, %(name2)s, %(fname)s,
+ %(sname)s, %(today)s, %(public)s, %(authorisedusers)s,
+ %(datascale)s)
+ """,
+ dataset)
+ return {**dataset, "datasetid": cursor.lastrowid}
+
+def probeset_dataset_by_id(conn: mdb.Connection, datasetid) -> Optional[dict]:
+ """Fetch a ProbeSet dataset by its ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Id=%s", (datasetid,))
+ result = cursor.fetchone()
+ if bool(result):
+ return dict(result)
+
+ return None
diff --git a/uploader/db/tissues.py b/uploader/db/tissues.py
new file mode 100644
index 0000000..9fe7bab
--- /dev/null
+++ b/uploader/db/tissues.py
@@ -0,0 +1,50 @@
+"""Handle db interactions for tissue."""
+from typing import Union, Optional
+
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+def all_tissues(conn: mdb.Connection) -> tuple[dict, ...]:
+ """All available tissue."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM Tissue ORDER BY TissueName")
+ return tuple(dict(row) for row in cursor.fetchall())
+
+
+def tissue_by_id(conn: mdb.Connection, tissueid) -> Optional[dict]:
+ """Retrieve a tissue by its ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM Tissue WHERE Id=%s", (tissueid,))
+ result = cursor.fetchone()
+ if bool(result):
+ return dict(result)
+
+ return None
+
+
+def create_new_tissue(
+ conn: mdb.Connection,
+ name: str,
+ shortname: str,
+ birnlexid: Optional[str] = None,
+ birnlexname: Optional[str] = None
+) -> dict[str, Union[int, str, None]]:
+ """Add a new tissue, organ or biological material to the database."""
+ with conn.cursor() as cursor:
+ cursor.execute(
+ "INSERT INTO "
+ "Tissue(TissueName, Name, Short_Name, BIRN_lex_ID, BIRN_lex_Name) "
+ "VALUES (%s, %s, %s, %s, %s)",
+ (name, name, shortname, birnlexid, birnlexname))
+ tissueid = cursor.lastrowid
+ cursor.execute("UPDATE Tissue SET TissueId=%s WHERE Id=%s",
+ (tissueid, tissueid))
+ return {
+ "Id": tissueid,
+ "TissueId": tissueid,
+ "TissueName": name,
+ "Name": name,
+ "Short_Name": shortname,
+ "BIRN_lex_ID": birnlexid,
+ "BIRN_lex_Name": birnlexname
+ }
diff --git a/uploader/db_utils.py b/uploader/db_utils.py
new file mode 100644
index 0000000..d9d521e
--- /dev/null
+++ b/uploader/db_utils.py
@@ -0,0 +1,20 @@
+"""module contains all db related stuff"""
+from typing import Any, Callable
+
+import MySQLdb as mdb
+from redis import Redis
+from flask import current_app as app
+from gn_libs.mysqldb import database_connection
+
+
+def with_db_connection(func: Callable[[mdb.Connection], Any]) -> Any:
+ """Call `func` with a MySQDdb database connection."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ return func(conn)
+
+
+def with_redis_connection(func: Callable[[Redis], Any]) -> Any:
+ """Call `func` with a redis connection."""
+ redisuri = app.config["REDIS_URL"]
+ with Redis.from_url(redisuri, decode_responses=True) as rconn:
+ return func(rconn)
diff --git a/uploader/default_settings.py b/uploader/default_settings.py
new file mode 100644
index 0000000..1acb247
--- /dev/null
+++ b/uploader/default_settings.py
@@ -0,0 +1,20 @@
+"""
+The default configuration file. The values here should be overridden in the
+actual configuration file used for the production and staging systems.
+"""
+LOG_LEVEL = "WARNING"
+SECRET_KEY = b"<Please! Please! Please! Change This!>"
+UPLOAD_FOLDER = "/tmp/qc_app_files"
+REDIS_URL = "redis://"
+JOBS_TTL_SECONDS = 1209600 # 14 days
+GNQC_REDIS_PREFIX="gn-uploader"
+SQL_URI = ""
+
+GN2_SERVER_URL = "https://genenetwork.org/"
+
+SESSION_TYPE = "redis"
+SESSION_PERMANENT = True
+SESSION_USE_SIGNER = True
+
+JWKS_ROTATION_AGE_DAYS = 7 # Days (from creation) to keep a JWK in use.
+JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before deleting it.
diff --git a/uploader/errors.py b/uploader/errors.py
new file mode 100644
index 0000000..3e7c893
--- /dev/null
+++ b/uploader/errors.py
@@ -0,0 +1,29 @@
+"""Application error handling."""
+import traceback
+from werkzeug.exceptions import HTTPException
+
+import MySQLdb as mdb
+from flask import Flask, request, render_template, current_app as app
+
+def handle_general_exception(exc: Exception):
+ """Handle generic exceptions."""
+ trace = traceback.format_exc()
+ app.logger.error(
+ "Error (%s.%s): Generic unhandled exception!! (URI: %s)\n%s",
+ exc.__class__.__module__, exc.__class__.__name__, request.url, trace)
+ return render_template("unhandled_exception.html", trace=trace), 500
+
+def handle_http_exception(exc: HTTPException):
+ """Handle HTTP exceptions."""
+ app.logger.error(
+ "HTTP Error %s: %s", exc.code, exc.description, exc_info=True)
+ return render_template("http-error.html",
+ request_url=request.url,
+ exc=exc,
+ trace=traceback.format_exception(exc)), exc.code
+
+def register_error_handlers(appl: Flask):
+ """Register top-level error/exception handlers."""
+ appl.register_error_handler(Exception, handle_general_exception)
+ appl.register_error_handler(HTTPException, handle_http_exception)
+ appl.register_error_handler(mdb.MySQLError, handle_general_exception)
diff --git a/uploader/expression_data/__init__.py b/uploader/expression_data/__init__.py
new file mode 100644
index 0000000..fc8bd41
--- /dev/null
+++ b/uploader/expression_data/__init__.py
@@ -0,0 +1,2 @@
+"""Package handling upload of files."""
+from .views import exprdatabp
diff --git a/uploader/expression_data/dbinsert.py b/uploader/expression_data/dbinsert.py
new file mode 100644
index 0000000..6d8ce80
--- /dev/null
+++ b/uploader/expression_data/dbinsert.py
@@ -0,0 +1,400 @@
+"Handle inserting data into the database"
+import os
+import json
+from typing import Union
+from functools import reduce
+from datetime import datetime
+
+from redis import Redis
+from MySQLdb.cursors import DictCursor
+from gn_libs.mysqldb import database_connection
+from flask import (
+ flash, request, url_for, Blueprint, redirect, render_template,
+ current_app as app)
+
+from uploader import jobs
+from uploader.authorisation import require_login
+from uploader.db_utils import with_db_connection
+from uploader.population.models import populations_by_species
+from uploader.species.models import all_species, species_by_id
+from uploader.platforms.models import platform_by_species_and_id
+
+dbinsertbp = Blueprint("dbinsert", __name__)
+
+def render_error(error_msg):
+ "Render the generic error page"
+ return render_template("dbupdate_error.html", error_message=error_msg), 400
+
+def make_menu_items_grouper(grouping_fn=lambda item: item):
+ "Build function to be used to group menu items."
+ def __grouper__(acc, row):
+ grouping = grouping_fn(row[2])
+ row_values = (row[0].strip(), row[1].strip())
+ if acc.get(grouping) is None:
+ return {**acc, grouping: (row_values,)}
+ return {**acc, grouping: (acc[grouping] + (row_values,))}
+ return __grouper__
+
+def genechips():
+ "Retrieve the genechip information from the database"
+ def __organise_by_species__(acc, chip):
+ speciesid = chip["SpeciesId"]
+ if acc.get(speciesid) is None:
+ return {**acc, speciesid: (chip,)}
+ return {**acc, speciesid: acc[speciesid] + (chip,)}
+
+ with database_connection(app.config["SQL_URI"]) as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM GeneChip ORDER BY GeneChipName ASC")
+ return reduce(__organise_by_species__, cursor.fetchall(), {})
+
+ return {}
+
+
+def studies_by_species_and_platform(speciesid:int, genechipid:int) -> tuple:
+ "Retrieve the studies by the related species and gene platform"
+ with database_connection(app.config["SQL_URI"]) as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ query = (
+ "SELECT Species.SpeciesId, ProbeFreeze.* "
+ "FROM Species INNER JOIN InbredSet "
+ "ON Species.SpeciesId=InbredSet.SpeciesId "
+ "INNER JOIN ProbeFreeze "
+ "ON InbredSet.InbredSetId=ProbeFreeze.InbredSetId "
+ "WHERE Species.SpeciesId = %s "
+ "AND ProbeFreeze.ChipId = %s")
+ cursor.execute(query, (speciesid, genechipid))
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+def organise_groups_by_family(acc:dict, group:dict) -> dict:
+ "Organise the group (InbredSet) information by the group field"
+ family = group["Family"]
+ if acc.get(family):
+ return {**acc, family: acc[family] + (group,)}
+ return {**acc, family: (group,)}
+
+def tissues() -> tuple:
+ "Retrieve type (Tissue) information from the database."
+ with database_connection(app.config["SQL_URI"]) as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM Tissue ORDER BY Name")
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+@dbinsertbp.route("/platform", methods=["POST"])
+@require_login
+def select_platform():
+ "Select the platform (GeneChipId) used for the data."
+ job_id = request.form["job_id"]
+ with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn,
+ database_connection(app.config["SQL_URI"]) as conn):
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+ if job:
+ filename = job["filename"]
+ filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}"
+ if os.path.exists(filepath):
+ default_species = 1
+ gchips = genechips()
+ return render_template(
+ "select_platform.html", filename=filename,
+ filetype=job["filetype"], totallines=int(job["currentline"]),
+ default_species=default_species, species=all_species(conn),
+ genechips=gchips[default_species],
+ genechips_data=json.dumps(gchips))
+ return render_error(f"File '{filename}' no longer exists.")
+ return render_error(f"Job '{job_id}' no longer exists.")
+ return render_error("Unknown error")
+
+@dbinsertbp.route("/study", methods=["POST"])
+@require_login
+def select_study():
+ "View to select/create the study (ProbeFreeze) associated with the data."
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+
+ speciesid = form["species"]
+ genechipid = form["genechipid"]
+
+ the_studies = studies_by_species_and_platform(speciesid, genechipid)
+ the_groups = reduce(
+ organise_groups_by_family,
+ with_db_connection(
+ lambda conn: populations_by_species(conn, speciesid)),
+ {})
+ return render_template(
+ "select_study.html", filename=form["filename"],
+ filetype=form["filetype"], totallines=form["totallines"],
+ species=speciesid, genechipid=genechipid, studies=the_studies,
+ groups=the_groups, tissues = tissues(),
+ selected_group=int(form.get("inbredsetid", -13)),
+ selected_tissue=int(form.get("tissueid", -13)))
+ except AssertionError as aserr:
+ return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/create-study", methods=["POST"])
+@require_login
+def create_study():
+ "Create a new study (ProbeFreeze)."
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+ assert form.get("studyname"), "study name"
+ assert form.get("inbredsetid"), "group"
+ assert form.get("tissueid"), "type/tissue"
+
+ with database_connection(app.config["SQL_URI"]) as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ values = (
+ form["genechipid"],
+ form["tissueid"],
+ form["studyname"],
+ form.get("studyfullname", ""),
+ form.get("studyshortname", ""),
+ datetime.now().date().strftime("%Y-%m-%d"),
+ form["inbredsetid"])
+ query = (
+ "INSERT INTO ProbeFreeze("
+ "ChipId, TissueId, Name, FullName, ShortName, CreateTime, "
+ "InbredSetId"
+ ") VALUES (%s, %s, %s, %s, %s, %s, %s)")
+ cursor.execute(query, values)
+ new_studyid = cursor.lastrowid
+ cursor.execute(
+ "UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s",
+ (new_studyid, new_studyid))
+ flash("Study created successfully", "alert-success")
+ return render_template(
+ "continue_from_create_study.html",
+ filename=form["filename"], filetype=form["filetype"],
+ totallines=form["totallines"], species=form["species"],
+ genechipid=form["genechipid"], studyid=new_studyid)
+ except AssertionError as aserr:
+ flash(f"Missing data: {aserr.args[0]}", "alert-error")
+ return redirect(url_for("dbinsert.select_study"), code=307)
+
+def datasets_by_study(studyid:int) -> tuple:
+ "Retrieve datasets associated with a study with the ID `studyid`."
+ with database_connection(app.config["SQL_URI"]) as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ query = "SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s"
+ cursor.execute(query, (studyid,))
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+def averaging_methods() -> tuple:
+ "Retrieve averaging methods from database"
+ with database_connection(app.config["SQL_URI"]) as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM AvgMethod")
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+def dataset_datascales() -> tuple:
+ "Retrieve datascales from database"
+ with database_connection(app.config["SQL_URI"]) as conn:
+ with conn.cursor() as cursor:
+ cursor.execute(
+ 'SELECT DISTINCT DataScale FROM ProbeSetFreeze '
+ 'WHERE DataScale IS NOT NULL AND DataScale != ""')
+ return tuple(
+ item for item in
+ (res[0].strip() for res in cursor.fetchall())
+ if (item is not None and item != ""))
+
+ return tuple()
+
+@dbinsertbp.route("/dataset", methods=["POST"])
+@require_login
+def select_dataset():
+ "Select the dataset to add the file contents against"
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+ assert form.get("studyid"), "study"
+
+ studyid = form["studyid"]
+ datasets = datasets_by_study(studyid)
+ return render_template(
+ "select_dataset.html", **{**form, "studyid": studyid},
+ datasets=datasets, avgmethods=averaging_methods(),
+ datascales=dataset_datascales())
+ except AssertionError as aserr:
+ return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/create-dataset", methods=["POST"])
+@require_login
+def create_dataset():
+ "Select the dataset to add the file contents against"
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+ assert form.get("studyid"), "study"
+ assert form.get("avgid"), "averaging method"
+ assert form.get("datasetname2"), "Dataset Name 2"
+ assert form.get("datasetfullname"), "Dataset Full Name"
+ assert form.get("datasetshortname"), "Dataset Short Name"
+ assert form.get("datasetpublic"), "Dataset public specification"
+ assert form.get("datasetconfidentiality"), "Dataset confidentiality"
+ assert form.get("datasetdatascale"), "Dataset Datascale"
+
+ with database_connection(app.config["SQL_URI"]) as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ datasetname = form["datasetname"]
+ cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Name=%s",
+ (datasetname,))
+ results = cursor.fetchall()
+ if bool(results):
+ flash("A dataset with that name already exists.",
+ "alert-error")
+ return redirect(url_for("dbinsert.select_dataset"), code=307)
+ values = (
+ form["studyid"], form["avgid"],
+ datasetname, form["datasetname2"],
+ form["datasetfullname"], form["datasetshortname"],
+ datetime.now().date().strftime("%Y-%m-%d"),
+ form["datasetpublic"], form["datasetconfidentiality"],
+ "williamslab", form["datasetdatascale"])
+ query = (
+ "INSERT INTO ProbeSetFreeze("
+ "ProbeFreezeId, AvgID, Name, Name2, FullName, "
+ "ShortName, CreateTime, OrderList, public, "
+ "confidentiality, AuthorisedUsers, DataScale) "
+ "VALUES"
+ "(%s, %s, %s, %s, %s, %s, %s, NULL, %s, %s, %s, %s)")
+ cursor.execute(query, values)
+ new_datasetid = cursor.lastrowid
+ return render_template(
+ "continue_from_create_dataset.html",
+ filename=form["filename"], filetype=form["filetype"],
+ species=form["species"], genechipid=form["genechipid"],
+ studyid=form["studyid"], datasetid=new_datasetid,
+ totallines=form["totallines"])
+ except AssertionError as aserr:
+ flash(f"Missing data {aserr.args[0]}", "alert-error")
+ return redirect(url_for("dbinsert.select_dataset"), code=307)
+
+def study_by_id(studyid:int) -> Union[dict, None]:
+ "Get a study by its Id"
+ with database_connection(app.config["SQL_URI"]) as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT * FROM ProbeFreeze WHERE Id=%s",
+ (studyid,))
+ return cursor.fetchone()
+
+def dataset_by_id(datasetid:int) -> Union[dict, None]:
+ "Retrieve a dataset by its id"
+ with database_connection(app.config["SQL_URI"]) as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ ("SELECT AvgMethod.Name AS AvgMethodName, ProbeSetFreeze.* "
+ "FROM ProbeSetFreeze INNER JOIN AvgMethod "
+ "ON ProbeSetFreeze.AvgId=AvgMethod.AvgMethodId "
+ "WHERE ProbeSetFreeze.Id=%s"),
+ (datasetid,))
+ return cursor.fetchone()
+
+def selected_keys(original: dict, keys: tuple) -> dict:
+ "Return a new dict from the `original` dict with only `keys` present."
+ return {key: value for key,value in original.items() if key in keys}
+
+@dbinsertbp.route("/final-confirmation", methods=["POST"])
+@require_login
+def final_confirmation():
+ "Preview the data before triggering entry into the database"
+ with database_connection(app.config["SQL_URI"]) as conn:
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+ assert form.get("studyid"), "study"
+ assert form.get("datasetid"), "dataset"
+
+ speciesid = form["species"]
+ genechipid = form["genechipid"]
+ studyid = form["studyid"]
+ datasetid=form["datasetid"]
+ return render_template(
+ "final_confirmation.html", filename=form["filename"],
+ filetype=form["filetype"], totallines=form["totallines"],
+ species=speciesid, genechipid=genechipid, studyid=studyid,
+ datasetid=datasetid, the_species=selected_keys(
+ with_db_connection(lambda conn: species_by_id(conn, speciesid)),
+ ("SpeciesName", "Name", "MenuName")),
+ platform=selected_keys(
+ platform_by_species_and_id(conn, speciesid, genechipid),
+ ("GeneChipName", "Name", "GeoPlatform", "Title", "GO_tree_value")),
+ study=selected_keys(
+ study_by_id(studyid), ("Name", "FullName", "ShortName")),
+ dataset=selected_keys(
+ dataset_by_id(datasetid),
+ ("AvgMethodName", "Name", "Name2", "FullName", "ShortName",
+ "DataScale")))
+ except AssertionError as aserr:
+ return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/insert-data", methods=["POST"])
+@require_login
+def insert_data():
+ "Trigger data insertion"
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+ assert form.get("studyid"), "study"
+ assert form.get("datasetid"), "dataset"
+
+ filename = form["filename"]
+ filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}"
+ redisurl = app.config["REDIS_URL"]
+ if os.path.exists(filepath):
+ with Redis.from_url(redisurl, decode_responses=True) as rconn:
+ job = jobs.launch_job(
+ jobs.data_insertion_job(
+ rconn, filepath, form["filetype"], form["totallines"],
+ form["species"], form["genechipid"], form["datasetid"],
+ app.config["SQL_URI"], redisurl,
+ app.config["JOBS_TTL_SECONDS"]),
+ redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+ return redirect(url_for("dbinsert.insert_status", job_id=job["jobid"]))
+ return render_error(f"File '{filename}' no longer exists.")
+ except AssertionError as aserr:
+ return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/status/<job_id>", methods=["GET"])
+def insert_status(job_id: str):
+ "Retrieve status of data insertion."
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+ if job:
+ job_status = job["status"]
+ if job_status == "success":
+ return render_template("insert_success.html", job=job)
+ if job["status"] == "error":
+ return render_template("insert_error.html", job=job)
+ return render_template("insert_progress.html", job=job)
+ return render_template("no_such_job.html", job_id=job_id), 400
diff --git a/uploader/expression_data/views.py b/uploader/expression_data/views.py
new file mode 100644
index 0000000..7629f3e
--- /dev/null
+++ b/uploader/expression_data/views.py
@@ -0,0 +1,385 @@
+"""Views for expression data"""
+import os
+import uuid
+import mimetypes
+from typing import Tuple
+from zipfile import ZipFile, is_zipfile
+
+import jsonpickle
+from redis import Redis
+from werkzeug.utils import secure_filename
+from gn_libs.mysqldb import database_connection
+from flask import (flash,
+ request,
+ url_for,
+ redirect,
+ Blueprint,
+ current_app as app)
+
+from quality_control.errors import InvalidValue, DuplicateHeading
+
+from uploader import jobs
+from uploader.datautils import order_by_family
+from uploader.ui import make_template_renderer
+from uploader.authorisation import require_login
+from uploader.db_utils import with_db_connection
+from uploader.species.models import all_species, species_by_id
+from uploader.population.models import (populations_by_species,
+ population_by_species_and_id)
+
+exprdatabp = Blueprint("expression-data", __name__)
+render_template = make_template_renderer("expression-data")
+
+def isinvalidvalue(item):
+ """Check whether item is of type InvalidValue"""
+ return isinstance(item, InvalidValue)
+
+
+def isduplicateheading(item):
+ """Check whether item is of type DuplicateHeading"""
+ return isinstance(item, DuplicateHeading)
+
+
+def errors(rqst) -> Tuple[str, ...]:
+ """Return a tuple of the errors found in the request `rqst`. If no error is
+ found, then an empty tuple is returned."""
+ def __filetype_error__():
+ return (
+ ("Invalid file type provided.",)
+ if rqst.form.get("filetype") not in ("average", "standard-error")
+ else tuple())
+
+ def __file_missing_error__():
+ return (
+ ("No file was uploaded.",)
+ if ("qc_text_file" not in rqst.files or
+ rqst.files["qc_text_file"].filename == "")
+ else tuple())
+
+ def __file_mimetype_error__():
+ text_file = rqst.files["qc_text_file"]
+ return (
+ (
+ ("Invalid file! Expected a tab-separated-values file, or a zip "
+ "file of the a tab-separated-values file."),)
+ if text_file.mimetype not in (
+ "text/plain", "text/tab-separated-values",
+ "application/zip")
+ else tuple())
+
+ return (
+ __filetype_error__() +
+ (__file_missing_error__() or __file_mimetype_error__()))
+
+
+def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]:
+ """Check the uploaded zip file for errors."""
+ zfile_errors: Tuple[str, ...] = tuple()
+ if is_zipfile(filepath):
+ with ZipFile(filepath, "r") as zfile:
+ infolist = zfile.infolist()
+ if len(infolist) != 1:
+ zfile_errors = zfile_errors + (
+ ("Expected exactly one (1) member file within the uploaded zip "
+ f"file. Got {len(infolist)} member files."),)
+ if len(infolist) == 1 and infolist[0].is_dir():
+ zfile_errors = zfile_errors + (
+ ("Expected a member text file in the uploaded zip file. Got a "
+ "directory/folder."),)
+
+ if len(infolist) == 1 and not infolist[0].is_dir():
+ zfile.extract(infolist[0], path=upload_dir)
+ mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}")
+ if mime[0] != "text/tab-separated-values":
+ zfile_errors = zfile_errors + (
+ ("Expected the member text file in the uploaded zip file to"
+ " be a tab-separated file."),)
+
+ return zfile_errors
+
+
+@exprdatabp.route("populations/expression-data", methods=["GET"])
+@require_login
+def index():
+ """Display the expression data index page."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ if not bool(request.args.get("species_id")):
+ return render_template("expression-data/index.html",
+ species=order_by_family(all_species(conn)),
+ activelink="expression-data")
+ species = species_by_id(conn, request.args.get("species_id"))
+ if not bool(species):
+ flash("Could not find species selected!", "alert-danger")
+ return redirect(url_for("species.populations.expression-data.index"))
+ return redirect(url_for(
+ "species.populations.expression-data.select_population",
+ species_id=species["SpeciesId"]))
+
+
+@exprdatabp.route("<int:species_id>/populations/expression-data/select-population",
+ methods=["GET"])
+@require_login
+def select_population(species_id: int):
+ """Select the expression data's population."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ if not bool(species):
+ flash("No such species!", "alert-danger")
+ return redirect(url_for("species.populations.expression-data.index"))
+
+ if not bool(request.args.get("population_id")):
+ return render_template("expression-data/select-population.html",
+ species=species,
+ populations=order_by_family(
+ populations_by_species(conn, species_id),
+ order_key="FamilyOrder"),
+ activelink="expression-data")
+
+ population = population_by_species_and_id(
+ conn, species_id, request.args.get("population_id"))
+ if not bool(population):
+ flash("No such population!", "alert-danger")
+ return redirect(url_for(
+ "species.populations.expression-data.select_population",
+ species_id=species_id))
+
+ return redirect(url_for("species.populations.expression-data.upload_file",
+ species_id=species_id,
+ population_id=population["Id"]))
+
+
+@exprdatabp.route("<int:species_id>/populations/<int:population_id>/"
+ "expression-data/upload",
+ methods=["GET", "POST"])
+@require_login
+def upload_file(species_id: int, population_id: int):
+ """Enables uploading the files"""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ population = population_by_species_and_id(conn, species_id, population_id)
+ if request.method == "GET":
+ return render_template("expression-data/select-file.html",
+ species=species,
+ population=population)
+
+ upload_dir = app.config["UPLOAD_FOLDER"]
+ request_errors = errors(request)
+ if request_errors:
+ for error in request_errors:
+ flash(error, "alert-danger error-expr-data")
+ return redirect(url_for("species.populations.expression-data.upload_file"))
+
+ filename = secure_filename(
+ request.files["qc_text_file"].filename)# type: ignore[arg-type]
+ if not os.path.exists(upload_dir):
+ os.mkdir(upload_dir)
+
+ filepath = os.path.join(upload_dir, filename)
+ request.files["qc_text_file"].save(os.path.join(upload_dir, filename))
+
+ zip_errors = zip_file_errors(filepath, upload_dir)
+ if zip_errors:
+ for error in zip_errors:
+ flash(error, "alert-danger error-expr-data")
+ return redirect(url_for("species.populations.expression-data.index.upload_file"))
+
+ return redirect(url_for("species.populations.expression-data.parse_file",
+ species_id=species_id,
+ population_id=population_id,
+ filename=filename,
+ filetype=request.form["filetype"]))
+
+
+@exprdatabp.route("/data-review", methods=["GET"])
+@require_login
+def data_review():
+ """Provide some help on data expectations to the user."""
+ return render_template("expression-data/data-review.html")
+
+
+@exprdatabp.route(
+ "<int:species_id>/populations/<int:population_id>/expression-data/parse",
+ methods=["GET"])
+@require_login
+def parse_file(species_id: int, population_id: int):
+ """Trigger file parsing"""
+ _errors = False
+ filename = request.args.get("filename")
+ filetype = request.args.get("filetype")
+
+ species = with_db_connection(lambda con: species_by_id(con, species_id))
+ if not bool(species):
+ flash("No such species.", "alert-danger")
+ _errors = True
+
+ if filename is None:
+ flash("No file provided", "alert-danger")
+ _errors = True
+
+ if filetype is None:
+ flash("No filetype provided", "alert-danger")
+ _errors = True
+
+ if filetype not in ("average", "standard-error"):
+ flash("Invalid filetype provided", "alert-danger")
+ _errors = True
+
+ if filename:
+ filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename)
+ if not os.path.exists(filepath):
+ flash("Selected file does not exist (any longer)", "alert-danger")
+ _errors = True
+
+ if _errors:
+ return redirect(url_for("species.populations.expression-data.upload_file"))
+
+ redisurl = app.config["REDIS_URL"]
+ with Redis.from_url(redisurl, decode_responses=True) as rconn:
+ job = jobs.launch_job(
+ jobs.build_file_verification_job(
+ rconn, app.config["SQL_URI"], redisurl,
+ species_id, filepath, filetype,# type: ignore[arg-type]
+ app.config["JOBS_TTL_SECONDS"]),
+ redisurl,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+ return redirect(url_for("species.populations.expression-data.parse_status",
+ species_id=species_id,
+ population_id=population_id,
+ job_id=job["jobid"]))
+
+
+@exprdatabp.route(
+ "<int:species_id>/populations/<int:population_id>/expression-data/parse/"
+ "status/<uuid:job_id>",
+ methods=["GET"])
+@require_login
+def parse_status(species_id: int, population_id: int, job_id: str):
+ "Retrieve the status of the job"
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ try:
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+ except jobs.JobNotFound as _exc:
+ return render_template("no_such_job.html", job_id=job_id), 400
+
+ error_filename = jobs.error_filename(
+ job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+ if os.path.exists(error_filename):
+ stat = os.stat(error_filename)
+ if stat.st_size > 0:
+ return redirect(url_for("parse.fail", job_id=job_id))
+
+ job_id = job["jobid"]
+ progress = float(job["percent"])
+ status = job["status"]
+ filename = job.get("filename", "uploaded file")
+ _errors = jsonpickle.decode(
+ job.get("errors", jsonpickle.encode(tuple())))
+ if status in ("success", "aborted"):
+ return redirect(url_for("species.populations.expression-data.results",
+ species_id=species_id,
+ population_id=population_id,
+ job_id=job_id))
+
+ if status == "parse-error":
+ return redirect(url_for("species.populations.expression-data.fail", job_id=job_id))
+
+ app.jinja_env.globals.update(
+ isinvalidvalue=isinvalidvalue,
+ isduplicateheading=isduplicateheading)
+ return render_template(
+ "expression-data/job-progress.html",
+ job_id = job_id,
+ job_status = status,
+ progress = progress,
+ message = job.get("message", ""),
+ job_name = f"Parsing '{filename}'",
+ errors=_errors,
+ species=with_db_connection(
+ lambda conn: species_by_id(conn, species_id)),
+ population=with_db_connection(
+ lambda conn: population_by_species_and_id(
+ conn, species_id, population_id)))
+
+
+@exprdatabp.route(
+ "<int:species_id>/populations/<int:population_id>/expression-data/parse/"
+ "<uuid:job_id>/results",
+ methods=["GET"])
+@require_login
+def results(species_id: int, population_id: int, job_id: uuid.UUID):
+ """Show results of parsing..."""
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+ if job:
+ filename = job["filename"]
+ _errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple())))
+ app.jinja_env.globals.update(
+ isinvalidvalue=isinvalidvalue,
+ isduplicateheading=isduplicateheading)
+ return render_template(
+ "expression-data/parse-results.html",
+ errors=_errors,
+ job_name = f"Parsing '{filename}'",
+ user_aborted = job.get("user_aborted"),
+ job_id=job["jobid"],
+ species=with_db_connection(
+ lambda conn: species_by_id(conn, species_id)),
+ population=with_db_connection(
+ lambda conn: population_by_species_and_id(
+ conn, species_id, population_id)))
+
+ return render_template("expression-data/no-such-job.html", job_id=job_id)
+
+
+@exprdatabp.route(
+ "<int:species_id>/populations/<int:population_id>/expression-data/parse/"
+ "<uuid:job_id>/fail",
+ methods=["GET"])
+@require_login
+def fail(species_id: int, population_id: int, job_id: str):
+ """Handle parsing failure"""
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+ if job:
+ error_filename = jobs.error_filename(
+ job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+ if os.path.exists(error_filename):
+ stat = os.stat(error_filename)
+ if stat.st_size > 0:
+ return render_template(
+ "worker_failure.html", job_id=job_id)
+
+ return render_template("parse_failure.html", job=job)
+
+ return render_template("expression-data/no-such-job.html",
+ **with_db_connection(lambda conn: {
+ "species_id": species_by_id(conn, species_id),
+ "population_id": population_by_species_and_id(
+ conn, species_id, population_id)}),
+ job_id=job_id)
+
+
+@exprdatabp.route(
+ "<int:species_id>/populations/<int:population_id>/expression-data/parse/"
+ "abort",
+ methods=["POST"])
+@require_login
+def abort(species_id: int, population_id: int):
+ """Handle user request to abort file processing"""
+ job_id = request.form["job_id"]
+
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+ if job:
+ rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id),
+ key="user_aborted",
+ value=int(True))
+
+ return redirect(url_for("species.populations.expression-data.parse_status",
+ species_id=species_id,
+ population_id=population_id,
+ job_id=job_id))
diff --git a/uploader/files/__init__.py b/uploader/files/__init__.py
new file mode 100644
index 0000000..53c3176
--- /dev/null
+++ b/uploader/files/__init__.py
@@ -0,0 +1,5 @@
+"""General files and chunks utilities."""
+from .chunks import chunked_binary_read
+from .functions import (fullpath,
+ save_file,
+ sha256_digest_over_file)
diff --git a/uploader/files/chunks.py b/uploader/files/chunks.py
new file mode 100644
index 0000000..c4360b5
--- /dev/null
+++ b/uploader/files/chunks.py
@@ -0,0 +1,32 @@
+"""Functions dealing with chunking of files."""
+from pathlib import Path
+from typing import Iterator
+
+from flask import current_app as app
+from werkzeug.utils import secure_filename
+
+
+def chunked_binary_read(filepath: Path, chunksize: int = 2048) -> Iterator:
+ """Read a file in binary mode in chunks."""
+ with open(filepath, "rb") as inputfile:
+ while True:
+ data = inputfile.read(chunksize)
+ if data != b"":
+ yield data
+ continue
+ break
+
+def chunk_name(uploadfilename: str, chunkno: int) -> str:
+ """Generate chunk name from original filename and chunk number"""
+ if uploadfilename == "":
+ raise ValueError("Name cannot be empty!")
+ if chunkno < 1:
+ raise ValueError("Chunk number must be greater than zero")
+ return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}"
+
+
+def chunks_directory(uniqueidentifier: str) -> Path:
+ """Compute the directory where chunks are temporarily stored."""
+ if uniqueidentifier == "":
+ raise ValueError("Unique identifier cannot be empty!")
+ return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}")
diff --git a/uploader/files/functions.py b/uploader/files/functions.py
new file mode 100644
index 0000000..7b9f06b
--- /dev/null
+++ b/uploader/files/functions.py
@@ -0,0 +1,42 @@
+"""Utilities to deal with uploaded files."""
+import hashlib
+from pathlib import Path
+from datetime import datetime
+
+from flask import current_app
+
+from werkzeug.utils import secure_filename
+from werkzeug.datastructures import FileStorage
+
+from .chunks import chunked_binary_read
+
+def save_file(fileobj: FileStorage, upload_dir: Path, hashed: bool = True) -> Path:
+ """Save the uploaded file and return the path."""
+ assert bool(fileobj), "Invalid file object!"
+ hashed_name = (
+ hashlib.sha512(
+ f"{fileobj.filename}::{datetime.now().isoformat()}".encode("utf8")
+ ).hexdigest()
+ if hashed else
+ fileobj.filename)
+ filename = Path(secure_filename(hashed_name)) # type: ignore[arg-type]
+ if not upload_dir.exists():
+ upload_dir.mkdir()
+
+ filepath = Path(upload_dir, filename)
+ fileobj.save(filepath)
+ return filepath
+
+
+def fullpath(filename: str):
+ """Get a file's full path. This makes use of `flask.current_app`."""
+ return Path(current_app.config["UPLOAD_FOLDER"], filename).absolute()
+
+
+def sha256_digest_over_file(filepath: Path) -> str:
+ """Compute the sha256 digest over a file's contents."""
+ filehash = hashlib.sha256()
+ for chunk in chunked_binary_read(filepath):
+ filehash.update(chunk)
+
+ return filehash.hexdigest()
diff --git a/uploader/files/views.py b/uploader/files/views.py
new file mode 100644
index 0000000..8d81654
--- /dev/null
+++ b/uploader/files/views.py
@@ -0,0 +1,116 @@
+"""Module for generic files endpoints."""
+import traceback
+from pathlib import Path
+
+from flask import request, jsonify, Blueprint, current_app as app
+
+from .chunks import chunk_name, chunks_directory
+
+files = Blueprint("files", __name__)
+
+def target_file(fileid: str) -> Path:
+ """Compute the full path for the target file."""
+ return Path(app.config["UPLOAD_FOLDER"], fileid)
+
+
+@files.route("/upload/resumable", methods=["GET"])
+def resumable_upload_get():
+ """Used for checking whether **ALL** chunks have been uploaded."""
+ fileid = request.args.get("resumableIdentifier", type=str) or ""
+ filename = request.args.get("resumableFilename", type=str) or ""
+ chunk = request.args.get("resumableChunkNumber", type=int) or 0
+ if not(fileid or filename or chunk):
+ return jsonify({
+ "message": "At least one required query parameter is missing.",
+ "error": "BadRequest",
+ "statuscode": 400
+ }), 400
+
+ # If the complete target file exists, return 200 for all chunks.
+ _targetfile = target_file(fileid)
+ if _targetfile.exists():
+ return jsonify({
+ "uploaded-file": _targetfile.name,
+ "original-name": filename,
+ "chunk": chunk,
+ "message": "The complete file already exists.",
+ "statuscode": 200
+ }), 200
+
+ if Path(chunks_directory(fileid),
+ chunk_name(filename, chunk)).exists():
+ return jsonify({
+ "chunk": chunk,
+ "message": f"Chunk {chunk} exists.",
+ "statuscode": 200
+ }), 200
+
+ return jsonify({
+ "message": f"Chunk {chunk} was not found.",
+ "error": "NotFound",
+ "statuscode": 404
+ }), 404
+
+
+def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path:
+ """Merge the chunks into a single file."""
+ with open(targetfile, "ab") as _target:
+ for chunkfile in chunkpaths:
+ with open(chunkfile, "rb") as _chunkdata:
+ _target.write(_chunkdata.read())
+
+ chunkfile.unlink()
+ return targetfile
+
+
+@files.route("/upload/resumable", methods=["POST"])
+def resumable_upload_post():
+ """Do the actual chunks upload here."""
+ _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0
+ _chunk = request.form.get("resumableChunkNumber", default=1, type=int)
+ _uploadfilename = request.form.get(
+ "resumableFilename", default="", type=str) or ""
+ _fileid = request.form.get(
+ "resumableIdentifier", default="", type=str) or ""
+ _targetfile = target_file(_fileid)
+
+ if _targetfile.exists():
+ return jsonify({
+ "uploaded-file": _targetfile.name,
+ "original-name": _uploadfilename,
+ "message": "File was uploaded successfully!",
+ "statuscode": 200
+ }), 200
+
+ try:
+ chunks_directory(_fileid).mkdir(exist_ok=True, parents=True)
+ request.files["file"].save(Path(chunks_directory(_fileid),
+ chunk_name(_uploadfilename, _chunk)))
+
+ # Check whether upload is complete
+ chunkpaths = tuple(
+ Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk))
+ for _achunk in range(1, _totalchunks+1))
+ if all(_file.exists() for _file in chunkpaths):
+ # merge_files and clean up chunks
+ __merge_chunks__(_targetfile, chunkpaths)
+ chunks_directory(_fileid).rmdir()
+ return jsonify({
+ "uploaded-file": _targetfile.name,
+ "original-name": _uploadfilename,
+ "message": "File was uploaded successfully!",
+ "statuscode": 200
+ }), 200
+ return jsonify({
+ "message": f"Chunk {int(_chunk)} uploaded successfully.",
+ "statuscode": 201
+ }), 201
+ except Exception as exc:# pylint: disable=[broad-except]
+ msg = "Error processing uploaded file chunks."
+ app.logger.error(msg, exc_info=True, stack_info=True)
+ return jsonify({
+ "message": msg,
+ "error": type(exc).__name__,
+ "error-description": " ".join(str(arg) for arg in exc.args),
+ "error-trace": traceback.format_exception(exc)
+ }), 500
diff --git a/uploader/genotypes/__init__.py b/uploader/genotypes/__init__.py
new file mode 100644
index 0000000..d0025d6
--- /dev/null
+++ b/uploader/genotypes/__init__.py
@@ -0,0 +1 @@
+"""The Genotypes module."""
diff --git a/uploader/genotypes/models.py b/uploader/genotypes/models.py
new file mode 100644
index 0000000..4c3e634
--- /dev/null
+++ b/uploader/genotypes/models.py
@@ -0,0 +1,102 @@
+"""Functions for handling genotypes."""
+from typing import Optional
+from datetime import datetime
+
+import MySQLdb as mdb
+from MySQLdb.cursors import Cursor, DictCursor
+from flask import current_app as app
+
+from gn_libs.mysqldb import debug_query
+
+def genocode_by_population(
+ conn: mdb.Connection, population_id: int) -> tuple[dict, ...]:
+ """Get the allele/genotype codes."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM GenoCode WHERE InbredSetId=%s",
+ (population_id,))
+ return tuple(dict(item) for item in cursor.fetchall())
+
+
+def genotype_markers_count(conn: mdb.Connection, species_id: int) -> int:
+ """Find the total count of the genotype markers for a species."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT COUNT(Name) AS markers_count FROM Geno WHERE SpeciesId=%s",
+ (species_id,))
+ return int(cursor.fetchone()["markers_count"])
+
+
+def genotype_markers(
+ conn: mdb.Connection,
+ species_id: int,
+ offset: int = 0,
+ limit: Optional[int] = None
+) -> tuple[dict, ...]:
+ """Retrieve markers from the database."""
+ _query = "SELECT * FROM Geno WHERE SpeciesId=%s"
+ if bool(limit) and limit > 0:# type: ignore[operator]
+ _query = _query + f" LIMIT {limit} OFFSET {offset}"
+
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(_query, (species_id,))
+ debug_query(cursor, app.logger)
+ return tuple(dict(row) for row in cursor.fetchall())
+
+
+def genotype_dataset(
+ conn: mdb.Connection,
+ species_id: int,
+ population_id: int,
+ dataset_id: Optional[int] = None
+) -> Optional[dict]:
+ """Retrieve genotype datasets from the database.
+
+ Apparently, you should only ever have one genotype dataset for a population.
+ """
+ _query = (
+ "SELECT gf.* FROM Species AS s INNER JOIN InbredSet AS iset "
+ "ON s.Id=iset.SpeciesId INNER JOIN GenoFreeze AS gf "
+ "ON iset.Id=gf.InbredSetId "
+ "WHERE s.Id=%s AND iset.Id=%s")
+ _params = (species_id, population_id)
+ if bool(dataset_id):
+ _query = _query + " AND gf.Id=%s"
+ _params = _params + (dataset_id,)# type: ignore[assignment]
+
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(_query, _params)
+ debug_query(cursor, app.logger)
+ result = cursor.fetchone()
+ if bool(result):
+ return dict(result)
+ return None
+
+
+def save_new_dataset(
+ cursor: Cursor,
+ population_id: int,
+ name: str,
+ fullname: str,
+ shortname: str
+) -> dict:
+ """Save a new genotype dataset into the database."""
+ params = {
+ "InbredSetId": population_id,
+ "Name": name,
+ "FullName": fullname,
+ "ShortName": shortname,
+ "CreateTime": datetime.now().date().isoformat(),
+ "public": 2,
+ "confidentiality": 0,
+ "AuthorisedUsers": None
+ }
+ cursor.execute(
+ "INSERT INTO GenoFreeze("
+ "Name, FullName, ShortName, CreateTime, public, InbredSetId, "
+ "confidentiality, AuthorisedUsers"
+ ") VALUES ("
+ "%(Name)s, %(FullName)s, %(ShortName)s, %(CreateTime)s, %(public)s, "
+ "%(InbredSetId)s, %(confidentiality)s, %(AuthorisedUsers)s"
+ ")",
+ params)
+ return {**params, "Id": cursor.lastrowid}
diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py
new file mode 100644
index 0000000..5105730
--- /dev/null
+++ b/uploader/genotypes/views.py
@@ -0,0 +1,225 @@
+"""Views for the genotypes."""
+from MySQLdb.cursors import DictCursor
+from gn_libs.mysqldb import database_connection
+from flask import (flash,
+ request,
+ url_for,
+ redirect,
+ Blueprint,
+ render_template,
+ current_app as app)
+
+from uploader.ui import make_template_renderer
+from uploader.oauth2.client import oauth2_post
+from uploader.authorisation import require_login
+from uploader.species.models import all_species, species_by_id
+from uploader.monadic_requests import make_either_error_handler
+from uploader.request_checks import with_species, with_population
+from uploader.datautils import safe_int, order_by_family, enumerate_sequence
+from uploader.population.models import (populations_by_species,
+ population_by_species_and_id)
+
+from .models import (genotype_markers,
+ genotype_dataset,
+ save_new_dataset,
+ genotype_markers_count,
+ genocode_by_population)
+
+genotypesbp = Blueprint("genotypes", __name__)
+render_template = make_template_renderer("genotypes")
+
+@genotypesbp.route("populations/genotypes", methods=["GET"])
+@require_login
+def index():
+ """Direct entry-point for genotypes."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ if not bool(request.args.get("species_id")):
+ return render_template("genotypes/index.html",
+ species=all_species(conn),
+ activelink="genotypes")
+
+ species_id = request.args.get("species_id")
+ if species_id == "CREATE-SPECIES":
+ return redirect(url_for(
+ "species.create_species",
+ return_to="species.populations.genotypes.select_population"))
+
+ species = species_by_id(conn, request.args.get("species_id"))
+ if not bool(species):
+ flash(f"Could not find species with ID '{request.args.get('species_id')}'!",
+ "alert-danger")
+ return redirect(url_for("species.populations.genotypes.index"))
+ return redirect(url_for("species.populations.genotypes.select_population",
+ species_id=species["SpeciesId"]))
+
+
+@genotypesbp.route("/<int:species_id>/populations/genotypes/select-population",
+ methods=["GET"])
+@require_login
+@with_species(redirect_uri="species.populations.genotypes.index")
+def select_population(species: dict, species_id: int):
+ """Select the population under which the genotypes go."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ if not bool(request.args.get("population_id")):
+ return render_template("genotypes/select-population.html",
+ species=species,
+ populations=populations_by_species(
+ conn, species_id),
+ activelink="genotypes")
+
+ population_id = request.args["population_id"]
+ if population_id == "CREATE-POPULATION":
+ return redirect(url_for(
+ "species.populations.create_population",
+ species_id=species["SpeciesId"],
+ return_to="species.populations.samples.list_genotypes"))
+
+ population = population_by_species_and_id(
+ conn, species_id, request.args.get("population_id"))
+ if not bool(population):
+ flash("Invalid population selected!", "alert-danger")
+ return redirect(url_for(
+ "species.populations.genotypes.select_population",
+ species_id=species_id))
+
+ return redirect(url_for("species.populations.genotypes.list_genotypes",
+ species_id=species_id,
+ population_id=population["Id"]))
+
+
+@genotypesbp.route(
+ "/<int:species_id>/populations/<int:population_id>/genotypes",
+ methods=["GET"])
+@require_login
+@with_population(species_redirect_uri="species.populations.genotypes.index",
+ redirect_uri="species.populations.genotypes.select_population")
+def list_genotypes(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument]
+ """List genotype details for species and population."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ return render_template("genotypes/list-genotypes.html",
+ species=species,
+ population=population,
+ genocode=genocode_by_population(
+ conn, population["Id"]),
+ total_markers=genotype_markers_count(
+ conn, species["SpeciesId"]),
+ dataset=genotype_dataset(conn,
+ species["SpeciesId"],
+ population["Id"]),
+ activelink="list-genotypes")
+
+
+@genotypesbp.route(
+ "/<int:species_id>/populations/<int:population_id>/genotypes/list-markers",
+ methods=["GET"])
+@require_login
+@with_population(species_redirect_uri="species.populations.genotypes.index",
+ redirect_uri="species.populations.genotypes.select_population")
+def list_markers(
+ species: dict,
+ population: dict,
+ **kwargs
+):# pylint: disable=[unused-argument]
+ """List a species' genetic markers."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ start_from = max(safe_int(request.args.get("start_from") or 0), 0)
+ count = safe_int(request.args.get("count") or 20)
+ return render_template("genotypes/list-markers.html",
+ species=species,
+ population=population,
+ total_markers=genotype_markers_count(
+ conn, species["SpeciesId"]),
+ start_from=start_from,
+ count=count,
+ markers=enumerate_sequence(
+ genotype_markers(conn,
+ species["SpeciesId"],
+ offset=start_from,
+ limit=count),
+ start=start_from+1),
+ activelink="list-markers")
+
+@genotypesbp.route(
+ "/<int:species_id>/populations/<int:population_id>/genotypes/datasets/"
+ "<int:dataset_id>/view",
+ methods=["GET"])
+@require_login
+def view_dataset(species_id: int, population_id: int, dataset_id: int):
+ """View details regarding a specific dataset."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ if not bool(species):
+ flash("Invalid species provided!", "alert-danger")
+ return redirect(url_for("species.populations.genotypes.index"))
+
+ population = population_by_species_and_id(
+ conn, species_id, population_id)
+ if not bool(population):
+ flash("Invalid population selected!", "alert-danger")
+ return redirect(url_for(
+ "species.populations.genotypes.select_population",
+ species_id=species_id))
+
+ dataset = genotype_dataset(conn, species_id, population_id, dataset_id)
+ if not bool(dataset):
+ flash("Could not find such a dataset!", "alert-danger")
+ return redirect(url_for(
+ "species.populations.genotypes.list_genotypes",
+ species_id=species_id,
+ population_id=population_id))
+
+ return render_template("genotypes/view-dataset.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ activelink="view-dataset")
+
+
+@genotypesbp.route(
+ "/<int:species_id>/populations/<int:population_id>/genotypes/datasets/"
+ "create",
+ methods=["GET", "POST"])
+@require_login
+@with_population(species_redirect_uri="species.populations.genotypes.index",
+ redirect_uri="species.populations.genotypes.select_population")
+def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument]
+ """Create a genotype dataset."""
+ with (database_connection(app.config["SQL_URI"]) as conn,
+ conn.cursor(cursorclass=DictCursor) as cursor):
+ if request.method == "GET":
+ return render_template("genotypes/create-dataset.html",
+ species=species,
+ population=population,
+ activelink="create-dataset")
+
+ form = request.form
+ new_dataset = save_new_dataset(
+ cursor,
+ population["Id"],
+ form["geno-dataset-name"],
+ form["geno-dataset-fullname"],
+ form["geno-dataset-shortname"])
+
+ def __success__(_success):
+ flash("Successfully created genotype dataset.", "alert-success")
+ return redirect(url_for(
+ "species.populations.genotypes.list_genotypes",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"]))
+
+ return oauth2_post(
+ "auth/resource/genotypes/create",
+ json={
+ **dict(request.form),
+ "species_id": species["SpeciesId"],
+ "population_id": population["Id"],
+ "dataset_id": new_dataset["Id"],
+ "dataset_name": form["geno-dataset-name"],
+ "dataset_fullname": form["geno-dataset-fullname"],
+ "dataset_shortname": form["geno-dataset-shortname"],
+ "public": "on"
+ }
+ ).either(
+ make_either_error_handler(
+ "There was an error creating the genotype dataset."),
+ __success__)
diff --git a/uploader/input_validation.py b/uploader/input_validation.py
new file mode 100644
index 0000000..627c69e
--- /dev/null
+++ b/uploader/input_validation.py
@@ -0,0 +1,71 @@
+"""Input validation utilities"""
+import re
+import json
+import base64
+from typing import Any
+
+def is_empty_string(value: str) -> bool:
+ """Check whether as string is empty"""
+ return (isinstance(value, str) and value.strip() == "")
+
+
+def is_empty_input(value: Any) -> bool:
+ """Check whether user provided an empty value."""
+ return (value is None or is_empty_string(value))
+
+
+def is_integer_input(value: Any) -> bool:
+ """
+ Check whether user provided a value that can be parsed into an integer.
+ """
+ def __is_int__(val, base):
+ try:
+ int(val, base=base)
+ except ValueError:
+ return False
+ return True
+ return isinstance(value, int) or (
+ (not is_empty_input(value)) and (
+ isinstance(value, str) and (
+ __is_int__(value, 10)
+ or __is_int__(value, 8)
+ or __is_int__(value, 16))))
+
+
+def is_valid_representative_name(repr_name: str) -> bool:
+ """
+ Check whether the given representative name is a valid according to our rules.
+
+ Parameters
+ ----------
+ repr_name: a string of characters.
+
+ Checks For
+ ----------
+ * The name MUST start with an alphabet [a-zA-Z]
+ * The name MUST end with an alphabet [a-zA-Z] or number [0-9]
+ * The name MUST be composed of alphabets [a-zA-Z], numbers [0-9],
+ underscores (_) and/or hyphens (-).
+
+ Returns
+ -------
+ Boolean indicating whether or not the name is valid.
+ """
+ pattern = re.compile(r"^[a-zA-Z]+[a-zA-Z0-9_-]*[a-zA-Z0-9]$")
+ return bool(pattern.match(repr_name))
+
+
+def encode_errors(errors: tuple[tuple[str, str], ...], form) -> bytes:
+ """Encode form errors into base64 string."""
+ return base64.b64encode(
+ json.dumps({
+ "errors": dict(errors),
+ "original_formdata": dict(form)
+ }).encode("utf8"))
+
+
+def decode_errors(errorstr) -> dict[str, dict]:
+ """Decode errors from base64 string"""
+ if not bool(errorstr):
+ return {"errors": {}, "original_formdata": {}}
+ return json.loads(base64.b64decode(errorstr.encode("utf8")).decode("utf8"))
diff --git a/uploader/jobs.py b/uploader/jobs.py
new file mode 100644
index 0000000..e86ee05
--- /dev/null
+++ b/uploader/jobs.py
@@ -0,0 +1,164 @@
+"""Handle jobs"""
+import os
+import sys
+import uuid
+import json
+import shlex
+import subprocess
+from uuid import UUID, uuid4
+from datetime import timedelta
+from typing import Union, Optional
+
+from redis import Redis
+from flask import current_app as app
+
+from functional_tools import take
+
+JOBS_PREFIX = "jobs"
+
+class JobNotFound(Exception):
+ """Raised if we try to retrieve a non-existent job."""
+
+def jobsnamespace():
+ """
+ Return the jobs namespace prefix. It depends on app configuration.
+
+ Calling this function outside of an application context will cause an
+ exception to be raised. It is mostly a convenience utility to use within the
+ application.
+ """
+ return f"{app.config['GNQC_REDIS_PREFIX']}:{JOBS_PREFIX}"
+
+def job_key(namespaceprefix: str, jobid: Union[str, UUID]) -> str:
+ """Build the key by appending it to the namespace prefix."""
+ return f"{namespaceprefix}:{jobid}"
+
+def raise_jobnotfound(rprefix:str, jobid: Union[str,UUID]):
+ """Utility to raise a `NoSuchJobError`"""
+ raise JobNotFound(f"Could not retrieve job '{jobid}' from '{rprefix}.")
+
+def error_filename(jobid, error_dir):
+ "Compute the path of the file where errors will be dumped."
+ return f"{error_dir}/job_{jobid}.error"
+
+def initialise_job(# pylint: disable=[too-many-arguments]
+ rconn: Redis, rprefix: str, jobid: str, command: list, job_type: str,
+ ttl_seconds: int = 86400, extra_meta: Optional[dict] = None) -> dict:
+ "Initialise a job 'object' and put in on redis"
+ the_job = {
+ "jobid": jobid, "command": shlex.join(command), "status": "pending",
+ "percent": 0, "job-type": job_type, **(extra_meta or {})
+ }
+ rconn.hset(job_key(rprefix, jobid), mapping=the_job)
+ rconn.expire(
+ name=job_key(rprefix, jobid), time=timedelta(seconds=ttl_seconds))
+ return the_job
+
+def build_file_verification_job(#pylint: disable=[too-many-arguments]
+ redis_conn: Redis,
+ dburi: str,
+ redisuri: str,
+ speciesid: int,
+ filepath: str,
+ filetype: str,
+ ttl_seconds: int):
+ "Build a file verification job"
+ jobid = str(uuid4())
+ command = [
+ sys.executable, "-m", "scripts.validate_file",
+ dburi, redisuri, jobsnamespace(), jobid,
+ "--redisexpiry", str(ttl_seconds),
+ str(speciesid), filetype, filepath,
+ ]
+ return initialise_job(
+ redis_conn, jobsnamespace(), jobid, command, "file-verification",
+ ttl_seconds, {
+ "filetype": filetype,
+ "filename": os.path.basename(filepath), "percent": 0
+ })
+
+def data_insertion_job(# pylint: disable=[too-many-arguments]
+ redis_conn: Redis, filepath: str, filetype: str, totallines: int,
+ speciesid: int, platformid: int, datasetid: int, databaseuri: str,
+ redisuri: str, ttl_seconds: int) -> dict:
+ "Build a data insertion job"
+ jobid = str(uuid4())
+ command = [
+ sys.executable, "-m", "scripts.insert_data", filetype, filepath,
+ speciesid, platformid, datasetid, databaseuri, redisuri
+ ]
+ return initialise_job(
+ redis_conn, jobsnamespace(), jobid, command, "data-insertion",
+ ttl_seconds, {
+ "filename": os.path.basename(filepath), "filetype": filetype,
+ "totallines": totallines
+ })
+
+def launch_job(the_job: dict, redisurl: str, error_dir):
+ """Launch a job in the background"""
+ if not os.path.exists(error_dir):
+ os.mkdir(error_dir)
+
+ jobid = the_job["jobid"]
+ with open(error_filename(jobid, error_dir),
+ "w",
+ encoding="utf-8") as errorfile:
+ subprocess.Popen( # pylint: disable=[consider-using-with]
+ [sys.executable, "-m", "scripts.worker", redisurl, jobsnamespace(),
+ jobid],
+ stderr=errorfile,
+ env={"PYTHONPATH": ":".join(sys.path)})
+
+ return the_job
+
+def job(rconn: Redis, rprefix: str, jobid: Union[str,UUID]):
+ "Retrieve the job"
+ thejob = (rconn.hgetall(job_key(rprefix, jobid)) or
+ raise_jobnotfound(rprefix, jobid))
+ return thejob
+
+def update_status(
+ rconn: Redis, rprefix: str, jobid: Union[str, UUID], status: str):
+ """Update status of job in redis."""
+ rconn.hset(name=job_key(rprefix, jobid), key="status", value=status)
+
+def update_stdout_stderr(rconn: Redis,
+ rprefix: str,
+ jobid: Union[str, UUID],
+ bytes_read: bytes,
+ stream: str):
+ "Update the stdout/stderr keys according to the value of `stream`."
+ thejob = job(rconn, rprefix, jobid)
+ contents = thejob.get(stream, '')
+ new_contents = contents + bytes_read.decode("utf-8")
+ rconn.hset(name=job_key(rprefix, jobid), key=stream, value=new_contents)
+
+
+def job_errors(
+ rconn: Redis,
+ prefix: str,
+ job_id: Union[str, uuid.UUID],
+ count: int = 100
+) -> list:
+ """Fetch job errors"""
+ return take(
+ (
+ json.loads(error)
+ for key in rconn.keys(f"{prefix}:{str(job_id)}:*:errors:*")
+ for error in rconn.lrange(key, 0, -1)),
+ count)
+
+
+def job_files_metadata(
+ rconn: Redis,
+ prefix: str,
+ job_id: Union[str, uuid.UUID]
+) -> dict:
+ """Get the metadata for specific job file."""
+ return {
+ key.split(":")[-1]: {
+ **rconn.hgetall(key),
+ "filetype": key.split(":")[-3]
+ }
+ for key in rconn.keys(f"{prefix}:{str(job_id)}:*:metadata*")
+ }
diff --git a/uploader/monadic_requests.py b/uploader/monadic_requests.py
new file mode 100644
index 0000000..c492df5
--- /dev/null
+++ b/uploader/monadic_requests.py
@@ -0,0 +1,104 @@
+"""Wrap requests functions with monads."""
+import traceback
+from typing import Union, Optional, Callable
+
+import requests
+from requests.models import Response
+from pymonad.either import Left, Right, Either
+from flask import (flash,
+ request,
+ redirect,
+ render_template,
+ current_app as app,
+ escape as flask_escape)
+
+# HTML Status codes indicating a successful request.
+SUCCESS_CODES = (200, 201, 202, 203, 204, 205, 206, 207, 208, 226)
+
+# Possible error(s) that can be encontered while attempting to do a request.
+PossibleError = Union[Response, Exception]
+
+
+def make_error_handler(
+ redirect_to: Optional[Response] = None,
+ cleanup_thunk: Callable = lambda *args: None
+) -> Callable[[PossibleError], Response]:
+ """
+ Build a function to gracefully handle errors encountered while doing
+ requests.
+
+ :rtype: Callable
+ """
+ redirect_to = redirect_to or redirect(request.url)
+ def __handler__(resp_or_exc: PossibleError) -> Response:
+ cleanup_thunk()
+ if issubclass(type(resp_or_exc), Exception):
+ # Is an exception!
+ return render_template(
+ "unhandled_exception.html",
+ trace=traceback.format_exception(resp_or_exc))
+ if isinstance(resp_or_exc, Response):
+ flash("The authorisation server responded with "
+ f"({flask_escape(resp_or_exc.status_code)}, "
+ f"{flask_escape(resp_or_exc.reason)}) for the request to "
+ f"'{flask_escape(resp_or_exc.request.url)}'",
+ "alert-danger")
+ return redirect_to
+
+ flash("Unspecified error!", "alert-danger")
+ app.logger.debug("Error (%s): %s", type(resp_or_exc), resp_or_exc)
+ return redirect_to
+ return __handler__
+
+
+def get(url, params=None, **kwargs) -> Either:
+ """
+ A wrapper around `requests.get` function.
+
+ Takes the same arguments as `requests.get`.
+
+ :rtype: pymonad.either.Either
+ """
+ try:
+ resp = requests.get(url, params=params, **kwargs)
+ if resp.status_code in SUCCESS_CODES:
+ return Right(resp.json())
+ return Left(resp)
+ except requests.exceptions.RequestException as exc:
+ return Left(exc)
+
+
+def post(url, data=None, json=None, **kwargs) -> Either:
+ """
+ A wrapper around `requests.post` function.
+
+ Takes the same arguments as `requests.post`.
+
+ :rtype: pymonad.either.Either
+ """
+ try:
+ resp = requests.post(url, data=data, json=json, **kwargs)
+ if resp.status_code in SUCCESS_CODES:
+ return Right(resp.json())
+ return Left(resp)
+ except requests.exceptions.RequestException as exc:
+ return Left(exc)
+
+
+def make_either_error_handler(msg):
+ """Make generic error handler for pymonads Either objects."""
+ def __fail__(error):
+ if issubclass(type(error), Exception):
+ app.logger.debug("\n\n%s (Exception)\n\n", msg, exc_info=True)
+ raise error
+ if issubclass(type(error), Response):
+ try:
+ _data = error.json()
+ except Exception as _exc:
+ raise Exception(error.content) from _exc
+ raise Exception(_data)
+
+ app.logger.debug("\n\n%s\n\n", msg)
+ raise Exception(error)
+
+ return __fail__
diff --git a/uploader/oauth2/__init__.py b/uploader/oauth2/__init__.py
new file mode 100644
index 0000000..aaea638
--- /dev/null
+++ b/uploader/oauth2/__init__.py
@@ -0,0 +1 @@
+"""Package to handle OAuth2 authentication/authorisation issues."""
diff --git a/uploader/oauth2/client.py b/uploader/oauth2/client.py
new file mode 100644
index 0000000..1efa299
--- /dev/null
+++ b/uploader/oauth2/client.py
@@ -0,0 +1,231 @@
+"""OAuth2 client utilities."""
+import json
+import time
+import random
+from datetime import datetime, timedelta
+from urllib.parse import urljoin, urlparse
+
+import requests
+from flask import request, current_app as app
+
+from pymonad.either import Left, Right, Either
+
+from authlib.common.urls import url_decode
+from authlib.jose.errors import BadSignatureError
+from authlib.jose import KeySet, JsonWebKey, JsonWebToken
+from authlib.integrations.requests_client import OAuth2Session
+
+from uploader import session
+import uploader.monadic_requests as mrequests
+
+SCOPE = ("profile group role resource register-client user masquerade "
+ "introspect migrate-data")
+
+
+def authserver_uri():
+ """Return URI to authorisation server."""
+ return app.config["AUTH_SERVER_URL"]
+
+
+def oauth2_clientid():
+ """Return the client id."""
+ return app.config["OAUTH2_CLIENT_ID"]
+
+
+def oauth2_clientsecret():
+ """Return the client secret."""
+ return app.config["OAUTH2_CLIENT_SECRET"]
+
+
+def __fetch_auth_server_jwks__() -> KeySet:
+ """Fetch the JWKs from the auth server."""
+ return KeySet([
+ JsonWebKey.import_key(key)
+ for key in requests.get(
+ urljoin(authserver_uri(), "auth/public-jwks")
+ ).json()["jwks"]])
+
+
+def __update_auth_server_jwks__(jwks) -> KeySet:
+ """Update the JWKs from the servers if necessary."""
+ last_updated = jwks["last-updated"]
+ now = datetime.now().timestamp()
+ # Maybe the `two_hours` variable below can be made into a configuration
+ # variable and passed in to this function
+ two_hours = timedelta(hours=2).seconds
+ if bool(last_updated) and (now - last_updated) < two_hours:
+ return jwks["jwks"]
+
+ return session.set_auth_server_jwks(__fetch_auth_server_jwks__())
+
+
+def auth_server_jwks() -> KeySet:
+ """Fetch the auth-server JSON Web Keys information."""
+ _jwks = session.session_info().get("auth_server_jwks") or {}
+ if bool(_jwks):
+ return __update_auth_server_jwks__({
+ "last-updated": _jwks["last-updated"],
+ "jwks": KeySet([
+ JsonWebKey.import_key(key) for key in _jwks.get(
+ "jwks", {"keys": []})["keys"]])
+ })
+
+ return __update_auth_server_jwks__({
+ "last-updated": (datetime.now() - timedelta(hours=3)).timestamp()
+ })
+
+
+def oauth2_client():
+ """Build the OAuth2 client for use fetching data."""
+ def __update_token__(token, refresh_token=None, access_token=None):# pylint: disable=[unused-argument]
+ """Update the token when refreshed."""
+ session.set_user_token(token)
+
+ def __json_auth__(client, _method, uri, headers, body):
+ return (
+ uri,
+ {**headers, "Content-Type": "application/json"},
+ json.dumps({
+ **dict(url_decode(body)),
+ "client_id": client.client_id,
+ "client_secret": client.client_secret
+ }))
+
+ def __client__(token) -> OAuth2Session:
+ client = OAuth2Session(
+ oauth2_clientid(),
+ oauth2_clientsecret(),
+ scope=SCOPE,
+ token_endpoint=urljoin(authserver_uri(), "/auth/token"),
+ token_endpoint_auth_method="client_secret_post",
+ token=token,
+ update_token=__update_token__)
+ client.register_client_auth_method(
+ ("client_secret_post", __json_auth__))
+ return client
+
+ def __token_expired__(token):
+ """Check whether the token has expired."""
+ jwks = auth_server_jwks()
+ if bool(jwks):
+ for jwk in jwks.keys:
+ try:
+ jwt = JsonWebToken(["RS256"]).decode(
+ token["access_token"], key=jwk)
+ if bool(jwt.get("exp")):
+ return datetime.now().timestamp() > jwt["exp"]
+ except BadSignatureError as _bse:
+ pass
+
+ return False
+
+ def __delay__():
+ """Do a tiny delay."""
+ time.sleep(random.choice(tuple(i/1000.0 for i in range(0,100))))
+
+ def __refresh_token__(token):
+ """Refresh the token if necessary — synchronise amongst threads."""
+ if __token_expired__(token):
+ __delay__()
+ if session.is_token_refreshing():
+ while session.is_token_refreshing():
+ __delay__()
+
+ return session.user_token().either(None, lambda _tok: _tok)
+
+ session.toggle_token_refreshing()
+ _client = __client__(token)
+ _client.get(urljoin(authserver_uri(), "auth/user/"))
+ session.toggle_token_refreshing()
+ return _client.token
+
+ return token
+
+ return session.user_token().then(__refresh_token__).either(
+ lambda _notok: __client__(None),
+ __client__)
+
+
+def user_logged_in():
+ """Check whether the user has logged in."""
+ suser = session.session_info()["user"]
+ return suser["logged_in"] and suser["token"].is_right()
+
+
+def authserver_authorise_uri():
+ """Build up the authorisation URI."""
+ req_baseurl = urlparse(request.base_url, scheme=request.scheme)
+ host_uri = f"{req_baseurl.scheme}://{req_baseurl.netloc}/"
+ return urljoin(
+ authserver_uri(),
+ "auth/authorise?response_type=code"
+ f"&client_id={oauth2_clientid()}"
+ f"&redirect_uri={urljoin(host_uri, 'oauth2/code')}")
+
+
+def __no_token__(_err) -> Left:
+ """Handle situation where request is attempted with no token."""
+ resp = requests.models.Response()
+ resp._content = json.dumps({#pylint: disable=[protected-access]
+ "error": "AuthenticationError",
+ "error-description": ("You need to authenticate to access requested "
+ "information.")}).encode("utf-8")
+ resp.status_code = 400
+ return Left(resp)
+
+
+def oauth2_get(url, **kwargs) -> Either:
+ """Do a get request to the authentication/authorisation server."""
+ def __get__(_token) -> Either:
+ _uri = urljoin(authserver_uri(), url)
+ try:
+ resp = oauth2_client().get(
+ _uri,
+ **{
+ **kwargs,
+ "headers": {
+ **kwargs.get("headers", {}),
+ "Content-Type": "application/json"
+ }
+ })
+ if resp.status_code in mrequests.SUCCESS_CODES:
+ return Right(resp.json())
+ return Left(resp)
+ except Exception as exc:#pylint: disable=[broad-except]
+ app.logger.error("Error retrieving data from auth server: (GET %s)",
+ _uri,
+ exc_info=True)
+ return Left(exc)
+ return session.user_token().either(__no_token__, __get__)
+
+
+def oauth2_post(url, data=None, json=None, **kwargs):#pylint: disable=[redefined-outer-name]
+ """Do a POST request to the authentication/authorisation server."""
+ def __post__(_token) -> Either:
+ _uri = urljoin(authserver_uri(), url)
+ _headers = ({
+ **kwargs.get("headers", {}),
+ "Content-Type": "application/json"
+ }
+ if bool(json) else kwargs.get("headers", {}))
+ try:
+ request_data = {
+ **(data or {}),
+ **(json or {}),
+ "client_id": oauth2_clientid(),
+ "client_secret": oauth2_clientsecret()
+ }
+ resp = oauth2_client().post(
+ _uri,
+ data=(request_data if bool(data) else None),
+ json=(request_data if bool(json) else None),
+ **{**kwargs, "headers": _headers})
+ if resp.status_code in mrequests.SUCCESS_CODES:
+ return Right(resp.json())
+ return Left(resp)
+ except Exception as exc:#pylint: disable=[broad-except]
+ app.logger.error("Error retrieving data from auth server: (POST %s)",
+ _uri,
+ exc_info=True)
+ return Left(exc)
+ return session.user_token().either(__no_token__, __post__)
diff --git a/uploader/oauth2/jwks.py b/uploader/oauth2/jwks.py
new file mode 100644
index 0000000..efd0499
--- /dev/null
+++ b/uploader/oauth2/jwks.py
@@ -0,0 +1,86 @@
+"""Utilities dealing with JSON Web Keys (JWK)"""
+import os
+from pathlib import Path
+from typing import Any, Union
+from datetime import datetime, timedelta
+
+from flask import Flask
+from authlib.jose import JsonWebKey
+from pymonad.either import Left, Right, Either
+
+def jwks_directory(app: Flask, configname: str) -> Path:
+ """Compute the directory where the JWKs are stored."""
+ appsecretsdir = Path(app.config[configname]).parent
+ if appsecretsdir.exists() and appsecretsdir.is_dir():
+ jwksdir = Path(appsecretsdir, "jwks/")
+ if not jwksdir.exists():
+ jwksdir.mkdir()
+ return jwksdir
+ raise ValueError(
+ "The `appsecretsdir` value should be a directory that actually exists.")
+
+
+def generate_and_save_private_key(
+ storagedir: Path,
+ kty: str = "RSA",
+ crv_or_size: Union[str, int] = 2048,
+ options: tuple[tuple[str, Any]] = (("iat", datetime.now().timestamp()),)
+) -> JsonWebKey:
+ """Generate a private key and save to `storagedir`."""
+ privatejwk = JsonWebKey.generate_key(
+ kty, crv_or_size, dict(options), is_private=True)
+ keyname = f"{privatejwk.thumbprint()}.private.pem"
+ with open(Path(storagedir, keyname), "wb") as pemfile:
+ pemfile.write(privatejwk.as_pem(is_private=True))
+
+ return privatejwk
+
+
+def pem_to_jwk(filepath: Path) -> JsonWebKey:
+ """Parse a PEM file into a JWK object."""
+ with open(filepath, "rb") as pemfile:
+ return JsonWebKey.import_key(pemfile.read())
+
+
+def __sorted_jwks_paths__(storagedir: Path) -> tuple[tuple[float, Path], ...]:
+ """A sorted list of the JWK file paths with their creation timestamps."""
+ return tuple(sorted(((os.stat(keypath).st_ctime, keypath)
+ for keypath in (Path(storagedir, keyfile)
+ for keyfile in os.listdir(storagedir)
+ if keyfile.endswith(".pem"))),
+ key=lambda tpl: tpl[0]))
+
+
+def list_jwks(storagedir: Path) -> tuple[JsonWebKey, ...]:
+ """
+ List all the JWKs in a particular directory in the order they were created.
+ """
+ return tuple(pem_to_jwk(keypath) for ctime,keypath in
+ __sorted_jwks_paths__(storagedir))
+
+
+def newest_jwk(storagedir: Path) -> Either:
+ """
+ Return an Either monad with the newest JWK or a message if none exists.
+ """
+ existingkeys = __sorted_jwks_paths__(storagedir)
+ if len(existingkeys) > 0:
+ return Right(pem_to_jwk(existingkeys[-1][1]))
+ return Left("No JWKs exist")
+
+
+def newest_jwk_with_rotation(jwksdir: Path, keyage: int) -> JsonWebKey:
+ """
+ Retrieve the latests JWK, creating a new one if older than `keyage` days.
+ """
+ def newer_than_days(jwkey):
+ filestat = os.stat(Path(
+ jwksdir, f"{jwkey.as_dict()['kid']}.private.pem"))
+ oldesttimeallowed = (datetime.now() - timedelta(days=keyage))
+ if filestat.st_ctime < (oldesttimeallowed.timestamp()):
+ return Left("JWK is too old!")
+ return jwkey
+
+ return newest_jwk(jwksdir).then(newer_than_days).either(
+ lambda _errmsg: generate_and_save_private_key(jwksdir),
+ lambda key: key)
diff --git a/uploader/oauth2/views.py b/uploader/oauth2/views.py
new file mode 100644
index 0000000..a7211cb
--- /dev/null
+++ b/uploader/oauth2/views.py
@@ -0,0 +1,138 @@
+"""Views for OAuth2 related functionality."""
+import uuid
+from datetime import datetime, timedelta
+from urllib.parse import urljoin, urlparse, urlunparse
+
+from authlib.jose import jwt
+from flask import (
+ flash,
+ jsonify,
+ url_for,
+ request,
+ redirect,
+ Blueprint,
+ current_app as app)
+
+from uploader import session
+from uploader import monadic_requests as mrequests
+from uploader.monadic_requests import make_error_handler
+
+from . import jwks
+from .client import (
+ SCOPE,
+ oauth2_get,
+ user_logged_in,
+ authserver_uri,
+ oauth2_clientid,
+ oauth2_clientsecret)
+
+oauth2 = Blueprint("oauth2", __name__)
+
+@oauth2.route("/code")
+def authorisation_code():
+ """Receive authorisation code from auth server and use it to get token."""
+ def __process_error__(resp_or_exception):
+ app.logger.debug("ERROR: (%s)", resp_or_exception)
+ flash("There was an error retrieving the authorisation token.",
+ "alert-danger")
+ return redirect("/")
+
+ def __fail_set_user_details__(_failure):
+ app.logger.debug("Fetching user details fails: %s", _failure)
+ flash("Could not retrieve the user details", "alert-danger")
+ return redirect("/")
+
+ def __success_set_user_details__(_success):
+ app.logger.debug("Session info: %s", _success)
+ return redirect("/")
+
+ def __success__(token):
+ session.set_user_token(token)
+ return oauth2_get("auth/user/").then(
+ lambda usrdets: session.set_user_details({
+ "user_id": uuid.UUID(usrdets["user_id"]),
+ "name": usrdets["name"],
+ "email": usrdets["email"],
+ "token": session.user_token(),
+ "logged_in": True})).either(
+ __fail_set_user_details__,
+ __success_set_user_details__)
+
+ code = request.args.get("code", "").strip()
+ if not bool(code):
+ flash("AuthorisationError: No code was provided.", "alert-danger")
+ return redirect("/")
+
+ baseurl = urlparse(request.base_url, scheme=request.scheme)
+ issued = datetime.now()
+ jwtkey = jwks.newest_jwk_with_rotation(
+ jwks.jwks_directory(app, "UPLOADER_SECRETS"),
+ int(app.config["JWKS_ROTATION_AGE_DAYS"]))
+ return mrequests.post(
+ urljoin(authserver_uri(), "auth/token"),
+ json={
+ "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
+ "code": code,
+ "scope": SCOPE,
+ "redirect_uri": urljoin(
+ urlunparse(baseurl),
+ url_for("oauth2.authorisation_code")),
+ "assertion": jwt.encode(
+ header={
+ "alg": "RS256",
+ "typ": "JWT",
+ "kid": jwtkey.as_dict()["kid"]
+ },
+ payload={
+ "iss": str(oauth2_clientid()),
+ "sub": request.args["user_id"],
+ "aud": urljoin(authserver_uri(),"auth/token"),
+ "exp": (issued + timedelta(minutes=5)).timestamp(),
+ "nbf": int(issued.timestamp()),
+ "iat": int(issued.timestamp()),
+ "jti": str(uuid.uuid4())
+ },
+ key=jwtkey).decode("utf8"),
+ "client_id": oauth2_clientid()
+ }).either(__process_error__, __success__)
+
+@oauth2.route("/public-jwks")
+def public_jwks():
+ """List the available JWKs"""
+ return jsonify({
+ "documentation": (
+ "The keys are listed in order of creation, from the oldest (first) "
+ "to the newest (last)."),
+ "jwks": tuple(key.as_dict() for key
+ in jwks.list_jwks(jwks.jwks_directory(
+ app, "UPLOADER_SECRETS")))
+ })
+
+
+@oauth2.route("/logout", methods=["GET"])
+def logout():
+ """Log out of any active sessions."""
+ def __unset_session__(session_info):
+ _user = session_info["user"]
+ _user_str = f"{_user['name']} ({_user['email']})"
+ session.clear_session_info()
+ flash("Successfully signed out.", "alert-success")
+ return redirect("/")
+
+ if user_logged_in():
+ return session.user_token().then(
+ lambda _tok: mrequests.post(
+ urljoin(authserver_uri(), "auth/revoke"),
+ json={
+ "token": _tok["refresh_token"],
+ "token_type_hint": "refresh_token",
+ "client_id": oauth2_clientid(),
+ "client_secret": oauth2_clientsecret()
+ })).either(
+ make_error_handler(
+ redirect_to=redirect("/"),
+ cleanup_thunk=lambda: __unset_session__(
+ session.session_info())),
+ lambda res: __unset_session__(session.session_info()))
+ flash("There is no user that is currently logged in.", "alert-info")
+ return redirect("/")
diff --git a/uploader/phenotypes/__init__.py b/uploader/phenotypes/__init__.py
new file mode 100644
index 0000000..c17d32c
--- /dev/null
+++ b/uploader/phenotypes/__init__.py
@@ -0,0 +1,2 @@
+"""Package for handling ('classical') phenotype data"""
+from .views import phenotypesbp
diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py
new file mode 100644
index 0000000..e1ec0c9
--- /dev/null
+++ b/uploader/phenotypes/models.py
@@ -0,0 +1,256 @@
+"""Database and utility functions for phenotypes."""
+from typing import Optional
+from functools import reduce
+from datetime import datetime
+
+import MySQLdb as mdb
+from MySQLdb.cursors import Cursor, DictCursor
+from flask import current_app as app
+
+from gn_libs.mysqldb import debug_query
+
+def datasets_by_population(
+ conn: mdb.Connection,
+ species_id: int,
+ population_id: int
+) -> tuple[dict, ...]:
+ """Retrieve all of a population's phenotype studies."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT s.SpeciesId, pf.* FROM Species AS s "
+ "INNER JOIN InbredSet AS iset ON s.Id=iset.SpeciesId "
+ "INNER JOIN PublishFreeze AS pf ON iset.Id=pf.InbredSetId "
+ "WHERE s.Id=%s AND iset.Id=%s;",
+ (species_id, population_id))
+ return tuple(dict(row) for row in cursor.fetchall())
+
+
+def dataset_by_id(conn: mdb.Connection,
+ species_id: int,
+ population_id: int,
+ dataset_id: int) -> dict:
+ """Fetch dataset details by identifier"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT s.SpeciesId, pf.* FROM Species AS s "
+ "INNER JOIN InbredSet AS iset ON s.Id=iset.SpeciesId "
+ "INNER JOIN PublishFreeze AS pf ON iset.Id=pf.InbredSetId "
+ "WHERE s.Id=%s AND iset.Id=%s AND pf.Id=%s",
+ (species_id, population_id, dataset_id))
+ return dict(cursor.fetchone())
+
+
+def phenotypes_count(conn: mdb.Connection,
+ population_id: int,
+ dataset_id: int) -> int:
+ """Count the number of phenotypes in the dataset."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT COUNT(*) AS total_phenos FROM Phenotype AS pheno "
+ "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId "
+ "INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId "
+ "WHERE pxr.InbredSetId=%s AND pf.Id=%s",
+ (population_id, dataset_id))
+ return int(cursor.fetchone()["total_phenos"])
+
+
+def phenotype_publication_data(conn, phenotype_id) -> Optional[dict]:
+ """Retrieve the publication data for a phenotype if it exists."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT DISTINCT pxr.PhenotypeId, pub.* FROM PublishXRef AS pxr "
+ "INNER JOIN Publication as pub ON pxr.PublicationId=pub.Id "
+ "WHERE pxr.PhenotypeId=%s",
+ (phenotype_id,))
+ res = cursor.fetchone()
+ if res is None:
+ return res
+ return dict(res)
+
+
+def dataset_phenotypes(conn: mdb.Connection,
+ population_id: int,
+ dataset_id: int,
+ offset: int = 0,
+ limit: Optional[int] = None) -> tuple[dict, ...]:
+ """Fetch the actual phenotypes."""
+ _query = (
+ "SELECT pheno.*, pxr.Id AS xref_id, ist.InbredSetCode FROM Phenotype AS pheno "
+ "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId "
+ "INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId "
+ "INNER JOIN InbredSet AS ist ON pf.InbredSetId=ist.Id "
+ "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + (
+ f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "")
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(_query, (population_id, dataset_id))
+ debug_query(cursor, app.logger)
+ return tuple(dict(row) for row in cursor.fetchall())
+
+
+def __phenotype_se__(cursor: Cursor, xref_id, dataids_and_strainids):
+ """Fetch standard-error values (if they exist) for a phenotype."""
+ paramstr = ", ".join(["(%s, %s)"] * len(dataids_and_strainids))
+ flat = tuple(item for sublist in dataids_and_strainids for item in sublist)
+ cursor.execute("SELECT * FROM PublishSE WHERE (DataId, StrainId) IN "
+ f"({paramstr})",
+ flat)
+ debug_query(cursor, app.logger)
+ _se = {
+ (row["DataId"], row["StrainId"]): {
+ "DataId": row["DataId"],
+ "StrainId": row["StrainId"],
+ "error": row["error"]
+ }
+ for row in cursor.fetchall()
+ }
+
+ cursor.execute("SELECT * FROM NStrain WHERE (DataId, StrainId) IN "
+ f"({paramstr})",
+ flat)
+ debug_query(cursor, app.logger)
+ _n = {
+ (row["DataId"], row["StrainId"]): {
+ "DataId": row["DataId"],
+ "StrainId": row["StrainId"],
+ "count": row["count"]
+ }
+ for row in cursor.fetchall()
+ }
+
+ keys = set(tuple(_se.keys()) + tuple(_n.keys()))
+ return {
+ key: {"xref_id": xref_id, **_se.get(key,{}), **_n.get(key,{})}
+ for key in keys
+ }
+
+def __organise_by_phenotype__(pheno, row):
+ """Organise disparate data rows into phenotype 'objects'."""
+ _pheno = pheno.get(row["Id"])
+ return {
+ **pheno,
+ row["Id"]: {
+ "Id": row["Id"],
+ "Pre_publication_description": row["Pre_publication_description"],
+ "Post_publication_description": row["Post_publication_description"],
+ "Original_description": row["Original_description"],
+ "Units": row["Units"],
+ "Pre_publication_abbreviation": row["Pre_publication_abbreviation"],
+ "Post_publication_abbreviation": row["Post_publication_abbreviation"],
+ "xref_id": row["pxr.Id"],
+ "data": {
+ **(_pheno["data"] if bool(_pheno) else {}),
+ (row["DataId"], row["StrainId"]): {
+ "DataId": row["DataId"],
+ "StrainId": row["StrainId"],
+ "mean": row["mean"],
+ "Locus": row["Locus"],
+ "LRS": row["LRS"],
+ "additive": row["additive"],
+ "Sequence": row["Sequence"],
+ "comments": row["comments"],
+ "value": row["value"],
+ "StrainName": row["Name"],
+ "StrainName2": row["Name2"],
+ "StrainSymbol": row["Symbol"],
+ "StrainAlias": row["Alias"]
+ }
+ }
+ }
+ }
+
+
+def __merge_pheno_data_and_se__(data, sedata) -> dict:
+ """Merge phenotype data with the standard errors."""
+ return {
+ key: {**value, **sedata.get(key, {})}
+ for key, value in data.items()
+ }
+
+
+def phenotype_by_id(
+ conn: mdb.Connection,
+ species_id: int,
+ population_id: int,
+ dataset_id: int,
+ xref_id
+) -> Optional[dict]:
+ """Fetch a specific phenotype."""
+ _dataquery = ("SELECT pheno.*, pxr.*, pd.*, str.*, iset.InbredSetCode "
+ "FROM Phenotype AS pheno "
+ "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId "
+ "INNER JOIN PublishData AS pd ON pxr.DataId=pd.Id "
+ "INNER JOIN Strain AS str ON pd.StrainId=str.Id "
+ "INNER JOIN StrainXRef AS sxr ON str.Id=sxr.StrainId "
+ "INNER JOIN PublishFreeze AS pf ON sxr.InbredSetId=pf.InbredSetId "
+ "INNER JOIN InbredSet AS iset ON pf.InbredSetId=iset.InbredSetId "
+ "WHERE "
+ "(str.SpeciesId, pxr.InbredSetId, pf.Id, pxr.Id)=(%s, %s, %s, %s)")
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(_dataquery,
+ (species_id, population_id, dataset_id, xref_id))
+ _pheno: dict = reduce(__organise_by_phenotype__, cursor.fetchall(), {})
+ if bool(_pheno) and len(_pheno.keys()) == 1:
+ _pheno = tuple(_pheno.values())[0]
+ return {
+ **_pheno,
+ "data": tuple(__merge_pheno_data_and_se__(
+ _pheno["data"],
+ __phenotype_se__(
+ cursor, xref_id, tuple(_pheno["data"].keys()))
+ ).values())
+ }
+ if bool(_pheno) and len(_pheno.keys()) > 1:
+ raise Exception(
+ "We found more than one phenotype with the same identifier!")
+
+ return None
+
+
+def phenotypes_data(conn: mdb.Connection,
+ population_id: int,
+ dataset_id: int,
+ offset: int = 0,
+ limit: Optional[int] = None) -> tuple[dict, ...]:
+ """Fetch the data for the phenotypes."""
+ # — Phenotype -> PublishXRef -> PublishData -> Strain -> StrainXRef -> PublishFreeze
+ _query = ("SELECT pheno.*, pxr.*, pd.*, str.*, iset.InbredSetCode "
+ "FROM Phenotype AS pheno "
+ "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId "
+ "INNER JOIN PublishData AS pd ON pxr.DataId=pd.Id "
+ "INNER JOIN Strain AS str ON pd.StrainId=str.Id "
+ "INNER JOIN StrainXRef AS sxr ON str.Id=sxr.StrainId "
+ "INNER JOIN PublishFreeze AS pf ON sxr.InbredSetId=pf.InbredSetId "
+ "INNER JOIN InbredSet AS iset ON pf.InbredSetId=iset.InbredSetId "
+ "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + (
+ f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "")
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(_query, (population_id, dataset_id))
+ debug_query(cursor, app.logger)
+ return tuple(dict(row) for row in cursor.fetchall())
+
+
+def save_new_dataset(cursor: Cursor,
+ population_id: int,
+ dataset_name: str,
+ dataset_fullname: str,
+ dataset_shortname: str) -> dict:
+ """Create a new phenotype dataset."""
+ params = {
+ "population_id": population_id,
+ "dataset_name": dataset_name,
+ "dataset_fullname": dataset_fullname,
+ "dataset_shortname": dataset_shortname,
+ "created": datetime.now().date().isoformat(),
+ "public": 2,
+ "confidentiality": 0,
+ "users": None
+ }
+ cursor.execute(
+ "INSERT INTO PublishFreeze(Name, FullName, ShortName, CreateTime, "
+ "public, InbredSetId, confidentiality, AuthorisedUsers) "
+ "VALUES(%(dataset_name)s, %(dataset_fullname)s, %(dataset_shortname)s, "
+ "%(created)s, %(public)s, %(population_id)s, %(confidentiality)s, "
+ "%(users)s)",
+ params)
+ debug_query(cursor, app.logger)
+ return {**params, "Id": cursor.lastrowid}
diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py
new file mode 100644
index 0000000..bcbb3a9
--- /dev/null
+++ b/uploader/phenotypes/views.py
@@ -0,0 +1,864 @@
+"""Views handling ('classical') phenotypes."""
+import sys
+import uuid
+import json
+import datetime
+from typing import Any
+from pathlib import Path
+from zipfile import ZipFile
+from functools import wraps, reduce
+from logging import INFO, ERROR, DEBUG, FATAL, CRITICAL, WARNING
+
+from redis import Redis
+from pymonad.either import Left
+from requests.models import Response
+from MySQLdb.cursors import DictCursor
+from gn_libs.mysqldb import database_connection
+from flask import (flash,
+ request,
+ url_for,
+ jsonify,
+ redirect,
+ Blueprint,
+ current_app as app)
+
+# from r_qtl import r_qtl2 as rqtl2
+from r_qtl import r_qtl2_qc as rqc
+from r_qtl import exceptions as rqe
+
+from uploader import jobs
+from uploader.files import save_file#, fullpath
+from uploader.ui import make_template_renderer
+from uploader.oauth2.client import oauth2_post
+from uploader.authorisation import require_login
+from uploader.species.models import all_species, species_by_id
+from uploader.monadic_requests import make_either_error_handler
+from uploader.request_checks import with_species, with_population
+from uploader.datautils import safe_int, order_by_family, enumerate_sequence
+from uploader.population.models import (populations_by_species,
+ population_by_species_and_id)
+from uploader.input_validation import (encode_errors,
+ decode_errors,
+ is_valid_representative_name)
+
+from .models import (dataset_by_id,
+ phenotype_by_id,
+ phenotypes_count,
+ save_new_dataset,
+ dataset_phenotypes,
+ datasets_by_population,
+ phenotype_publication_data)
+
+phenotypesbp = Blueprint("phenotypes", __name__)
+render_template = make_template_renderer("phenotypes")
+
+_FAMILIES_WITH_SE_AND_N_ = (
+ "Reference Populations (replicate average, SE, N)",)
+
+@phenotypesbp.route("/phenotypes", methods=["GET"])
+@require_login
+def index():
+ """Direct entry-point for phenotypes data handling."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ if not bool(request.args.get("species_id")):
+ return render_template("phenotypes/index.html",
+ species=all_species(conn),
+ activelink="phenotypes")
+
+ species_id = request.args.get("species_id")
+ if species_id == "CREATE-SPECIES":
+ return redirect(url_for(
+ "species.create_species",
+ return_to="species.populations.phenotypes.select_population"))
+
+ species = species_by_id(conn, species_id)
+ if not bool(species):
+ flash("No such species!", "alert-danger")
+ return redirect(url_for("species.populations.phenotypes.index"))
+ return redirect(url_for("species.populations.phenotypes.select_population",
+ species_id=species["SpeciesId"]))
+
+
+@phenotypesbp.route("<int:species_id>/phenotypes/select-population",
+ methods=["GET"])
+@require_login
+@with_species(redirect_uri="species.populations.phenotypes.index")
+def select_population(species: dict, **kwargs):# pylint: disable=[unused-argument]
+ """Select the population for your phenotypes."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ if not bool(request.args.get("population_id")):
+ return render_template("phenotypes/select-population.html",
+ species=species,
+ populations=populations_by_species(
+ conn, species["SpeciesId"]),
+ activelink="phenotypes")
+
+ population_id = request.args["population_id"]
+ if population_id == "CREATE-POPULATION":
+ return redirect(url_for(
+ "species.populations.create_population",
+ species_id=species["SpeciesId"],
+ return_to="species.populations.phenotypes.list_datasets"))
+ population = population_by_species_and_id(
+ conn, species["SpeciesId"], int(population_id))
+ if not bool(population):
+ flash("No such population found!", "alert-danger")
+ return redirect(url_for(
+ "species.populations.phenotypes.select_population",
+ species_id=species["SpeciesId"]))
+
+ return redirect(url_for("species.populations.phenotypes.list_datasets",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"]))
+
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets",
+ methods=["GET"])
+@require_login
+@with_population(species_redirect_uri="species.populations.phenotypes.index",
+ redirect_uri="species.populations.phenotypes.select_population")
+def list_datasets(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument]
+ """List available phenotype datasets."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ datasets = datasets_by_population(
+ conn, species["SpeciesId"], population["Id"])
+ if len(datasets) == 1:
+ return redirect(url_for(
+ "species.populations.phenotypes.view_dataset",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ dataset_id=datasets[0]["Id"]))
+ return render_template("phenotypes/list-datasets.html",
+ species=species,
+ population=population,
+ datasets=datasets,
+ activelink="list-datasets")
+
+
+def with_dataset(
+ species_redirect_uri: str,
+ population_redirect_uri: str,
+ redirect_uri: str
+):
+ """Ensure the dataset actually exists."""
+ def __decorator__(func):
+ @wraps(func)
+ @with_population(species_redirect_uri, population_redirect_uri)
+ def __with_dataset__(**kwargs):
+ try:
+ _spcid = int(kwargs["species_id"])
+ _popid = int(kwargs["population_id"])
+ _dsetid = int(kwargs.get("dataset_id"))
+ select_dataset_uri = redirect(url_for(
+ redirect_uri, species_id=_spcid, population_id=_popid))
+ if not bool(_dsetid):
+ flash("You need to select a valid 'dataset_id' value.",
+ "alert-danger")
+ return select_dataset_uri
+ with database_connection(app.config["SQL_URI"]) as conn:
+ dataset = dataset_by_id(conn, _spcid, _popid, _dsetid)
+ if not bool(dataset):
+ flash("You must select a valid dataset.",
+ "alert-danger")
+ return select_dataset_uri
+ except ValueError as _verr:
+ app.logger.debug(
+ "Exception converting 'dataset_id' to integer: %s",
+ kwargs.get("dataset_id"),
+ exc_info=True)
+ flash("Expected 'dataset_id' value to be an integer."
+ "alert-danger")
+ return select_dataset_uri
+ return func(dataset=dataset, **kwargs)
+ return __with_dataset__
+ return __decorator__
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/view",
+ methods=["GET"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def view_dataset(# pylint: disable=[unused-argument]
+ species: dict, population: dict, dataset: dict, **kwargs):
+ """View a specific dataset"""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ dataset = dataset_by_id(
+ conn, species["SpeciesId"], population["Id"], dataset["Id"])
+ if not bool(dataset):
+ flash("Could not find such a phenotype dataset!", "alert-danger")
+ return redirect(url_for(
+ "species.populations.phenotypes.list_datasets",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"]))
+
+ start_at = max(safe_int(request.args.get("start_at") or 0), 0)
+ count = int(request.args.get("count") or 20)
+ return render_template("phenotypes/view-dataset.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ phenotype_count=phenotypes_count(
+ conn, population["Id"], dataset["Id"]),
+ phenotypes=enumerate_sequence(
+ dataset_phenotypes(
+ conn,
+ population["Id"],
+ dataset["Id"])),
+ start_from=start_at,
+ count=count,
+ activelink="view-dataset")
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/phenotype/<xref_id>",
+ methods=["GET"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def view_phenotype(# pylint: disable=[unused-argument]
+ species: dict,
+ population: dict,
+ dataset: dict,
+ xref_id: int,
+ **kwargs
+):
+ """View an individual phenotype from the dataset."""
+ def __render__(privileges):
+ phenotype = phenotype_by_id(conn,
+ species["SpeciesId"],
+ population["Id"],
+ dataset["Id"],
+ xref_id)
+ def __non_empty__(value) -> bool:
+ if isinstance(value, str):
+ return value.strip() != ""
+ return bool(value)
+
+ return render_template(
+ "phenotypes/view-phenotype.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ xref_id=xref_id,
+ phenotype=phenotype,
+ has_se=any(bool(item.get("error")) for item in phenotype["data"]),
+ publish_data={
+ key.replace("_", " "): val
+ for key,val in
+ (phenotype_publication_data(conn, phenotype["Id"]) or {}).items()
+ if (key in ("PubMed_ID", "Authors", "Title", "Journal")
+ and __non_empty__(val))
+ },
+ privileges=(privileges
+ ### For demo! Do not commit this part
+ + ("group:resource:edit-resource",
+ "group:resource:delete-resource",)
+ ### END: For demo! Do not commit this part
+ ),
+ activelink="view-phenotype")
+
+ def __fail__(error):
+ if isinstance(error, Response) and error.json() == "No linked resource!":
+ return __render__(tuple())
+ return make_either_error_handler(
+ "There was an error fetching the roles and privileges.")(error)
+
+ with database_connection(app.config["SQL_URI"]) as conn:
+ return oauth2_post(
+ "/auth/resource/phenotypes/individual/linked-resource",
+ json={
+ "species_id": species["SpeciesId"],
+ "population_id": population["Id"],
+ "dataset_id": dataset["Id"],
+ "xref_id": xref_id
+ }
+ ).then(
+ lambda resource: tuple(
+ privilege["privilege_id"] for role in resource["roles"]
+ for privilege in role["privileges"])
+ ).then(__render__).either(__fail__, lambda resp: resp)
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets/create",
+ methods=["GET", "POST"])
+@require_login
+@with_population(
+ species_redirect_uri="species.populations.phenotypes.index",
+ redirect_uri="species.populations.phenotypes.select_population")
+def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument]
+ """Create a new phenotype dataset."""
+ with (database_connection(app.config["SQL_URI"]) as conn,
+ conn.cursor(cursorclass=DictCursor) as cursor):
+ if request.method == "GET":
+ return render_template("phenotypes/create-dataset.html",
+ activelink="create-dataset",
+ species=species,
+ population=population,
+ **decode_errors(
+ request.args.get("error_values", "")))
+
+ form = request.form
+ _errors: tuple[tuple[str, str], ...] = tuple()
+ if not is_valid_representative_name(
+ (form.get("dataset-name") or "").strip()):
+ _errors = _errors + (("dataset-name", "Invalid dataset name."),)
+
+ if not bool((form.get("dataset-fullname") or "").strip()):
+ _errors = _errors + (("dataset-fullname",
+ "You must provide a value for 'Full Name'."),)
+
+ if bool(_errors) > 0:
+ return redirect(url_for(
+ "species.populations.phenotypes.create_dataset",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ error_values=encode_errors(_errors, form)))
+
+ dataset_shortname = (
+ form["dataset-shortname"] or form["dataset-name"]).strip()
+ _pheno_dataset = save_new_dataset(
+ cursor,
+ population["Id"],
+ form["dataset-name"].strip(),
+ form["dataset-fullname"].strip(),
+ dataset_shortname)
+ return redirect(url_for("species.populations.phenotypes.list_datasets",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"]))
+
+
+def process_phenotypes_rqtl2_bundle(error_uri):
+ """Process phenotypes from the uploaded R/qtl2 bundle."""
+ _redisuri = app.config["REDIS_URL"]
+ _sqluri = app.config["SQL_URI"]
+ try:
+ ## Handle huge files here...
+ phenobundle = save_file(request.files["phenotypes-bundle"],
+ Path(app.config["UPLOAD_FOLDER"]))
+ rqc.validate_bundle(phenobundle)
+ return phenobundle
+ except AssertionError as _aerr:
+ app.logger.debug("File upload error!", exc_info=True)
+ flash("Expected a zipped bundle of files with phenotypes' "
+ "information.",
+ "alert-danger")
+ return error_uri
+ except rqe.RQTLError as rqtlerr:
+ app.logger.debug("Bundle validation error!", exc_info=True)
+ flash("R/qtl2 Error: " + " ".join(rqtlerr.args), "alert-danger")
+ return error_uri
+
+
+def process_phenotypes_individual_files(error_uri):
+ """Process the uploaded individual files."""
+ form = request.form
+ cdata = {
+ "sep": form["file-separator"],
+ "comment.char": form["file-comment-character"],
+ "na.strings": form["file-na"].split(" "),
+ }
+ bundlepath = Path(app.config["UPLOAD_FOLDER"],
+ f"{str(uuid.uuid4()).replace('-', '')}.zip")
+ with ZipFile(bundlepath,mode="w") as zfile:
+ for rqtlkey, formkey in (("phenocovar", "phenotype-descriptions"),
+ ("pheno", "phenotype-data"),
+ ("phenose", "phenotype-se"),
+ ("phenonum", "phenotype-n")):
+ if form.get("resumable-upload", False):
+ # Chunked upload of large files was used
+ filedata = json.loads(form[formkey])
+ zfile.write(
+ Path(app.config["UPLOAD_FOLDER"], filedata["uploaded-file"]),
+ arcname=filedata["original-name"])
+ cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filedata["original-name"]]
+ else:
+ # TODO: Check this path: fix any bugs.
+ _sentfile = request.files[formkey]
+ if not bool(_sentfile):
+ flash(f"Expected file ('{formkey}') was not provided.",
+ "alert-danger")
+ return error_uri
+
+ filepath = save_file(
+ _sentfile, Path(app.config["UPLOAD_FOLDER"]), hashed=False)
+ zfile.write(
+ Path(app.config["UPLOAD_FOLDER"], filepath),
+ arcname=filepath.name)
+ cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filepath.name]
+
+ zfile.writestr("control_data.json", data=json.dumps(cdata, indent=2))
+
+ return bundlepath
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/add-phenotypes",
+ methods=["GET", "POST"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# pylint: disable=[unused-argument, too-many-locals]
+ """Add one or more phenotypes to the dataset."""
+ use_bundle = request.args.get("use_bundle", "").lower() == "true"
+ add_phenos_uri = redirect(url_for(
+ "species.populations.phenotypes.add_phenotypes",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ dataset_id=dataset["Id"]))
+ _redisuri = app.config["REDIS_URL"]
+ _sqluri = app.config["SQL_URI"]
+ with (Redis.from_url(_redisuri, decode_responses=True) as rconn,
+ # database_connection(_sqluri) as conn,
+ # conn.cursor(cursorclass=DictCursor) as cursor
+ ):
+ if request.method == "GET":
+ today = datetime.date.today()
+ return render_template(
+ ("phenotypes/add-phenotypes-with-rqtl2-bundle.html"
+ if use_bundle else "phenotypes/add-phenotypes-raw-files.html"),
+ species=species,
+ population=population,
+ dataset=dataset,
+ monthnames=(
+ "January", "February", "March", "April",
+ "May", "June", "July", "August",
+ "September", "October", "November",
+ "December"),
+ current_month=today.strftime("%B"),
+ current_year=int(today.strftime("%Y")),
+ families_with_se_and_n=_FAMILIES_WITH_SE_AND_N_,
+ use_bundle=use_bundle,
+ activelink="add-phenotypes")
+
+ phenobundle = (process_phenotypes_rqtl2_bundle(add_phenos_uri)
+ if use_bundle else
+ process_phenotypes_individual_files(add_phenos_uri))
+
+ _jobid = uuid.uuid4()
+ _namespace = jobs.jobsnamespace()
+ _ttl_seconds = app.config["JOBS_TTL_SECONDS"]
+ _job = jobs.launch_job(
+ jobs.initialise_job(
+ rconn,
+ _namespace,
+ str(_jobid),
+ [sys.executable, "-m", "scripts.rqtl2.phenotypes_qc", _sqluri,
+ _redisuri, _namespace, str(_jobid), str(species["SpeciesId"]),
+ str(population["Id"]),
+ # str(dataset["Id"]),
+ str(phenobundle),
+ "--loglevel",
+ {
+ INFO: "INFO",
+ ERROR: "ERROR",
+ DEBUG: "DEBUG",
+ FATAL: "FATAL",
+ CRITICAL: "CRITICAL",
+ WARNING: "WARNING"
+ }[app.logger.getEffectiveLevel()],
+ "--redisexpiry",
+ str(_ttl_seconds)], "phenotype_qc", _ttl_seconds,
+ {"job-metadata": json.dumps({
+ "speciesid": species["SpeciesId"],
+ "populationid": population["Id"],
+ "datasetid": dataset["Id"],
+ "bundle": str(phenobundle.absolute())})}),
+ _redisuri,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+ app.logger.debug("JOB DETAILS: %s", _job)
+ jobstatusuri = url_for("species.populations.phenotypes.job_status",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ dataset_id=dataset["Id"],
+ job_id=str(_job["jobid"]))
+ return ((jsonify({
+ "redirect-to": jobstatusuri,
+ "statuscode": 200,
+ "message": ("Follow the 'redirect-to' URI to see the state "
+ "of the quality-control job started for your "
+ "uploaded files.")
+ }), 200)
+ if request.form.get("resumable-upload", False) else
+ redirect(jobstatusuri))
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/job/<uuid:job_id>",
+ methods=["GET"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def job_status(
+ species: dict,
+ population: dict,
+ dataset: dict,
+ job_id: uuid.UUID,
+ **kwargs
+):# pylint: disable=[unused-argument]
+ """Retrieve current status of a particular phenotype QC job."""
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ try:
+ job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id))
+ except jobs.JobNotFound as _jnf:
+ job = None
+ return render_template("phenotypes/job-status.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ job_id=job_id,
+ job=job,
+ errors=jobs.job_errors(
+ rconn, jobs.jobsnamespace(), job['jobid']),
+ metadata=jobs.job_files_metadata(
+ rconn, jobs.jobsnamespace(), job['jobid']),
+ activelink="add-phenotypes")
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/job/<uuid:job_id>/review",
+ methods=["GET"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def review_job_data(
+ species: dict,
+ population: dict,
+ dataset: dict,
+ job_id: uuid.UUID,
+ **kwargs
+):# pylint: disable=[unused-argument]
+ """Review data one more time before entering it into the database."""
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ try:
+ job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id))
+ except jobs.JobNotFound as _jnf:
+ job = None
+
+ def __metadata_by_type__(by_type, item):
+ filetype = item[1]["filetype"]
+ return {
+ **by_type,
+ filetype: (by_type.get(filetype, tuple())
+ + ({"filename": item[0], **item[1]},))
+ }
+ metadata: dict[str, Any] = reduce(
+ __metadata_by_type__,
+ (jobs.job_files_metadata(
+ rconn, jobs.jobsnamespace(), job['jobid'])
+ if job else {}).items(),
+ {})
+
+ def __desc__(filetype):
+ match filetype:
+ case "phenocovar":
+ desc = "phenotypes"
+ case "pheno":
+ desc = "phenotypes data"
+ case "phenose":
+ desc = "phenotypes standard-errors"
+ case "phenonum":
+ desc = "phenotypes samples"
+ case _:
+ desc = f"unknown file type '{filetype}'."
+
+ return desc
+
+ def __summarise__(filetype, files):
+ return {
+ "filetype": filetype,
+ "number-of-files": len(files),
+ "total-data-rows": sum(
+ int(afile["linecount"]) - 1 for afile in files),
+ "description": __desc__(filetype)
+ }
+
+ summary = {
+ filetype: __summarise__(filetype, meta)
+ for filetype,meta in metadata.items()
+ }
+ return render_template("phenotypes/review-job-data.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ job_id=job_id,
+ job=job,
+ summary=summary,
+ activelink="add-phenotypes")
+
+
+def update_phenotype_metadata(conn, metadata: dict):
+ """Update a phenotype's basic metadata values."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM Phenotype WHERE Id=%(phenotype-id)s",
+ metadata)
+ res = {
+ **{
+ _key: _val for _key,_val in {
+ key.lower().replace("_", "-"): value
+ for key, value in (cursor.fetchone() or {}).items()
+ }.items()
+ if _key in metadata.keys()
+ },
+ "phenotype-id": metadata.get("phenotype-id")
+ }
+ if res == metadata:
+ return False
+
+ cursor.execute(
+ "UPDATE Phenotype SET "
+ "Pre_publication_description=%(pre-publication-description)s, "
+ "Post_publication_description=%(post-publication-description)s, "
+ "Original_description=%(original-description)s, "
+ "Units=%(units)s, "
+ "Pre_publication_abbreviation=%(pre-publication-abbreviation)s, "
+ "Post_publication_abbreviation=%(post-publication-abbreviation)s "
+ "WHERE Id=%(phenotype-id)s",
+ metadata)
+ return cursor.rowcount
+
+
+def update_phenotype_values(conn, values):
+ """Update a phenotype's data values."""
+ with conn.cursor() as cursor:
+ cursor.executemany(
+ "UPDATE PublishData SET value=%(new)s "
+ "WHERE Id=%(data_id)s AND StrainId=%(strain_id)s",
+ tuple(item for item in values if item["new"] is not None))
+ cursor.executemany(
+ "DELETE FROM PublishData "
+ "WHERE Id=%(data_id)s AND StrainId=%(strain_id)s",
+ tuple(item for item in values if item["new"] is None))
+ return len(values)
+ return 0
+
+
+def update_phenotype_se(conn, serrs):
+ """Update a phenotype's standard-error values."""
+ with conn.cursor() as cursor:
+ cursor.executemany(
+ "INSERT INTO PublishSE(DataId, StrainId, error) "
+ "VALUES(%(data_id)s, %(strain_id)s, %(new)s) "
+ "ON DUPLICATE KEY UPDATE error=VALUES(error)",
+ tuple(item for item in serrs if item["new"] is not None))
+ cursor.executemany(
+ "DELETE FROM PublishSE "
+ "WHERE DataId=%(data_id)s AND StrainId=%(strain_id)s",
+ tuple(item for item in serrs if item["new"] is None))
+ return len(serrs)
+ return 0
+
+
+def update_phenotype_n(conn, counts):
+ """Update a phenotype's strain counts."""
+ with conn.cursor() as cursor:
+ cursor.executemany(
+ "INSERT INTO NStrain(DataId, StrainId, count) "
+ "VALUES(%(data_id)s, %(strain_id)s, %(new)s) "
+ "ON DUPLICATE KEY UPDATE count=VALUES(count)",
+ tuple(item for item in counts if item["new"] is not None))
+ cursor.executemany(
+ "DELETE FROM NStrain "
+ "WHERE DataId=%(data_id)s AND StrainId=%(strain_id)s",
+ tuple(item for item in counts if item["new"] is None))
+ return len(counts)
+
+ return 0
+
+
+def update_phenotype_data(conn, data: dict):
+ """Update the numeric data for a phenotype."""
+ def __organise_by_dataid_and_strainid__(acc, current):
+ _key, dataid, strainid = current[0].split("::")
+ _keysrc, _keytype = _key.split("-")
+ newkey = f"{dataid}::{strainid}"
+ newitem = acc.get(newkey, {})
+ newitem[_keysrc] = newitem.get(_keysrc, {})
+ newitem[_keysrc][_keytype] = current[1]
+ return {**acc, newkey: newitem}
+
+ def __separate_items__(acc, row):
+ key, val = row
+ return ({
+ **acc[0],
+ key: {
+ **val["value"],
+ "changed?": (not val["value"]["new"] == val["value"]["original"])
+ }
+ }, {
+ **acc[1],
+ key: {
+ **val["se"],
+ "changed?": (not val["se"]["new"] == val["se"]["original"])
+ }
+ },{
+ **acc[2],
+ key: {
+ **val["n"],
+ "changed?": (not val["n"]["new"] == val["n"]["original"])
+ }
+ })
+
+ values, serrs, counts = tuple(
+ tuple({
+ "data_id": row[0].split("::")[0],
+ "strain_id": row[0].split("::")[1],
+ "new": row[1]["new"]
+ } for row in item)
+ for item in (
+ filter(lambda val: val[1]["changed?"], item.items())# type: ignore[arg-type]
+ for item in reduce(# type: ignore[var-annotated]
+ __separate_items__,
+ reduce(__organise_by_dataid_and_strainid__,
+ data.items(),
+ {}).items(),
+ ({}, {}, {}))))
+
+ return (update_phenotype_values(conn, values),
+ update_phenotype_se(conn, serrs),
+ update_phenotype_n(conn, counts))
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/phenotype/<int:xref_id>/edit",
+ methods=["GET", "POST"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def edit_phenotype_data(# pylint: disable=[unused-argument]
+ species: dict,
+ population: dict,
+ dataset: dict,
+ xref_id: int,
+ **kwargs
+):
+ """Edit the data for a particular phenotype."""
+ def __render__(**kwargs):
+ processed_kwargs = {
+ **kwargs,
+ "privileges": (kwargs.get("privileges", tuple())
+ ### For demo! Do not commit this part
+ + ("group:resource:edit-resource",
+ "group:resource:delete-resource",)
+ ### END: For demo! Do not commit this part
+ )
+ }
+ return render_template(
+ "phenotypes/edit-phenotype.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ xref_id=xref_id,
+ families_with_se_and_n=_FAMILIES_WITH_SE_AND_N_,
+ **processed_kwargs,
+ activelink="edit-phenotype")
+
+ with database_connection(app.config["SQL_URI"]) as conn:
+ if request.method == "GET":
+ def __fetch_phenotype__(privileges):
+ phenotype = phenotype_by_id(conn,
+ species["SpeciesId"],
+ population["Id"],
+ dataset["Id"],
+ xref_id)
+ if phenotype is None:
+ msg = ("Could not find the phenotype with cross-reference ID"
+ f" '{xref_id}' from dataset '{dataset['FullName']}' "
+ f" from the '{population['FullName']}' population of "
+ f" species '{species['FullName']}'.")
+ return Left({"privileges": privileges, "phenotype-error": msg})
+ return {"privileges": privileges, "phenotype": phenotype}
+
+ def __fetch_publication_data__(**kwargs):
+ pheno = kwargs["phenotype"]
+ return {
+ **kwargs,
+ "publication_data": phenotype_publication_data(
+ conn, pheno["Id"])
+ }
+
+ def __fail__(failure_object):
+ # process the object
+ return __render__(failure_object=failure_object)
+
+ return oauth2_post(
+ "/auth/resource/phenotypes/individual/linked-resource",
+ json={
+ "species_id": species["SpeciesId"],
+ "population_id": population["Id"],
+ "dataset_id": dataset["Id"],
+ "xref_id": xref_id
+ }
+ ).then(
+ lambda resource: tuple(
+ privilege["privilege_id"] for role in resource["roles"]
+ for privilege in role["privileges"])
+ ).then(
+ __fetch_phenotype__
+ ).then(
+ lambda args: __fetch_publication_data__(**args)
+ ).either(__fail__, lambda args: __render__(**args))
+
+ ## POST
+ _change = False
+ match request.form.get("submit", "invalid-action"):
+ case "update basic metadata":
+ _change = update_phenotype_metadata(conn, {
+ key: value.strip() if bool(value.strip()) else None
+ for key, value in request.form.items()
+ if key not in ("submit",)
+ })
+ msg = "Basic metadata was updated successfully."
+ case "update data":
+ _update = update_phenotype_data(conn, {
+ key: value.strip() if bool(value.strip()) else None
+ for key, value in request.form.items()
+ if key not in ("submit",)
+ })
+ msg = (f"{_update[0]} value rows, {_update[1]} standard-error "
+ f"rows and {_update[2]} 'N' rows were updated.")
+ _change = any(item != 0 for item in _update)
+ case "update publication":
+ flash("NOT IMPLEMENTED: Would update publication data.", "alert-success")
+ case _:
+ flash("Invalid phenotype editing action.", "alert-danger")
+
+ if _change:
+ flash(msg, "alert-success")
+ return redirect(url_for(
+ "species.populations.phenotypes.view_phenotype",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ dataset_id=dataset["Id"],
+ xref_id=xref_id))
+
+ flash("No change was made by the user.", "alert-info")
+ return redirect(url_for(
+ "species.populations.phenotypes.edit_phenotype_data",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ dataset_id=dataset["Id"],
+ xref_id=xref_id))
diff --git a/uploader/platforms/__init__.py b/uploader/platforms/__init__.py
new file mode 100644
index 0000000..8cb89c9
--- /dev/null
+++ b/uploader/platforms/__init__.py
@@ -0,0 +1,2 @@
+"""Module to handle management of genetic platforms."""
+from .views import platformsbp
diff --git a/uploader/platforms/models.py b/uploader/platforms/models.py
new file mode 100644
index 0000000..a859371
--- /dev/null
+++ b/uploader/platforms/models.py
@@ -0,0 +1,95 @@
+"""Handle db interactions for platforms."""
+from typing import Optional
+
+import MySQLdb as mdb
+from MySQLdb.cursors import Cursor, DictCursor
+
+def platforms_by_species(
+ conn: mdb.Connection,
+ speciesid: int,
+ offset: int = 0,
+ limit: Optional[int] = None
+) -> tuple[dict, ...]:
+ """Retrieve platforms by the species"""
+ _query = ("SELECT * FROM GeneChip WHERE SpeciesId=%s "
+ "ORDER BY GeneChipName ASC")
+ if bool(limit) and limit > 0:# type: ignore[operator]
+ _query = f"{_query} LIMIT {limit} OFFSET {offset}"
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(_query, (speciesid,))
+ return tuple(dict(row) for row in cursor.fetchall())
+
+
+def species_platforms_count(conn: mdb.Connection, species_id: int) -> int:
+ """Get the number of platforms in the database for a particular species."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT COUNT(GeneChipName) AS count FROM GeneChip "
+ "WHERE SpeciesId=%s",
+ (species_id,))
+ return int(cursor.fetchone()["count"])
+
+
+def platform_by_id(conn: mdb.Connection, platformid: int) -> Optional[dict]:
+ """Retrieve a platform by its ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM GeneChip WHERE Id=%s",
+ (platformid,))
+ result = cursor.fetchone()
+ if bool(result):
+ return dict(result)
+
+ return None
+
+
+def platform_by_species_and_id(
+ conn: mdb.Connection, species_id: int, platformid: int
+) -> Optional[dict]:
+ """Retrieve a platform by its species and ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM GeneChip WHERE SpeciesId=%s AND Id=%s",
+ (species_id, platformid))
+ result = cursor.fetchone()#pylint: disable=[duplicate-code]
+ if bool(result):
+ return dict(result)
+
+ return None
+
+
+def save_new_platform(# pylint: disable=[too-many-arguments]
+ cursor: Cursor,
+ species_id: int,
+ geo_platform: str,
+ platform_name: str,
+ platform_shortname: str,
+ platform_title: str,
+ go_tree_value: Optional[str]
+) -> dict:
+ """Save a new platform to the database."""
+ params = {
+ "species_id": species_id,
+ "GeoPlatform": geo_platform,
+ "GeneChipName": platform_name,
+ "Name": platform_shortname,
+ "Title": platform_title,
+ "GO_tree_value": go_tree_value
+ }
+ cursor.execute("SELECT SpeciesId, GeoPlatform FROM GeneChip")
+ assert (species_id, geo_platform) not in (
+ (row["SpeciesId"], row["GeoPlatform"]) for row in cursor.fetchall())
+ cursor.execute(
+ "INSERT INTO "
+ "GeneChip(SpeciesId, GeneChipName, Name, GeoPlatform, Title, GO_tree_value) "
+ "VALUES("
+ "%(species_id)s, %(GeneChipName)s, %(Name)s, %(GeoPlatform)s, "
+ "%(Title)s, %(GO_tree_value)s"
+ ")",
+ params)
+ new_id = cursor.lastrowid
+ cursor.execute("UPDATE GeneChip SET GeneChipId=%s WHERE Id=%s",
+ (new_id, new_id))
+ return {
+ **params,
+ "Id": new_id,
+ "GeneChipId": new_id
+ }
diff --git a/uploader/platforms/views.py b/uploader/platforms/views.py
new file mode 100644
index 0000000..114c1a9
--- /dev/null
+++ b/uploader/platforms/views.py
@@ -0,0 +1,118 @@
+"""The endpoints for the platforms"""
+from MySQLdb.cursors import DictCursor
+from gn_libs.mysqldb import database_connection
+from flask import (
+ flash,
+ request,
+ url_for,
+ redirect,
+ Blueprint,
+ current_app as app)
+
+from uploader.ui import make_template_renderer
+from uploader.authorisation import require_login
+from uploader.species.models import all_species, species_by_id
+from uploader.datautils import safe_int, order_by_family, enumerate_sequence
+
+from .models import (save_new_platform,
+ platforms_by_species,
+ species_platforms_count)
+
+platformsbp = Blueprint("platforms", __name__)
+render_template = make_template_renderer("platforms")
+
+@platformsbp.route("platforms", methods=["GET"])
+@require_login
+def index():
+ """Entry-point to the platforms feature."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ if not bool(request.args.get("species_id")):
+ return render_template(
+ "platforms/index.html",
+ species=all_species(conn),
+ activelink="platforms")
+
+ species_id = request.args.get("species_id")
+ if species_id == "CREATE-SPECIES":
+ return redirect(url_for(
+ "species.create_species",
+ return_to="species.platforms.list_platforms"))
+
+ species = species_by_id(conn, request.args["species_id"])
+ if not bool(species):
+ flash("No species selected.", "alert-danger")
+ return redirect(url_for("species.platforms.index"))
+
+ return redirect(url_for("species.platforms.list_platforms",
+ species_id=species["SpeciesId"]))
+
+
+@platformsbp.route("<int:species_id>/platforms", methods=["GET"])
+@require_login
+def list_platforms(species_id: int):
+ """List all the available genetic sequencing platforms."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ if not bool(species):
+ flash("No species provided.", "alert-danger")
+ return redirect(url_for("species.platforms.index"))
+
+ start_from = max(safe_int(request.args.get("start_from") or 0), 0)
+ count = safe_int(request.args.get("count") or 20)
+ return render_template(
+ "platforms/list-platforms.html",
+ species=species,
+ platforms=enumerate_sequence(
+ platforms_by_species(conn,
+ species_id,
+ offset=start_from,
+ limit=count),
+ start=start_from+1),
+ start_from=start_from,
+ count=count,
+ total_platforms=species_platforms_count(conn, species_id),
+ activelink="list-platforms")
+
+
+@platformsbp.route("<int:species_id>/platforms/create", methods=["GET", "POST"])
+@require_login
+def create_platform(species_id: int):
+ """Create a new genetic sequencing platform."""
+ with (database_connection(app.config["SQL_URI"]) as conn,
+ conn.cursor(cursorclass=DictCursor) as cursor):
+ species = species_by_id(conn, species_id)
+ if not bool(species):
+ flash("No species provided.", "alert-danger")
+ return redirect(url_for("species.platforms.index"))
+
+ if request.method == "GET":
+ return render_template(
+ "platforms/create-platform.html",
+ species=species,
+ activelink="create-platform")
+
+ try:
+ form = request.form
+ _new_platform = save_new_platform(
+ cursor,
+ species_id,
+ form["geo-platform"],
+ form["platform-name"],
+ form["platform-shortname"],
+ form["platform-title"],
+ form.get("go-tree-value") or None)
+ except KeyError as _kerr:
+ flash(f"Required value for field {_kerr.args[0]} was not provided.",
+ "alert-danger")
+ return redirect(url_for("species.platforms.create_platform",
+ species_id=species_id))
+ except AssertionError as _aerr:
+ flash(f"Platform with GeoPlatform value of '{form['geo-platform']}'"
+ f" already exists for species '{species['FullName']}'.",
+ "alert-danger")
+ return redirect(url_for("species.platforms.create_platform",
+ species_id=species_id))
+
+ flash("Platform created successfully", "alert-success")
+ return redirect(url_for("species.platforms.list_platforms",
+ species_id=species_id))
diff --git a/uploader/population/__init__.py b/uploader/population/__init__.py
new file mode 100644
index 0000000..bf6bf3c
--- /dev/null
+++ b/uploader/population/__init__.py
@@ -0,0 +1,3 @@
+"""Package to handle creation and management of Populations/InbredSets"""
+
+from .views import popbp
diff --git a/uploader/population/models.py b/uploader/population/models.py
new file mode 100644
index 0000000..d78a821
--- /dev/null
+++ b/uploader/population/models.py
@@ -0,0 +1,87 @@
+"""Functions for accessing the database relating to species populations."""
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+def population_by_id(conn: mdb.Connection, population_id) -> dict:
+ """Get the grouping/population by id."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM InbredSet WHERE InbredSetId=%s",
+ (population_id,))
+ return cursor.fetchone()
+
+def population_by_species_and_id(
+ conn: mdb.Connection, species_id, population_id) -> dict:
+ """Retrieve a population by its identifier and species."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM InbredSet WHERE SpeciesId=%s AND Id=%s",
+ (species_id, population_id))
+ return cursor.fetchone()
+
+def populations_by_species(conn: mdb.Connection, speciesid) -> tuple:
+ "Retrieve group (InbredSet) information from the database."
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ query = "SELECT * FROM InbredSet WHERE SpeciesId=%s"
+ cursor.execute(query, (speciesid,))
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+
+def population_families(conn) -> tuple:
+ """Fetch the families under which populations are grouped."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT DISTINCT(Family) FROM InbredSet WHERE Family IS NOT NULL")
+ return tuple(row["Family"] for row in cursor.fetchall())
+
+
+def population_genetic_types(conn) -> tuple:
+ """Fetch the families under which populations are grouped."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT DISTINCT(GeneticType) FROM InbredSet WHERE GeneticType IS "
+ "NOT NULL")
+ return tuple(row["GeneticType"] for row in cursor.fetchall())
+
+
+def save_population(cursor: mdb.cursors.Cursor, population_details: dict) -> dict:
+ """Save the population details to the db."""
+ cursor.execute("SELECT DISTINCT(Family), FamilyOrder FROM InbredSet "
+ "WHERE Family IS NOT NULL AND Family != '' "
+ "AND FamilyOrder IS NOT NULL "
+ "ORDER BY FamilyOrder ASC")
+ _families = {
+ row["Family"]: int(row["FamilyOrder"])
+ for row in cursor.fetchall()
+ }
+ params = {
+ "MenuOrderId": 0,
+ "InbredSetId": 0,
+ "public": 2,
+ **population_details,
+ "FamilyOrder": _families.get(
+ population_details["Family"],
+ max((0,) + tuple(_families.values()))+1)
+ }
+ cursor.execute(
+ "INSERT INTO InbredSet("
+ "InbredSetId, InbredSetName, Name, SpeciesId, FullName, "
+ "public, MappingMethodId, GeneticType, Family, FamilyOrder,"
+ " MenuOrderId, InbredSetCode, Description"
+ ") "
+ "VALUES ("
+ "%(InbredSetId)s, %(InbredSetName)s, %(Name)s, %(SpeciesId)s, "
+ "%(FullName)s, %(public)s, %(MappingMethodId)s, %(GeneticType)s, "
+ "%(Family)s, %(FamilyOrder)s, %(MenuOrderId)s, %(InbredSetCode)s, "
+ "%(Description)s"
+ ")",
+ params)
+ new_id = cursor.lastrowid
+ cursor.execute("UPDATE InbredSet SET InbredSetId=%s WHERE Id=%s",
+ (new_id, new_id))
+ return {
+ **params,
+ "Id": new_id,
+ "InbredSetId": new_id,
+ "population_id": new_id
+ }
diff --git a/uploader/population/rqtl2.py b/uploader/population/rqtl2.py
new file mode 100644
index 0000000..044cdd4
--- /dev/null
+++ b/uploader/population/rqtl2.py
@@ -0,0 +1,953 @@
+"""Module to handle uploading of R/qtl2 bundles."""#pylint: disable=[too-many-lines]
+import sys
+import json
+import traceback
+from pathlib import Path
+from uuid import UUID, uuid4
+from functools import partial
+from zipfile import ZipFile, is_zipfile
+from typing import Union, Callable, Optional
+
+import MySQLdb as mdb
+from redis import Redis
+from MySQLdb.cursors import DictCursor
+from gn_libs.mysqldb import database_connection
+from flask import (
+ flash,
+ escape,
+ request,
+ url_for,
+ redirect,
+ Response,
+ Blueprint,
+ render_template,
+ current_app as app)
+
+from r_qtl import r_qtl2
+
+from uploader import jobs
+from uploader.files import save_file, fullpath
+from uploader.species.models import all_species
+from uploader.db_utils import with_db_connection
+
+from uploader.authorisation import require_login
+from uploader.platforms.models import platform_by_id, platforms_by_species
+from uploader.db.averaging import averaging_methods, averaging_method_by_id
+from uploader.db.tissues import all_tissues, tissue_by_id, create_new_tissue
+from uploader.population.models import (populations_by_species,
+ population_by_species_and_id)
+from uploader.species.models import species_by_id
+from uploader.db.datasets import (
+ geno_dataset_by_id,
+ geno_datasets_by_species_and_population,
+
+ probeset_study_by_id,
+ probeset_create_study,
+ probeset_dataset_by_id,
+ probeset_create_dataset,
+ probeset_datasets_by_study,
+ probeset_studies_by_species_and_population)
+
+rqtl2 = Blueprint("rqtl2", __name__)
+
+
+@rqtl2.route("/", methods=["GET", "POST"])
+@rqtl2.route("/select-species", methods=["GET", "POST"])
+@require_login
+def select_species():
+ """Select the species."""
+ if request.method == "GET":
+ return render_template("expression-data/rqtl2/index.html",
+ species=with_db_connection(all_species))
+
+ species_id = request.form.get("species_id")
+ species = with_db_connection(
+ lambda conn: species_by_id(conn, species_id))
+ if bool(species):
+ return redirect(url_for(
+ "species.populations.expression-data.rqtl2.select_population",
+ species_id=species_id))
+ flash("Invalid species or no species selected!", "alert-error error-rqtl2")
+ return redirect(url_for("expression-data.rqtl2.select_species"))
+
+
+@rqtl2.route("<int:species_id>/expression-data/rqtl2/select-population",
+ methods=["GET", "POST"])
+@require_login
+def select_population(species_id: int):
+ """Select/Create the population to organise data under."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ if not bool(species):
+ flash("Invalid species selected!", "alert-error error-rqtl2")
+ return redirect(url_for("expression-data.rqtl2.select_species"))
+
+ if request.method == "GET":
+ return render_template(
+ "expression-data/rqtl2/select-population.html",
+ species=species,
+ populations=populations_by_species(conn, species_id))
+
+ population = population_by_species_and_id(
+ conn, species["SpeciesId"], request.form.get("inbredset_id"))
+ if not bool(population):
+ flash("Invalid Population!", "alert-error error-rqtl2")
+ return redirect(
+ url_for("expression-data.rqtl2.select_population", pgsrc="error"),
+ code=307)
+
+ return redirect(url_for("expression-data.rqtl2.upload_rqtl2_bundle",
+ species_id=species["SpeciesId"],
+ population_id=population["InbredSetId"]))
+
+
+class __RequestError__(Exception): #pylint: disable=[invalid-name]
+ """Internal class to avoid pylint's `too-many-return-statements` error."""
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle"),
+ methods=["GET", "POST"])
+@require_login
+def upload_rqtl2_bundle(species_id: int, population_id: int):
+ """Allow upload of R/qtl2 bundle."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ population = population_by_species_and_id(
+ conn, species["SpeciesId"], population_id)
+ if not bool(species):
+ flash("Invalid species!", "alert-error error-rqtl2")
+ return redirect(url_for("expression-data.rqtl2.select_species"))
+ if not bool(population):
+ flash("Invalid Population!", "alert-error error-rqtl2")
+ return redirect(
+ url_for("expression-data.rqtl2.select_population", pgsrc="error"),
+ code=307)
+ if request.method == "GET" or (
+ request.method == "POST"
+ and bool(request.args.get("pgsrc"))):
+ return render_template(
+ "expression-data/rqtl2/upload-rqtl2-bundle-step-01.html",
+ species=species,
+ population=population)
+
+ try:
+ app.logger.debug("Files in the form: %s", request.files)
+ the_file = save_file(request.files["rqtl2_bundle_file"],
+ Path(app.config["UPLOAD_FOLDER"]))
+ except AssertionError:
+ app.logger.debug(traceback.format_exc())
+ flash("Please provide a valid R/qtl2 zip bundle.",
+ "alert-error error-rqtl2")
+ return redirect(url_for("expression-data.rqtl2.upload_rqtl2_bundle",
+ species_id=species_id,
+ population_id=population_id))
+
+ if not is_zipfile(str(the_file)):
+ app.logger.debug("The file is not a zip file.")
+ raise __RequestError__("Invalid file! Expected a zip file.")
+
+ jobid = trigger_rqtl2_bundle_qc(
+ species_id,
+ population_id,
+ the_file,
+ request.files["rqtl2_bundle_file"].filename)#type: ignore[arg-type]
+ return redirect(url_for(
+ "expression-data.rqtl2.rqtl2_bundle_qc_status", jobid=jobid))
+
+
+def trigger_rqtl2_bundle_qc(
+ species_id: int,
+ population_id: int,
+ rqtl2bundle: Path,
+ originalfilename: str
+) -> UUID:
+ """Trigger QC on the R/qtl2 bundle."""
+ redisuri = app.config["REDIS_URL"]
+ with Redis.from_url(redisuri, decode_responses=True) as rconn:
+ jobid = uuid4()
+ redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"]
+ jobs.launch_job(
+ jobs.initialise_job(
+ rconn,
+ jobs.jobsnamespace(),
+ str(jobid),
+ [sys.executable, "-m", "scripts.qc_on_rqtl2_bundle",
+ app.config["SQL_URI"], app.config["REDIS_URL"],
+ jobs.jobsnamespace(), str(jobid), str(species_id),
+ str(population_id), "--redisexpiry",
+ str(redis_ttl_seconds)],
+ "rqtl2-bundle-qc-job",
+ redis_ttl_seconds,
+ {"job-metadata": json.dumps({
+ "speciesid": species_id,
+ "populationid": population_id,
+ "rqtl2-bundle-file": str(rqtl2bundle.absolute()),
+ "original-filename": originalfilename})}),
+ redisuri,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors")
+ return jobid
+
+
+@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>",
+ methods=["GET", "POST"])
+@require_login
+def rqtl2_bundle_qc_status(jobid: UUID):
+ """Check the status of the QC jobs."""
+ with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn,
+ database_connection(app.config["SQL_URI"]) as dbconn):
+ try:
+ thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid)
+ messagelistname = thejob.get("log-messagelist")
+ logmessages = (rconn.lrange(messagelistname, 0, -1)
+ if bool(messagelistname) else [])
+ jobstatus = thejob["status"]
+ if jobstatus == "error":
+ return render_template(
+ "expression-data/rqtl2/rqtl2-qc-job-error.html",
+ job=thejob,
+ errorsgeneric=json.loads(
+ thejob.get("errors-generic", "[]")),
+ errorsgeno=json.loads(
+ thejob.get("errors-geno", "[]")),
+ errorspheno=json.loads(
+ thejob.get("errors-pheno", "[]")),
+ errorsphenose=json.loads(
+ thejob.get("errors-phenose", "[]")),
+ errorsphenocovar=json.loads(
+ thejob.get("errors-phenocovar", "[]")),
+ messages=logmessages)
+ if jobstatus == "success":
+ jobmeta = json.loads(thejob["job-metadata"])
+ species = species_by_id(dbconn, jobmeta["speciesid"])
+ return render_template(
+ "expression-data/rqtl2/rqtl2-qc-job-results.html",
+ species=species,
+ population=population_by_species_and_id(
+ dbconn, species["SpeciesId"], jobmeta["populationid"]),
+ rqtl2bundle=Path(jobmeta["rqtl2-bundle-file"]).name,
+ rqtl2bundleorig=jobmeta["original-filename"])
+
+ def compute_percentage(thejob, filetype) -> Union[str, None]:
+ if f"{filetype}-linecount" in thejob:
+ return "100"
+ if f"{filetype}-filesize" in thejob:
+ percent = ((int(thejob.get(f"{filetype}-checked", 0))
+ /
+ int(thejob.get(f"{filetype}-filesize", 1)))
+ * 100)
+ return f"{percent:.2f}"
+ return None
+
+ return render_template(
+ "expression-data/rqtl2/rqtl2-qc-job-status.html",
+ job=thejob,
+ geno_percent=compute_percentage(thejob, "geno"),
+ pheno_percent=compute_percentage(thejob, "pheno"),
+ phenose_percent=compute_percentage(thejob, "phenose"),
+ messages=logmessages)
+ except jobs.JobNotFound:
+ return render_template("expression-data/rqtl2/no-such-job.html", jobid=jobid)
+
+
+def redirect_on_error(flaskroute, **kwargs):
+ """Utility to redirect on error"""
+ return redirect(url_for(flaskroute, **kwargs, pgsrc="error"),
+ code=(307 if request.method == "POST" else 302))
+
+
+def check_species(conn: mdb.Connection, formargs: dict) -> Optional[
+ tuple[str, Response]]:
+ """
+ Check whether the 'species_id' value is provided, and whether a
+ corresponding species exists in the database.
+
+ Maybe give the function a better name..."""
+ speciespage = redirect_on_error("expression-data.rqtl2.select_species")
+ if "species_id" not in formargs:
+ return "You MUST provide the Species identifier.", speciespage
+
+ if not bool(species_by_id(conn, formargs["species_id"])):
+ return "No species with the provided identifier exists.", speciespage
+
+ return None
+
+
+def check_population(conn: mdb.Connection,
+ formargs: dict,
+ species_id) -> Optional[tuple[str, Response]]:
+ """
+ Check whether the 'population_id' value is provided, and whether a
+ corresponding population exists in the database.
+
+ Maybe give the function a better name..."""
+ poppage = redirect_on_error(
+ "expression-data.rqtl2.select_species", species_id=species_id)
+ if "population_id" not in formargs:
+ return "You MUST provide the Population identifier.", poppage
+
+ if not bool(population_by_species_and_id(
+ conn, species_id, formargs["population_id"])):
+ return "No population with the provided identifier exists.", poppage
+
+ return None
+
+
+def check_r_qtl2_bundle(formargs: dict,
+ species_id,
+ population_id) -> Optional[tuple[str, Response]]:
+ """Check for the existence of the R/qtl2 bundle."""
+ fileuploadpage = redirect_on_error("expression-data.rqtl2.upload_rqtl2_bundle",
+ species_id=species_id,
+ population_id=population_id)
+ if not "rqtl2_bundle_file" in formargs:
+ return (
+ "You MUST provide a R/qtl2 zip bundle for expression-data.", fileuploadpage)
+
+ if not Path(fullpath(formargs["rqtl2_bundle_file"])).exists():
+ return "No R/qtl2 bundle with the given name exists.", fileuploadpage
+
+ return None
+
+
+def check_geno_dataset(conn: mdb.Connection,
+ formargs: dict,
+ species_id,
+ population_id) -> Optional[tuple[str, Response]]:
+ """Check for the Genotype dataset."""
+ genodsetpg = redirect_on_error("expression-data.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id)
+ if not bool(formargs.get("geno-dataset-id")):
+ return (
+ "You MUST provide a valid Genotype dataset identifier", genodsetpg)
+
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s",
+ (formargs["geno-dataset-id"],))
+ results = cursor.fetchall()
+ if not bool(results):
+ return ("No genotype dataset with the provided identifier exists.",
+ genodsetpg)
+ if len(results) > 1:
+ return (
+ "Data corruption: More than one genotype dataset with the same "
+ "identifier.",
+ genodsetpg)
+
+ return None
+
+def check_tissue(
+ conn: mdb.Connection,formargs: dict) -> Optional[tuple[str, Response]]:
+ """Check for tissue/organ/biological material."""
+ selectdsetpg = redirect_on_error("expression-data.rqtl2.select_dataset_info",
+ species_id=formargs["species_id"],
+ population_id=formargs["population_id"])
+ if not bool(formargs.get("tissueid", "").strip()):
+ return ("No tissue/organ/biological material provided.", selectdsetpg)
+
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM Tissue WHERE Id=%s",
+ (formargs["tissueid"],))
+ results = cursor.fetchall()
+ if not bool(results):
+ return ("No tissue/organ with the provided identifier exists.",
+ selectdsetpg)
+
+ if len(results) > 1:
+ return (
+ "Data corruption: More than one tissue/organ with the same "
+ "identifier.",
+ selectdsetpg)
+
+ return None
+
+
+def check_probe_study(conn: mdb.Connection,
+ formargs: dict,
+ species_id,
+ population_id) -> Optional[tuple[str, Response]]:
+ """Check for the ProbeSet study."""
+ dsetinfopg = redirect_on_error("expression-data.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id)
+ if not bool(formargs.get("probe-study-id")):
+ return "No probeset study was selected!", dsetinfopg
+
+ if not bool(probeset_study_by_id(conn, formargs["probe-study-id"])):
+ return ("No probeset study with the provided identifier exists",
+ dsetinfopg)
+
+ return None
+
+
+def check_probe_dataset(conn: mdb.Connection,
+ formargs: dict,
+ species_id,
+ population_id) -> Optional[tuple[str, Response]]:
+ """Check for the ProbeSet dataset."""
+ dsetinfopg = redirect_on_error("expression-data.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id)
+ if not bool(formargs.get("probe-dataset-id")):
+ return "No probeset dataset was selected!", dsetinfopg
+
+ if not bool(probeset_dataset_by_id(conn, formargs["probe-dataset-id"])):
+ return ("No probeset dataset with the provided identifier exists",
+ dsetinfopg)
+
+ return None
+
+
+def with_errors(endpointthunk: Callable, *checkfns):
+ """Run 'endpointthunk' with error checking."""
+ formargs = {**dict(request.args), **dict(request.form)}
+ errors = tuple(item for item in (_fn(formargs=formargs) for _fn in checkfns)
+ if item is not None)
+ if len(errors) > 0:
+ flash(errors[0][0], "alert-error error-rqtl2")
+ return errors[0][1]
+
+ return endpointthunk()
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/select-geno-dataset"),
+ methods=["POST"])
+@require_login
+def select_geno_dataset(species_id: int, population_id: int):
+ """Select from existing geno datasets."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ geno_dset = geno_datasets_by_species_and_population(
+ conn, species_id, population_id)
+ if not bool(geno_dset):
+ flash("No genotype dataset was provided!",
+ "alert-error error-rqtl2")
+ return redirect(url_for("expression-data.rqtl2.select_geno_dataset",
+ species_id=species_id,
+ population_id=population_id,
+ pgsrc="error"),
+ code=307)
+
+ flash("Genotype accepted", "alert-success error-rqtl2")
+ return redirect(url_for("expression-data.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id,
+ pgsrc="expression-data.rqtl2.select_geno_dataset"),
+ code=307)
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population, conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/select-tissue"),
+ methods=["POST"])
+@require_login
+def select_tissue(species_id: int, population_id: int):
+ """Select from existing tissues."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ if not bool(request.form.get("tissueid", "").strip()):
+ flash("Invalid tissue selection!",
+ "alert-error error-select-tissue error-rqtl2")
+
+ return redirect(url_for("expression-data.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id,
+ pgsrc="expression-data.rqtl2.select_geno_dataset"),
+ code=307)
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/create-tissue"),
+ methods=["POST"])
+@require_login
+def create_tissue(species_id: int, population_id: int):
+ """Add new tissue, organ or biological material to the system."""
+ form = request.form
+ datasetinfopage = redirect(
+ url_for("expression-data.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id,
+ pgsrc="expression-data.rqtl2.select_geno_dataset"),
+ code=307)
+ with database_connection(app.config["SQL_URI"]) as conn:
+ tissuename = form.get("tissuename", "").strip()
+ tissueshortname = form.get("tissueshortname", "").strip()
+ if not bool(tissuename):
+ flash("Organ/Tissue name MUST be provided.",
+ "alert-error error-create-tissue error-rqtl2")
+ return datasetinfopage
+
+ if not bool(tissueshortname):
+ flash("Organ/Tissue short name MUST be provided.",
+ "alert-error error-create-tissue error-rqtl2")
+ return datasetinfopage
+
+ try:
+ tissue = create_new_tissue(conn, tissuename, tissueshortname)
+ flash("Tissue created successfully!", "alert-success")
+ return render_template(
+ "expression-data/rqtl2/create-tissue-success.html",
+ species=species_by_id(conn, species_id),
+ population=population_by_species_and_id(
+ conn, species_id, population_id),
+ rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
+ geno_dataset=geno_dataset_by_id(
+ conn,
+ int(request.form["geno-dataset-id"])),
+ tissue=tissue)
+ except mdb.IntegrityError as _ierr:
+ flash("Tissue/Organ with that short name already exists!",
+ "alert-error error-create-tissue error-rqtl2")
+ return datasetinfopage
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/select-probeset-study"),
+ methods=["POST"])
+@require_login
+def select_probeset_study(species_id: int, population_id: int):
+ """Select or create a probeset study."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ summary_page = redirect(url_for("expression-data.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id),
+ code=307)
+ if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))):
+ flash("Invalid study selected!", "alert-error error-rqtl2")
+ return summary_page
+
+ return summary_page
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_tissue, conn=conn),
+ partial(check_probe_study,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/select-probeset-dataset"),
+ methods=["POST"])
+@require_login
+def select_probeset_dataset(species_id: int, population_id: int):
+ """Select or create a probeset dataset."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ summary_page = redirect(url_for("expression-data.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id),
+ code=307)
+ if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))):
+ flash("Invalid study selected!", "alert-error error-rqtl2")
+ return summary_page
+
+ return summary_page
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_tissue, conn=conn),
+ partial(check_probe_study,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_probe_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/create-probeset-study"),
+ methods=["POST"])
+@require_login
+def create_probeset_study(species_id: int, population_id: int):
+ """Create a new probeset study."""
+ errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-study"
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ form = request.form
+ dataset_info_page = redirect(
+ url_for("expression-data.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id),
+ code=307)
+
+ if not (bool(form.get("platformid")) and
+ bool(platform_by_id(conn, int(form["platformid"])))):
+ flash("Invalid platform selected.", errorclasses)
+ return dataset_info_page
+
+ if not (bool(form.get("tissueid")) and
+ bool(tissue_by_id(conn, int(form["tissueid"])))):
+ flash("Invalid tissue selected.", errorclasses)
+ return dataset_info_page
+
+ studyname = form["studyname"]
+ try:
+ study = probeset_create_study(
+ conn, population_id, int(form["platformid"]), int(form["tissueid"]),
+ studyname, form.get("studyfullname") or "",
+ form.get("studyshortname") or "")
+ except mdb.IntegrityError as _ierr:
+ flash(f"ProbeSet study with name '{escape(studyname)}' already "
+ "exists.",
+ errorclasses)
+ return dataset_info_page
+ return render_template(
+ "expression-data/rqtl2/create-probe-study-success.html",
+ species=species_by_id(conn, species_id),
+ population=population_by_species_and_id(
+ conn, species_id, population_id),
+ rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
+ geno_dataset=geno_dataset_by_id(
+ conn,
+ int(request.form["geno-dataset-id"])),
+ study=study)
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_tissue, conn=conn))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/create-probeset-dataset"),
+ methods=["POST"])
+@require_login
+def create_probeset_dataset(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements]
+ """Create a new probeset dataset."""
+ errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-dataset"
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():#pylint: disable=[too-many-return-statements]
+ form = request.form
+ summary_page = redirect(url_for("expression-data.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id),
+ code=307)
+ if not bool(form.get("averageid")):
+ flash("Averaging method not selected!", errorclasses)
+ return summary_page
+ if not bool(form.get("datasetname")):
+ flash("Dataset name not provided!", errorclasses)
+ return summary_page
+ if not bool(form.get("datasetfullname")):
+ flash("Dataset full name not provided!", errorclasses)
+ return summary_page
+
+ tissue = tissue_by_id(conn, form.get("tissueid", "").strip())
+
+ study = probeset_study_by_id(conn, int(form["probe-study-id"]))
+ if not bool(study):
+ flash("Invalid ProbeSet study provided!", errorclasses)
+ return summary_page
+
+ avgmethod = averaging_method_by_id(conn, int(form["averageid"]))
+ if not bool(avgmethod):
+ flash("Invalid averaging method provided!", errorclasses)
+ return summary_page
+
+ try:
+ dset = probeset_create_dataset(conn,
+ int(form["probe-study-id"]),
+ int(form["averageid"]),
+ form["datasetname"],
+ form["datasetfullname"],
+ form["datasetshortname"],
+ form["datasetpublic"] == "on",
+ form.get(
+ "datasetdatascale", "log2"))
+ except mdb.IntegrityError as _ierr:
+ app.logger.debug("Possible integrity error: %s", traceback.format_exc())
+ flash(("IntegrityError: The data you provided has some errors: "
+ f"{_ierr.args}"),
+ errorclasses)
+ return summary_page
+ except Exception as _exc:# pylint: disable=[broad-except]
+ app.logger.debug("Error creating ProbeSet dataset: %s",
+ traceback.format_exc())
+ flash(("There was a problem creating your dataset. Please try "
+ "again."),
+ errorclasses)
+ return summary_page
+ return render_template(
+ "expression-data/rqtl2/create-probe-dataset-success.html",
+ species=species_by_id(conn, species_id),
+ population=population_by_species_and_id(
+ conn, species_id, population_id),
+ rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
+ geno_dataset=geno_dataset_by_id(
+ conn,
+ int(request.form["geno-dataset-id"])),
+ tissue=tissue,
+ study=study,
+ avgmethod=avgmethod,
+ dataset=dset)
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_tissue, conn=conn),
+ partial(check_probe_study,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/dataset-info"),
+ methods=["POST"])
+@require_login
+def select_dataset_info(species_id: int, population_id: int):
+ """
+ If `geno` files exist in the R/qtl2 bundle, prompt user to provide the
+ dataset the genotypes belong to.
+ """
+ form = request.form
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ species = species_by_id(conn, species_id)
+ population = population_by_species_and_id(
+ conn, species_id, population_id)
+ thefile = fullpath(form["rqtl2_bundle_file"])
+ with ZipFile(str(thefile), "r") as zfile:
+ cdata = r_qtl2.control_data(zfile)
+
+ geno_dataset = geno_dataset_by_id(
+ conn,form.get("geno-dataset-id", "").strip())
+ if "geno" in cdata and not bool(form.get("geno-dataset-id")):
+ return render_template(
+ "expression-data/rqtl2/select-geno-dataset.html",
+ species=species,
+ population=population,
+ rqtl2_bundle_file=thefile.name,
+ datasets=geno_datasets_by_species_and_population(
+ conn, species_id, population_id))
+
+ tissue = tissue_by_id(conn, form.get("tissueid", "").strip())
+ if "pheno" in cdata and not bool(tissue):
+ return render_template(
+ "expression-data/rqtl2/select-tissue.html",
+ species=species,
+ population=population,
+ rqtl2_bundle_file=thefile.name,
+ geno_dataset=geno_dataset,
+ studies=probeset_studies_by_species_and_population(
+ conn, species_id, population_id),
+ platforms=platforms_by_species(conn, species_id),
+ tissues=all_tissues(conn))
+
+ probeset_study = probeset_study_by_id(
+ conn, form.get("probe-study-id", "").strip())
+ if "pheno" in cdata and not bool(probeset_study):
+ return render_template(
+ "expression-data/rqtl2/select-probeset-study-id.html",
+ species=species,
+ population=population,
+ rqtl2_bundle_file=thefile.name,
+ geno_dataset=geno_dataset,
+ studies=probeset_studies_by_species_and_population(
+ conn, species_id, population_id),
+ platforms=platforms_by_species(conn, species_id),
+ tissue=tissue)
+ probeset_study = probeset_study_by_id(
+ conn, int(form["probe-study-id"]))
+
+ probeset_dataset = probeset_dataset_by_id(
+ conn, form.get("probe-dataset-id", "").strip())
+ if "pheno" in cdata and not bool(probeset_dataset):
+ return render_template(
+ "expression-data/rqtl2/select-probeset-dataset.html",
+ species=species,
+ population=population,
+ rqtl2_bundle_file=thefile.name,
+ geno_dataset=geno_dataset,
+ probe_study=probeset_study,
+ tissue=tissue,
+ datasets=probeset_datasets_by_study(
+ conn, int(form["probe-study-id"])),
+ avgmethods=averaging_methods(conn))
+
+ return render_template("expression-data/rqtl2/summary-info.html",
+ species=species,
+ population=population,
+ rqtl2_bundle_file=thefile.name,
+ geno_dataset=geno_dataset,
+ tissue=tissue,
+ probe_study=probeset_study,
+ probe_dataset=probeset_dataset)
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/confirm-bundle-details"),
+ methods=["POST"])
+@require_login
+def confirm_bundle_details(species_id: int, population_id: int):
+ """Confirm the details and trigger R/qtl2 bundle processing..."""
+ redisuri = app.config["REDIS_URL"]
+ with (database_connection(app.config["SQL_URI"]) as conn,
+ Redis.from_url(redisuri, decode_responses=True) as rconn):
+ def __thunk__():
+ redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"]
+ jobid = str(uuid4())
+ _job = jobs.launch_job(
+ jobs.initialise_job(
+ rconn,
+ jobs.jobsnamespace(),
+ jobid,
+ [
+ sys.executable, "-m", "scripts.process_rqtl2_bundle",
+ app.config["SQL_URI"], app.config["REDIS_URL"],
+ jobs.jobsnamespace(), jobid, "--redisexpiry",
+ str(redis_ttl_seconds)],
+ "R/qtl2 Bundle Upload",
+ redis_ttl_seconds,
+ {
+ "bundle-metadata": json.dumps({
+ "speciesid": species_id,
+ "populationid": population_id,
+ "rqtl2-bundle-file": str(fullpath(
+ request.form["rqtl2_bundle_file"])),
+ "geno-dataset-id": request.form.get(
+ "geno-dataset-id", ""),
+ "probe-study-id": request.form.get(
+ "probe-study-id", ""),
+ "probe-dataset-id": request.form.get(
+ "probe-dataset-id", ""),
+ **({
+ "platformid": probeset_study_by_id(
+ conn,
+ int(request.form["probe-study-id"]))["ChipId"]
+ } if bool(request.form.get("probe-study-id")) else {})
+ })
+ }),
+ redisuri,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+ return redirect(url_for("expression-data.rqtl2.rqtl2_processing_status",
+ jobid=jobid))
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_probe_study,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_probe_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route("/status/<uuid:jobid>")
+def rqtl2_processing_status(jobid: UUID):
+ """Retrieve the status of the job processing the uploaded R/qtl2 bundle."""
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ try:
+ thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid)
+
+ messagelistname = thejob.get("log-messagelist")
+ logmessages = (rconn.lrange(messagelistname, 0, -1)
+ if bool(messagelistname) else [])
+
+ if thejob["status"] == "error":
+ return render_template(
+ "expression-data/rqtl2/rqtl2-job-error.html",
+ job=thejob,
+ messages=logmessages)
+ if thejob["status"] == "success":
+ return render_template(
+ "expression-data/rqtl2/rqtl2-job-results.html",
+ job=thejob,
+ messages=logmessages)
+
+ return render_template(
+ "expression-data/rqtl2/rqtl2-job-status.html",
+ job=thejob,
+ messages=logmessages)
+ except jobs.JobNotFound as _exc:
+ return render_template("expression-data/rqtl2/no-such-job.html",
+ jobid=jobid)
diff --git a/uploader/population/views.py b/uploader/population/views.py
new file mode 100644
index 0000000..f42e547
--- /dev/null
+++ b/uploader/population/views.py
@@ -0,0 +1,217 @@
+"""Views dealing with populations/inbredsets"""
+import json
+import base64
+
+from MySQLdb.cursors import DictCursor
+from gn_libs.mysqldb import database_connection
+from flask import (flash,
+ escape,
+ request,
+ url_for,
+ redirect,
+ Blueprint,
+ current_app as app)
+
+from uploader.samples.views import samplesbp
+from uploader.oauth2.client import oauth2_post
+from uploader.ui import make_template_renderer
+from uploader.authorisation import require_login
+from uploader.genotypes.views import genotypesbp
+from uploader.datautils import enumerate_sequence
+from uploader.phenotypes.views import phenotypesbp
+from uploader.expression_data.views import exprdatabp
+from uploader.monadic_requests import make_either_error_handler
+from uploader.input_validation import is_valid_representative_name
+from uploader.species.models import (all_species,
+ species_by_id,
+ order_species_by_family)
+
+from .models import (save_population,
+ population_families,
+ populations_by_species,
+ population_genetic_types,
+ population_by_species_and_id)
+
+__active_link__ = "populations"
+popbp = Blueprint("populations", __name__)
+popbp.register_blueprint(samplesbp, url_prefix="/")
+popbp.register_blueprint(genotypesbp, url_prefix="/")
+popbp.register_blueprint(phenotypesbp, url_prefix="/")
+popbp.register_blueprint(exprdatabp, url_prefix="/")
+render_template = make_template_renderer("populations")
+
+
+@popbp.route("/populations", methods=["GET", "POST"])
+@require_login
+def index():
+ """Entry point for populations."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ if not bool(request.args.get("species_id")):
+ return render_template(
+ "populations/index.html",
+ species=all_species(conn),
+ activelink="populations")
+
+ species_id = request.args.get("species_id")
+ if species_id == "CREATE-SPECIES":
+ return redirect(url_for(
+ "species.create_species",
+ return_to="species.populations.list_species_populations"))
+
+ species = species_by_id(conn, request.args.get("species_id"))
+ if not bool(species):
+ flash("Invalid species identifier provided!", "alert-danger")
+ return redirect(url_for("species.populations.index"))
+ return redirect(url_for("species.populations.list_species_populations",
+ species_id=species["SpeciesId"]))
+
+@popbp.route("/<int:species_id>/populations", methods=["GET"])
+@require_login
+def list_species_populations(species_id: int):
+ """List a particular species' populations."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ if not bool(species):
+ flash("No species was found for given ID.", "alert-danger")
+ return redirect(url_for("species.populations.index"))
+ return render_template(
+ "populations/list-populations.html",
+ species=species,
+ populations=enumerate_sequence(populations_by_species(
+ conn, species_id)),
+ activelink="list-populations")
+
+
+@popbp.route("/<int:species_id>/populations/create", methods=["GET", "POST"])
+@require_login
+def create_population(species_id: int):
+ """Create a new population."""
+ with (database_connection(app.config["SQL_URI"]) as conn,
+ conn.cursor(cursorclass=DictCursor) as cursor):
+ species = species_by_id(conn, species_id)
+
+ if request.method == "GET":
+ error_values = request.args.get("error_values")
+ if not bool(error_values):
+ error_values = base64.b64encode(
+ '{"errors":{}, "error_values": {}}'.encode("utf8")
+ ).decode("utf8")
+
+ error_values = json.loads(base64.b64decode(
+ error_values.encode("utf8")).decode("utf8"))# type: ignore[union-attr]
+ return render_template(
+ "populations/create-population.html",
+ species=species,
+ families = population_families(conn),
+ genetic_types = population_genetic_types(conn),
+ mapping_methods=(
+ {"id": "0", "value": "No mapping support"},
+ {"id": "1", "value": "GEMMA, QTLReaper, R/qtl"},
+ {"id": "2", "value": "GEMMA"},
+ {"id": "3", "value": "R/qtl"},
+ {"id": "4", "value": "GEMMA, PLINK"}),
+ return_to=(request.args.get("return_to") or ""),
+ activelink="create-population",
+ **error_values)
+
+ if not bool(species):
+ flash("You must select a species.", "alert-danger")
+ return redirect(url_for("species.populations.index"))
+
+ errors: tuple[tuple[str, str], ...] = tuple()
+
+ population_name = (request.form.get(
+ "population_name") or "").strip()
+ if not bool(population_name):
+ errors = errors + (("population_name",
+ "You must provide a name for the population!"),)
+
+ if not is_valid_representative_name(population_name):
+ errors = errors + ((
+ "population_name",
+ "The population name can only contain letters, numbers, "
+ "hyphens and underscores."),)
+
+ population_fullname = (request.form.get(
+ "population_fullname") or "").strip()
+ if not bool(population_fullname):
+ errors = errors + (
+ ("population_fullname", "Full Name MUST be provided."),)
+
+ if bool(errors):
+ values = base64.b64encode(
+ json.dumps({
+ "errors": dict(errors),
+ "error_values": dict(request.form)
+ }).encode("utf8"))
+ return redirect(url_for("species.populations.create_population",
+ species_id=species["SpeciesId"],
+ error_values=values))
+
+ new_population = save_population(cursor, {
+ "SpeciesId": species["SpeciesId"],
+ "Name": population_name,
+ "InbredSetName": population_fullname,
+ "FullName": population_fullname,
+ "InbredSetCode": request.form.get("population_code") or None,
+ "Description": request.form.get("population_description") or None,
+ "Family": request.form.get("population_family") or None,
+ "MappingMethodId": request.form.get("population_mapping_method_id"),
+ "GeneticType": request.form.get("population_genetic_type") or None
+ })
+
+ def __flash_success__(_success):
+ flash("Successfully created population "
+ f"{escape(new_population['FullName'])}.",
+ "alert-success")
+ return_to = request.form.get("return_to") or ""
+ if return_to:
+ return redirect(url_for(
+ return_to,
+ species_id=species["SpeciesId"],
+ population_id=new_population["InbredSetId"]))
+ return redirect(url_for(
+ "species.populations.view_population",
+ species_id=species["SpeciesId"],
+ population_id=new_population["InbredSetId"]))
+
+ app.logger.debug("We begin setting up the privileges here…")
+ return oauth2_post(
+ "auth/resource/populations/create",
+ json={
+ **dict(request.form),
+ "species_id": species_id,
+ "population_id": new_population["Id"],
+ "public": "on"
+ }
+ ).either(
+ make_either_error_handler(
+ "There was an error creating the population"),
+ __flash_success__)
+
+
+@popbp.route("/<int:species_id>/populations/<int:population_id>",
+ methods=["GET"])
+@require_login
+def view_population(species_id: int, population_id: int):
+ """View the details of a population."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ population = population_by_species_and_id(conn, species_id, population_id)
+ error = False
+
+ if not bool(species):
+ flash("You must select a species.", "alert-danger")
+ error = True
+
+ if not bool(population):
+ flash("You must select a population.", "alert-danger")
+ error = True
+
+ if error:
+ return redirect(url_for("species.populations.index"))
+
+ return render_template("populations/view-population.html",
+ species=species,
+ population=population,
+ activelink="view-population")
diff --git a/uploader/request_checks.py b/uploader/request_checks.py
new file mode 100644
index 0000000..f1d8027
--- /dev/null
+++ b/uploader/request_checks.py
@@ -0,0 +1,75 @@
+"""Functions to perform common checks.
+
+These are useful for reusability, and hence maintainability of the code.
+"""
+from functools import wraps
+
+from gn_libs.mysqldb import database_connection
+from flask import flash, url_for, redirect, current_app as app
+
+from uploader.species.models import species_by_id
+from uploader.population.models import population_by_species_and_id
+
+def with_species(redirect_uri: str):
+ """Ensure the species actually exists."""
+ def __decorator__(function):
+ @wraps(function)
+ def __with_species__(**kwargs):
+ try:
+ species_id = int(kwargs.get("species_id"))
+ if not bool(species_id):
+ flash("Expected species_id value to be present!",
+ "alert-danger")
+ return redirect(url_for(redirect_uri))
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ if not bool(species):
+ flash("Could not find species with that ID",
+ "alert-danger")
+ return redirect(url_for(redirect_uri))
+ except ValueError as _verr:
+ app.logger.debug(
+ "Exception converting value to integer: %s",
+ kwargs.get("species_id"),
+ exc_info=True)
+ flash("Expected an integer for 'species_id' value.",
+ "alert-danger")
+ return redirect(url_for(redirect_uri))
+ return function(**{**kwargs, "species": species})
+ return __with_species__
+ return __decorator__
+
+
+def with_population(species_redirect_uri: str, redirect_uri: str):
+ """Ensure the population actually exists."""
+ def __decorator__(function):
+ @wraps(function)
+ @with_species(redirect_uri=species_redirect_uri)
+ def __with_population__(**kwargs):
+ try:
+ species_id = int(kwargs["species_id"])
+ population_id = int(kwargs.get("population_id"))
+ select_population_uri = redirect(url_for(
+ redirect_uri, species_id=species_id))
+ if not bool(population_id):
+ flash("Expected population_id value to be present!",
+ "alert-danger")
+ return select_population_uri
+ with database_connection(app.config["SQL_URI"]) as conn:
+ population = population_by_species_and_id(
+ conn, species_id, population_id)
+ if not bool(population):
+ flash("Could not find population with that ID",
+ "alert-danger")
+ return select_population_uri
+ except ValueError as _verr:
+ app.logger.debug(
+ "Exception converting value to integer: %s",
+ kwargs.get("population_id"),
+ exc_info=True)
+ flash("Expected an integer for 'population_id' value.",
+ "alert-danger")
+ return select_population_uri
+ return function(**{**kwargs, "population": population})
+ return __with_population__
+ return __decorator__
diff --git a/uploader/samples/__init__.py b/uploader/samples/__init__.py
new file mode 100644
index 0000000..1bd6d2d
--- /dev/null
+++ b/uploader/samples/__init__.py
@@ -0,0 +1 @@
+"""Samples package. Handle samples uploads and editing."""
diff --git a/uploader/samples/models.py b/uploader/samples/models.py
new file mode 100644
index 0000000..d7d5384
--- /dev/null
+++ b/uploader/samples/models.py
@@ -0,0 +1,104 @@
+"""Functions for handling samples."""
+import csv
+from typing import Iterator
+
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+from functional_tools import take
+
+def samples_by_species_and_population(
+ conn: mdb.Connection,
+ species_id: int,
+ population_id: int
+) -> tuple[dict, ...]:
+ """Fetch the samples by their species and population."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT iset.InbredSetId, s.* FROM InbredSet AS iset "
+ "INNER JOIN StrainXRef AS sxr ON iset.InbredSetId=sxr.InbredSetId "
+ "INNER JOIN Strain AS s ON sxr.StrainId=s.Id "
+ "WHERE s.SpeciesId=%(species_id)s "
+ "AND iset.InbredSetId=%(population_id)s",
+ {"species_id": species_id, "population_id": population_id})
+ return tuple(cursor.fetchall())
+
+
+def read_samples_file(filepath, separator: str, firstlineheading: bool, **kwargs) -> Iterator[dict]:
+ """Read the samples file."""
+ with open(filepath, "r", encoding="utf-8") as inputfile:
+ reader = csv.DictReader(
+ inputfile,
+ fieldnames=(
+ None if firstlineheading
+ else ("Name", "Name2", "Symbol", "Alias")),
+ delimiter=separator,
+ quotechar=kwargs.get("quotechar", '"'))
+ for row in reader:
+ yield row
+
+
+def save_samples_data(conn: mdb.Connection,
+ speciesid: int,
+ file_data: Iterator[dict]):
+ """Save the samples to DB."""
+ data = ({**row, "SpeciesId": speciesid} for row in file_data)
+ total = 0
+ with conn.cursor() as cursor:
+ while True:
+ batch = take(data, 5000)
+ if len(batch) == 0:
+ break
+ cursor.executemany(
+ "INSERT INTO Strain(Name, Name2, SpeciesId, Symbol, Alias) "
+ "VALUES("
+ " %(Name)s, %(Name2)s, %(SpeciesId)s, %(Symbol)s, %(Alias)s"
+ ") ON DUPLICATE KEY UPDATE Name=Name",
+ batch)
+ total += len(batch)
+ print(f"\tSaved {total} samples total so far.")
+
+
+def cross_reference_samples(conn: mdb.Connection,
+ species_id: int,
+ population_id: int,
+ strain_names: Iterator[str]):
+ """Link samples to their population."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT MAX(OrderId) AS loid FROM StrainXRef WHERE InbredSetId=%s",
+ (population_id,))
+ last_order_id = (cursor.fetchone()["loid"] or 10)
+ total = 0
+ while True:
+ batch = take(strain_names, 5000)
+ if len(batch) == 0:
+ break
+ params_str = ", ".join(["%s"] * len(batch))
+ ## This query is slow -- investigate.
+ cursor.execute(
+ "SELECT s.Id FROM Strain AS s LEFT JOIN StrainXRef AS sx "
+ "ON s.Id = sx.StrainId WHERE s.SpeciesId=%s AND s.Name IN "
+ f"({params_str}) AND sx.StrainId IS NULL",
+ (species_id,) + tuple(batch))
+ strain_ids = (sid["Id"] for sid in cursor.fetchall())
+ params = tuple({
+ "pop_id": population_id,
+ "strain_id": strain_id,
+ "order_id": last_order_id + (order_id * 10),
+ "mapping": "N",
+ "pedigree": None
+ } for order_id, strain_id in enumerate(strain_ids, start=1))
+ cursor.executemany(
+ "INSERT INTO StrainXRef( "
+ " InbredSetId, StrainId, OrderId, Used_for_mapping, PedigreeStatus"
+ ")"
+ "VALUES ("
+ " %(pop_id)s, %(strain_id)s, %(order_id)s, %(mapping)s, "
+ " %(pedigree)s"
+ ")",
+ params)
+ last_order_id += (len(params) * 10)
+ total += len(batch)
+ print(f"\t{total} total samples cross-referenced to the population "
+ "so far.")
diff --git a/uploader/samples/views.py b/uploader/samples/views.py
new file mode 100644
index 0000000..95a6f8c
--- /dev/null
+++ b/uploader/samples/views.py
@@ -0,0 +1,292 @@
+"""Code regarding samples"""
+import os
+import sys
+import uuid
+from pathlib import Path
+
+from redis import Redis
+from flask import (flash,
+ request,
+ url_for,
+ redirect,
+ Blueprint,
+ current_app as app)
+
+from uploader import jobs
+from uploader.files import save_file
+from uploader.ui import make_template_renderer
+from uploader.authorisation import require_login
+from uploader.request_checks import with_population
+from uploader.input_validation import is_integer_input
+from uploader.datautils import safe_int, order_by_family, enumerate_sequence
+from uploader.population.models import population_by_id, populations_by_species
+from uploader.db_utils import (with_db_connection,
+ database_connection,
+ with_redis_connection)
+from uploader.species.models import (all_species,
+ species_by_id,
+ order_species_by_family)
+
+from .models import samples_by_species_and_population
+
+samplesbp = Blueprint("samples", __name__)
+render_template = make_template_renderer("samples")
+
+@samplesbp.route("/samples", methods=["GET"])
+@require_login
+def index():
+ """Direct entry-point for uploading/handling the samples."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ if not bool(request.args.get("species_id")):
+ return render_template(
+ "samples/index.html",
+ species=all_species(conn),
+ activelink="samples")
+
+ species_id = request.args.get("species_id")
+ if species_id == "CREATE-SPECIES":
+ return redirect(url_for(
+ "species.create_species",
+ return_to="species.populations.samples.select_population"))
+
+ species = species_by_id(conn, request.args.get("species_id"))
+ if not bool(species):
+ flash("No such species!", "alert-danger")
+ return redirect(url_for("species.populations.samples.index"))
+ return redirect(url_for("species.populations.samples.select_population",
+ species_id=species["SpeciesId"]))
+
+
+@samplesbp.route("<int:species_id>/samples/select-population", methods=["GET"])
+@require_login
+def select_population(species_id: int):
+ """Select the population to use for the samples."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ if not bool(species):
+ flash("Invalid species!", "alert-danger")
+ return redirect(url_for("species.populations.samples.index"))
+
+ if not bool(request.args.get("population_id")):
+ return render_template("samples/select-population.html",
+ species=species,
+ populations=populations_by_species(
+ conn,
+ species_id),
+ activelink="samples")
+
+ population_id = request.args["population_id"]
+ if population_id == "CREATE-POPULATION":
+ return redirect(url_for(
+ "species.populations.create_population",
+ species_id=species["SpeciesId"],
+ return_to="species.populations.samples.list_samples"))
+
+ population = population_by_id(conn, request.args.get("population_id"))
+ if not bool(population):
+ flash("Population not found!", "alert-danger")
+ return redirect(url_for(
+ "species.populations.samples.select_population",
+ species_id=species_id))
+
+ return redirect(url_for("species.populations.samples.list_samples",
+ species_id=species_id,
+ population_id=population["Id"]))
+
+@samplesbp.route("<int:species_id>/populations/<int:population_id>/samples")
+@require_login
+def list_samples(species_id: int, population_id: int):
+ """
+ List the samples in a particular population and give the ability to upload
+ new ones.
+ """
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ if not bool(species):
+ flash("Invalid species!", "alert-danger")
+ return redirect(url_for("species.populations.samples.index"))
+
+ population = population_by_id(conn, population_id)
+ if not bool(population):
+ flash("Population not found!", "alert-danger")
+ return redirect(url_for(
+ "species.populations.samples.select_population",
+ species_id=species_id))
+
+ all_samples = enumerate_sequence(samples_by_species_and_population(
+ conn, species_id, population_id))
+ total_samples = len(all_samples)
+ offset = max(safe_int(request.args.get("from") or 0), 0)
+ count = int(request.args.get("count") or 20)
+ return render_template("samples/list-samples.html",
+ species=species,
+ population=population,
+ samples=all_samples[offset:offset+count],
+ offset=offset,
+ count=count,
+ total_samples=total_samples,
+ activelink="list-samples")
+
+
+def build_sample_upload_job(# pylint: disable=[too-many-arguments]
+ speciesid: int,
+ populationid: int,
+ samplesfile: Path,
+ separator: str,
+ firstlineheading: bool,
+ quotechar: str):
+ """Define the async command to run the actual samples data upload."""
+ return [
+ sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"],
+ str(speciesid), str(populationid), str(samplesfile.absolute()),
+ separator, f"--redisuri={app.config['REDIS_URL']}",
+ f"--quotechar={quotechar}"
+ ] + (["--firstlineheading"] if firstlineheading else [])
+
+
+@samplesbp.route("<int:species_id>/populations/<int:population_id>/upload-samples",
+ methods=["GET", "POST"])
+@require_login
+def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements]
+ """Upload the samples."""
+ samples_uploads_page = redirect(url_for(
+ "species.populations.samples.upload_samples",
+ species_id=species_id,
+ population_id=population_id))
+ if not is_integer_input(species_id):
+ flash("You did not provide a valid species. Please select one to "
+ "continue.",
+ "alert-danger")
+ return redirect(url_for("expression-data.samples.select_species"))
+ species = with_db_connection(lambda conn: species_by_id(conn, species_id))
+ if not bool(species):
+ flash("Species with given ID was not found.", "alert-danger")
+ return redirect(url_for("expression-data.samples.select_species"))
+
+ if not is_integer_input(population_id):
+ flash("You did not provide a valid population. Please select one "
+ "to continue.",
+ "alert-danger")
+ return redirect(url_for("species.populations.samples.select_population",
+ species_id=species_id),
+ code=307)
+ population = with_db_connection(
+ lambda conn: population_by_id(conn, int(population_id)))
+ if not bool(population):
+ flash("Invalid grouping/population!", "alert-error")
+ return redirect(url_for("species.populations.samples.select_population",
+ species_id=species_id),
+ code=307)
+
+ if request.method == "GET" or request.files.get("samples_file") is None:
+ return render_template("samples/upload-samples.html",
+ species=species,
+ population=population)
+
+ try:
+ samples_file = save_file(request.files["samples_file"],
+ Path(app.config["UPLOAD_FOLDER"]))
+ except AssertionError:
+ flash("You need to provide a file with the samples data.",
+ "alert-error")
+ return samples_uploads_page
+
+ firstlineheading = (request.form.get("first_line_heading") == "on")
+
+ separator = request.form.get("separator", ",")
+ if separator == "other":
+ separator = request.form.get("other_separator", ",")
+ if not bool(separator):
+ flash("You need to provide a separator character.", "alert-error")
+ return samples_uploads_page
+
+ quotechar = (request.form.get("field_delimiter", '"') or '"')
+
+ redisuri = app.config["REDIS_URL"]
+ with Redis.from_url(redisuri, decode_responses=True) as rconn:
+ #TODO: Add a QC step here — what do we check?
+ # 1. Does any sample in the uploaded file exist within the database?
+ # If yes, what is/are its/their species and population?
+ # 2. If yes 1. above, provide error with notes on which species and
+ # populations already own the samples.
+ the_job = jobs.launch_job(
+ jobs.initialise_job(
+ rconn,
+ jobs.jobsnamespace(),
+ str(uuid.uuid4()),
+ build_sample_upload_job(
+ species["SpeciesId"],
+ population["InbredSetId"],
+ samples_file,
+ separator,
+ firstlineheading,
+ quotechar),
+ "samples_upload",
+ app.config["JOBS_TTL_SECONDS"],
+ {"job_name": f"Samples Upload: {samples_file.name}"}),
+ redisuri,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors")
+ return redirect(url_for(
+ "species.populations.samples.upload_status",
+ species_id=species_id,
+ population_id=population_id,
+ job_id=the_job["jobid"]))
+
+
+@samplesbp.route("<int:species_id>/populations/<int:population_id>/"
+ "upload-samples/status/<uuid:job_id>",
+ methods=["GET"])
+@require_login
+@with_population(species_redirect_uri="species.populations.samples.index",
+ redirect_uri="species.populations.samples.select_population")
+def upload_status(species: dict, population: dict, job_id: uuid.UUID, **kwargs):# pylint: disable=[unused-argument]
+ """Check on the status of a samples upload job."""
+ job = with_redis_connection(lambda rconn: jobs.job(
+ rconn, jobs.jobsnamespace(), job_id))
+ if job:
+ status = job["status"]
+ if status == "success":
+ return render_template("samples/upload-success.html",
+ job=job,
+ species=species,
+ population=population,)
+
+ if status == "error":
+ return redirect(url_for(
+ "species.populations.samples.upload_failure", job_id=job_id))
+
+ error_filename = Path(jobs.error_filename(
+ job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors"))
+ if error_filename.exists():
+ stat = os.stat(error_filename)
+ if stat.st_size > 0:
+ return redirect(url_for(
+ "samples.upload_failure", job_id=job_id))
+
+ return render_template("samples/upload-progress.html",
+ species=species,
+ population=population,
+ job=job) # maybe also handle this?
+
+ return render_template("no_such_job.html",
+ job_id=job_id,
+ species=species,
+ population=population), 400
+
+@samplesbp.route("/upload/failure/<uuid:job_id>", methods=["GET"])
+@require_login
+def upload_failure(job_id: uuid.UUID):
+ """Display the errors of the samples upload failure."""
+ job = with_redis_connection(lambda rconn: jobs.job(
+ rconn, jobs.jobsnamespace(), job_id))
+ if not bool(job):
+ return render_template("no_such_job.html", job_id=job_id), 400
+
+ error_filename = Path(jobs.error_filename(
+ job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors"))
+ if error_filename.exists():
+ stat = os.stat(error_filename)
+ if stat.st_size > 0:
+ return render_template("worker_failure.html", job_id=job_id)
+
+ return render_template("samples/upload-failure.html", job=job)
diff --git a/uploader/session.py b/uploader/session.py
new file mode 100644
index 0000000..b538187
--- /dev/null
+++ b/uploader/session.py
@@ -0,0 +1,118 @@
+"""Deal with user sessions"""
+from uuid import UUID, uuid4
+from datetime import datetime
+from typing import Any, Optional, TypedDict
+
+from authlib.jose import KeySet
+from flask import request, session
+from pymonad.either import Left, Right, Either
+
+
+class UserDetails(TypedDict):
+ """Session information relating specifically to the user."""
+ user_id: UUID
+ name: str
+ email: str
+ token: Either
+ logged_in: bool
+
+
+class SessionInfo(TypedDict):
+ """All Session information we save."""
+ session_id: UUID
+ user: UserDetails
+ anon_id: UUID
+ user_agent: str
+ ip_addr: str
+ masquerade: Optional[UserDetails]
+ auth_server_jwks: Optional[dict[str, Any]]
+
+
+__SESSION_KEY__ = "GN::uploader::session_info" # Do not use this outside this module!!
+
+
+def clear_session_info():
+ """Clears the session."""
+ session.pop(__SESSION_KEY__)
+
+
+def save_session_info(sess_info: SessionInfo) -> SessionInfo:
+ """Save `session_info`."""
+ # T0d0: if it is an existing session, verify that certain important security
+ # bits have not changed before saving.
+ # old_session_info = session.get(__SESSION_KEY__)
+ # if bool(old_session_info):
+ # if old_session_info["user_agent"] == request.headers.get("User-Agent"):
+ # session[__SESSION_KEY__] = sess_info
+ # return sess_info
+ # # request session verification
+ # return verify_session(sess_info)
+ # New session
+ session[__SESSION_KEY__] = sess_info
+ return sess_info
+
+
+def session_info() -> SessionInfo:
+ """Retrieve the session information"""
+ anon_id = uuid4()
+ return save_session_info(
+ session.get(__SESSION_KEY__, {
+ "session_id": uuid4(),
+ "user": {
+ "user_id": anon_id,
+ "name": "Anonymous User",
+ "email": "anon@ymous.user",
+ "token": Left("INVALID-TOKEN"),
+ "logged_in": False
+ },
+ "anon_id": anon_id,
+ "user_agent": request.headers.get("User-Agent"),
+ "ip_addr": request.environ.get("HTTP_X_FORWARDED_FOR",
+ request.remote_addr),
+ "masquerading": None
+ }))
+
+
+def set_user_token(token: str) -> SessionInfo:
+ """Set the user's token."""
+ info = session_info()
+ return save_session_info({
+ **info, "user": {**info["user"], "token": Right(token)}})#type: ignore[misc]
+
+
+def set_user_details(userdets: UserDetails) -> SessionInfo:
+ """Set the user details information"""
+ return save_session_info({**session_info(), "user": userdets})#type: ignore[misc]
+
+def user_details() -> UserDetails:
+ """Retrieve user details."""
+ return session_info()["user"]
+
+def user_token() -> Either:
+ """Retrieve the user token."""
+ return session_info()["user"]["token"]
+
+
+def set_auth_server_jwks(keyset: KeySet) -> KeySet:
+ """Update the JSON Web Keys in the session."""
+ save_session_info({
+ **session_info(),# type: ignore[misc]
+ "auth_server_jwks": {
+ "last-updated": datetime.now().timestamp(),
+ "jwks": keyset.as_dict()
+ }
+ })
+ return keyset
+
+
+def toggle_token_refreshing():
+ """Toggle the state of the token_refreshing variable."""
+ _session = session_info()
+ return save_session_info({
+ **_session,
+ "token_refreshing": not _session.get("token_refreshing", False)})
+
+
+def is_token_refreshing():
+ """Returns whether the token is being refreshed or not."""
+ return session_info().get("token_refreshing", False)
diff --git a/uploader/species/__init__.py b/uploader/species/__init__.py
new file mode 100644
index 0000000..83f2165
--- /dev/null
+++ b/uploader/species/__init__.py
@@ -0,0 +1,2 @@
+"""Package to handle creation and management of species."""
+from .views import speciesbp
diff --git a/uploader/species/models.py b/uploader/species/models.py
new file mode 100644
index 0000000..9477aa8
--- /dev/null
+++ b/uploader/species/models.py
@@ -0,0 +1,154 @@
+"""Database functions for species."""
+import math
+from typing import Optional
+from functools import reduce
+
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+def all_species(conn: mdb.Connection) -> tuple:
+ "Retrieve the species from the database."
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT Id AS SpeciesId, SpeciesName, LOWER(Name) AS Name, "
+ "MenuName, FullName, TaxonomyId, Family, FamilyOrderId, OrderId "
+ "FROM Species ORDER BY FamilyOrderId ASC, OrderID ASC")
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+def order_species_by_family(species: tuple[dict, ...]) -> list:
+ """Order the species by their family"""
+ def __family_order_id__(item):
+ orderid = item["FamilyOrderId"]
+ return math.inf if orderid is None else orderid
+ def __order__(ordered, current):
+ _key = (__family_order_id__(current), current["Family"])
+ return {
+ **ordered,
+ _key: ordered.get(_key, tuple()) + (current,)
+ }
+ ordered = reduce(__order__, species, {})# type: ignore[var-annotated]
+ return sorted(tuple(ordered.items()), key=lambda item: item[0][0])
+
+
+def species_by_id(conn: mdb.Connection, speciesid) -> dict:
+ "Retrieve the species from the database by id."
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT Id AS SpeciesId, SpeciesName, LOWER(Name) AS Name, "
+ "MenuName, FullName, TaxonomyId, Family, FamilyOrderId, OrderId "
+ "FROM Species WHERE SpeciesId=%s",
+ (speciesid,))
+ return cursor.fetchone()
+
+
+def save_species(conn: mdb.Connection,
+ common_name: str,
+ scientific_name: str,
+ family: str,
+ taxon_id: Optional[str] = None) -> dict:
+ """
+ Save a new species to the database.
+
+ Parameters
+ ----------
+ conn: A connection to the MariaDB database.
+ taxon_id: The taxonomy identifier for the new species.
+ common_name: The species' common name.
+ scientific_name; The species' scientific name.
+ """
+ genus, *species_name = scientific_name.split(" ")
+ species_name = " ".join(species_name)
+ families = species_families(conn)
+ with conn.cursor() as cursor:
+ cursor.execute("SELECT MAX(OrderId) FROM Species")
+ species = {
+ "common_name": common_name,
+ "common_name_lower": common_name.lower(),
+ "menu_name": f"{common_name} ({genus[0]}. {species_name.lower()})",
+ "scientific_name": scientific_name,
+ "family": family,
+ "family_order": families.get(family, 999999),
+ "taxon_id": taxon_id,
+ "species_order": cursor.fetchone()[0] + 5
+ }
+ cursor.execute(
+ "INSERT INTO Species("
+ "SpeciesName, Name, MenuName, FullName, Family, FamilyOrderId, "
+ "TaxonomyId, OrderId"
+ ") VALUES ("
+ "%(common_name)s, %(common_name_lower)s, %(menu_name)s, "
+ "%(scientific_name)s, %(family)s, %(family_order)s, %(taxon_id)s, "
+ "%(species_order)s"
+ ")",
+ species)
+ species_id = cursor.lastrowid
+ cursor.execute("UPDATE Species SET SpeciesId=%s WHERE Id=%s",
+ (species_id, species_id))
+ return {
+ **species,
+ "species_id": species_id
+ }
+
+
+def update_species(# pylint: disable=[too-many-arguments]
+ conn: mdb.Connection,
+ species_id: int,
+ common_name: str,
+ scientific_name: str,
+ family: str,
+ family_order: int,
+ species_order: int
+):
+ """Update a species' details.
+
+ Parameters
+ ----------
+ conn: A connection to the MariaDB database.
+ species_id: The species identifier
+
+ Key-Word Arguments
+ ------------------
+ common_name: A layman's name for the species
+ scientific_name: A binomial nomenclature name for the species
+ family: The grouping under which the species falls
+ family_order: The ordering for the "family" above
+ species_order: The ordering of this species in relation to others
+ """
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ genus, *species_name = scientific_name.split(" ")
+ species_name = " ".join(species_name)
+ species = {
+ "species_id": species_id,
+ "common_name": common_name,
+ "common_name_lower": common_name.lower(),
+ "menu_name": f"{common_name} ({genus[0]}. {species_name.lower()})",
+ "scientific_name": scientific_name,
+ "family": family,
+ "family_order": family_order,
+ "species_order": species_order
+ }
+ cursor.execute(
+ "UPDATE Species SET "
+ "SpeciesName=%(common_name)s, "
+ "Name=%(common_name_lower)s, "
+ "MenuName=%(menu_name)s, "
+ "FullName=%(scientific_name)s, "
+ "Family=%(family)s, "
+ "FamilyOrderId=%(family_order)s, "
+ "OrderId=%(species_order)s "
+ "WHERE Id=%(species_id)s",
+ species)
+
+
+def species_families(conn: mdb.Connection) -> dict:
+ """Retrieve the families under which species are grouped."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT DISTINCT(Family), FamilyOrderId FROM Species "
+ "WHERE Family IS NOT NULL")
+ return {
+ fam["Family"]: fam["FamilyOrderId"]
+ for fam in cursor.fetchall()
+ }
diff --git a/uploader/species/views.py b/uploader/species/views.py
new file mode 100644
index 0000000..f0798d6
--- /dev/null
+++ b/uploader/species/views.py
@@ -0,0 +1,210 @@
+"""Endpoints handling species."""
+from pymonad.either import Left, Right, Either
+from gn_libs.mysqldb import database_connection
+from flask import (flash,
+ escape,
+ request,
+ url_for,
+ redirect,
+ Blueprint,
+ current_app as app)
+
+from uploader.population import popbp
+from uploader.platforms import platformsbp
+from uploader.ui import make_template_renderer
+from uploader.oauth2.client import oauth2_get, oauth2_post
+from uploader.authorisation import require_login, require_token
+from uploader.datautils import order_by_family, enumerate_sequence
+
+from .models import (all_species,
+ save_species,
+ species_by_id,
+ update_species,
+ species_families)
+
+
+speciesbp = Blueprint("species", __name__)
+speciesbp.register_blueprint(popbp, url_prefix="/")
+speciesbp.register_blueprint(platformsbp, url_prefix="/")
+render_template = make_template_renderer("species")
+
+
+@speciesbp.route("/", methods=["GET"])
+@require_login
+def list_species():
+ """List and display all the species in the database."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ return render_template("species/list-species.html",
+ allspecies=enumerate_sequence(all_species(conn)))
+
+@speciesbp.route("/<int:species_id>", methods=["GET"])
+@require_login
+def view_species(species_id: int):
+ """View details of a particular species and menus to act upon it."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ if bool(species):
+ return render_template("species/view-species.html",
+ species=species,
+ activelink="view-species")
+ flash("Could not find a species with the given identifier.",
+ "alert-danger")
+ return redirect(url_for("species.view_species"))
+
+@speciesbp.route("/create", methods=["GET", "POST"])
+@require_login
+def create_species():
+ """Create a new species."""
+ # We can use uniprot's API to fetch the details with something like
+ # https://rest.uniprot.org/taxonomy/<taxonID> e.g.
+ # https://rest.uniprot.org/taxonomy/6239
+ with (database_connection(app.config["SQL_URI"]) as conn,
+ conn.cursor() as cursor):
+ if request.method == "GET":
+ return render_template("species/create-species.html",
+ families=species_families(conn),
+ return_to=(
+ request.args.get("return_to") or ""),
+ activelink="create-species")
+
+ error = False
+ taxon_id = request.form.get("species_taxonomy_id", "").strip() or None
+
+ common_name = request.form.get("common_name", "").strip()
+ if not bool(common_name):
+ flash("The common species name MUST be provided.", "alert-danger")
+ error = True
+
+ scientific_name = request.form.get("scientific_name", "").strip()
+ if not bool(scientific_name):
+ flash("The species' scientific name MUST be provided.",
+ "alert-danger")
+ error = True
+
+ parts = tuple(name.strip() for name in scientific_name.split(" "))
+ if (len(parts) != 2 and len(parts) != 3) or not all(bool(name) for name in parts):
+ flash("The scientific name you provided is invalid.", "alert-danger")
+ error = True
+
+ cursor.execute(
+ "SELECT * FROM Species WHERE FullName=%s", (scientific_name,))
+ res = cursor.fetchone()
+ if bool(res):
+ flash("A species already exists with the provided scientific name.",
+ "alert-danger")
+ error = True
+
+ family = request.form.get("species_family", "").strip()
+ if not bool(family):
+ flash("The species' family MUST be selected.", "alert-danger")
+ error = True
+
+ if bool(taxon_id):
+ cursor.execute(
+ "SELECT * FROM Species WHERE TaxonomyId=%s", (taxon_id,))
+ res = cursor.fetchone()
+ if bool(res):
+ flash("A species already exists with the provided scientific name.",
+ "alert-danger")
+ error = True
+
+ if error:
+ return redirect(url_for("species.create_species",
+ common_name=common_name,
+ scientific_name=scientific_name,
+ taxon_id=taxon_id))
+
+ species = save_species(
+ conn, common_name, scientific_name, family, taxon_id)
+ flash(
+ f"You have successfully added species '{species['scientific_name']} "
+ f"({species['common_name']})'.",
+ "alert-success")
+
+ return_to = request.form.get("return_to").strip()
+ if return_to:
+ return redirect(url_for(return_to, species_id=species["species_id"]))
+ return redirect(url_for("species.view_species", species_id=species["species_id"]))
+
+
+@speciesbp.route("/<int:species_id>/edit-extra", methods=["GET", "POST"])
+@require_login
+@require_token
+#def edit_species(species_id: int):
+def edit_species_extra(token: dict, species_id: int):# pylint: disable=[unused-argument]
+ """Edit a species' details.
+
+ Parameters
+ ----------
+ token: A JWT token used for authorisation.
+ species_id: An identifier for the species being edited.
+ """
+ def __failure__(res):
+ app.logger.debug(
+ "There was an error in the attempt to edit the species: %s", res)
+ flash(res, "alert-danger")
+ return redirect(url_for("species.view_species", species_id=species_id))
+
+ def __system_resource_uuid__(resources) -> Either:
+ sys_res = [
+ resource for resource in resources
+ if resource["resource_category"]["resource_category_key"] == "system"
+ ]
+ if len(sys_res) != 1:
+ return Left("Could not find/identify a valid system resource.")
+ return Right(sys_res[0]["resource_id"])
+
+ def __check_privileges__(authorisations):
+ if len(authorisations.items()) != 1:
+ return Left("Got authorisations for more than a single resource!")
+
+ auths = tuple(authorisations.items())[0][1]
+ authorised = "system:species:edit-extra-info" in tuple(
+ privilege["privilege_id"]
+ for role in auths["roles"]
+ for privilege in role["privileges"])
+ if authorised:
+ return Right(authorised)
+ return Left("You are not authorised to edit species extra details.")
+
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ all_the_species = all_species(conn)
+ families = species_families(conn)
+ family_order = tuple(
+ item[0] for item in order_by_family(all_the_species)
+ if item[0][1] is not None)
+ if bool(species) and request.method == "GET":
+ return oauth2_get("auth/user/resources").then(
+ __system_resource_uuid__
+ ).then(
+ lambda resource_id: oauth2_post(
+ "auth/resource/authorisation",
+ json={"resource-ids": [resource_id]})
+ ).then(__check_privileges__).then(
+ lambda authorisations: render_template(
+ "species/edit-species.html",
+ species=species,
+ families=families,
+ family_order=family_order,
+ max_order_id = max(
+ row["OrderId"] for row in all_the_species
+ if row["OrderId"] is not None),
+ activelink="edit-species")
+ ).either(__failure__, lambda res: res)
+
+ if bool(species) and request.method == "POST":
+ update_species(conn,
+ species_id,
+ request.form["species_name"],
+ request.form["species_fullname"],
+ request.form["species_family"],
+ int(request.form["species_familyorderid"]),
+ int(request.form["species_orderid"]))
+ flash("Updated species successfully.", "alert-success")
+ return redirect(url_for("species.edit_species_extra",
+ species_id=species_id))
+
+ flash("Species with the given identifier was not found!",
+ "alert-danger")
+ return redirect(url_for("species.list_species"))
diff --git a/uploader/static/css/custom-bootstrap.css b/uploader/static/css/custom-bootstrap.css
new file mode 100644
index 0000000..67f1199
--- /dev/null
+++ b/uploader/static/css/custom-bootstrap.css
@@ -0,0 +1,23 @@
+/** Customize some bootstrap selectors **/
+.btn {
+ text-transform: capitalize;
+}
+
+.navbar-inverse {
+ background-color: #336699;
+ border-color: #080808;
+ color: #FFFFFF;
+ background-image: none;
+}
+
+.navbar-inverse .navbar-nav>li>a {
+ color: #FFFFFF;
+}
+
+.navbar-nav > li > a {
+ padding: 5px;
+}
+
+.navbar {
+ min-height: 30px;
+}
diff --git a/uploader/static/css/styles.css b/uploader/static/css/styles.css
new file mode 100644
index 0000000..35c0627
--- /dev/null
+++ b/uploader/static/css/styles.css
@@ -0,0 +1,154 @@
+* {
+ box-sizing: border-box;
+}
+
+body {
+ margin: 0.7em;
+ display: grid;
+ grid-template-columns: 1fr 9fr;
+ grid-gap: 20px;
+
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+ font-style: normal;
+ font-size: 20px;
+}
+
+#header {
+ /* Place it in the parent element */
+ grid-column-start: 1;
+ grid-column-end: 3;
+
+ /* Define layout for the children elements */
+ display: grid;
+ grid-template-columns: 8fr 2fr;
+
+ /* Content styling */
+ background-color: #336699;
+ color: #FFFFFF;
+ border-radius: 3px;
+ min-height: 30px;
+}
+
+#header #header-text {
+ /* Place it in the parent element */
+ grid-column-start: 1;
+ grid-column-end: 2;
+
+ /* Content styling */
+ padding-left: 1em;
+}
+
+#header #header-nav {
+ /* Place it in the parent element */
+ grid-column-start: 2;
+ grid-column-end: 3;
+}
+
+#header #header-nav .nav li a {
+ /* Content styling */
+ color: #FFFFFF;
+ background: #4477AA;
+ border: solid 5px #336699;
+ border-radius: 5px;
+ font-size: 0.7em;
+ text-align: center;
+ padding: 1px 7px;
+}
+
+#nav-sidebar {
+ /* Place it in the parent element */
+ grid-column-start: 1;
+ grid-column-end: 2;
+}
+
+#nav-sidebar .nav li a:hover {
+ border-radius: 0.5em;
+}
+
+#nav-sidebar .nav .activemenu {
+ border-style: solid;
+ border-radius: 0.5em;
+ border-color: #AAAAAA;
+ background-color: #EFEFEF;
+}
+
+#main {
+ /* Place it in the parent element */
+ grid-column-start: 2;
+ grid-column-end: 3;
+
+ /* Define layout for the children elements */
+ display: grid;
+ grid-template-columns: 1fr;
+ grid-template-rows: 4em 100%;
+ grid-gap: 1em;
+}
+
+#main #pagetitle {
+ /* Place it in the parent element */
+ grid-column-start: 1;
+ grid-column-end: 3;
+
+ /* Content-styling */
+ border-radius: 3px;
+ background-color: #88BBEE;
+}
+
+#main #pagetitle .title {
+ font-size: 1.4em;
+ text-transform: capitalize;
+ padding-left: 0.5em;
+}
+
+#main #all-content {
+ /* Place it in the parent element */
+ grid-column-start: 1;
+ grid-column-end: 3;
+
+ /* Define layout for the children elements */
+ display: grid;
+ grid-template-columns: 7fr 3fr; /* For a maximum screen width of 1366 pixels */
+ grid-gap: 1.5em;
+}
+
+#main #all-content .row {
+ margin: 0 2px;
+}
+
+#main #all-content #main-content {
+ background: #FFFFFF;
+ max-width: 950px;
+}
+
+#pagetitle .breadcrumb {
+ background: none;
+ text-transform: capitalize;
+ font-size: 0.75em;
+}
+
+#pagetitle .breadcrumb .active a {
+ color: #333333;
+}
+
+#pagetitle .breadcrumb a {
+ color: #666666;
+}
+
+.heading {
+ border-bottom: solid #EEBB88;
+ text-transform: capitalize;
+}
+
+.subheading {
+ padding: 1em 0 0.1em 0.5em;
+ border-bottom: solid #88BBEE;
+ text-transform: capitalize;
+}
+
+input[type="search"] {
+ border-radius: 5px;
+}
+
+.btn {
+ text-transform: Capitalize;
+}
diff --git a/uploader/static/css/two-column-with-separator.css b/uploader/static/css/two-column-with-separator.css
new file mode 100644
index 0000000..b6efd46
--- /dev/null
+++ b/uploader/static/css/two-column-with-separator.css
@@ -0,0 +1,27 @@
+.two-column-with-separator {
+ display: grid;
+ grid-template-columns: 9fr 1fr 9fr;
+}
+
+.two-col-sep-col1 {
+ grid-column: 1 / 2;
+}
+
+.two-col-sep-separator {
+ grid-column: 2 / 3;
+ text-align: center;
+ color: #FE3535;
+ font-weight: bolder;
+}
+
+.two-col-sep-col2 {
+ grid-column: 3 / 4;
+}
+
+.two-col-sep-col1, .two-col-sep-col2 {
+ border-style: solid;
+ border-color: #FE3535;
+ border-width: 1px;
+ border-radius: 2em;
+ padding: 2em 3em 2em 3em;
+}
diff --git a/uploader/static/images/CITGLogo.png b/uploader/static/images/CITGLogo.png
new file mode 100644
index 0000000..ae99fed
--- /dev/null
+++ b/uploader/static/images/CITGLogo.png
Binary files differ
diff --git a/uploader/static/js/files.js b/uploader/static/js/files.js
new file mode 100644
index 0000000..9d6bca1
--- /dev/null
+++ b/uploader/static/js/files.js
@@ -0,0 +1,118 @@
+var readFirstNLines = (thefile, count, process_content_fns) => {
+ var reader = new FileReader();
+ if(typeof thefile !== "undefined" && thefile !== null) {
+ reader.addEventListener("load", (event) => {
+ var content = event
+ .target
+ .result
+ .split("\n")
+ .slice(0, count)
+ .map((line) => {return line.trim("\r");});
+ process_content_fns.forEach((fn) => {fn(content);});
+ });
+ reader.readAsText(thefile);
+ }
+};
+var read_first_n_lines = readFirstNLines;
+
+
+var readBinaryFile = (file) => {
+ return new Promise((resolve, reject) => {
+ var _reader = new FileReader();
+ _reader.onload = (event) => {resolve(_reader.result);};
+ _reader.readAsArrayBuffer(file);
+ });
+};
+
+
+var Uint8ArrayToHex = (arr) => {
+ var toHex = (val) => {
+ _hex = val.toString(16);
+ if(_hex.length < 2) {
+ return "0" + val;
+ }
+ return _hex;
+ };
+ _hexstr = ""
+ arr.forEach((val) => {_hexstr += toHex(val)});
+ return _hexstr
+};
+
+
+var computeFileChecksum = (file) => {
+ return readBinaryFile(file)
+ .then((content) => {
+ return window.crypto.subtle.digest(
+ "SHA-256", new Uint8Array(content));
+ }).then((digest) => {
+ return Uint8ArrayToHex(new Uint8Array(digest))
+ });
+};
+
+
+var defaultResumableHandler = (event) => {
+ throw new Error("Please provide a valid event handler!");
+};
+
+var addHandler = (resumable, handlername, handler) => {
+ if(resumable.support) {
+ resumable.on(handlername, (handler || defaultResumableHandler));
+ }
+ return resumable;
+};
+
+
+var makeResumableHandler = (handlername) => {
+ return (resumable, handler) => {
+ return addHandler(resumable, handlername, handler);
+ };
+};
+
+
+var fileSuccessHandler = makeResumableHandler("fileSuccess");
+var fileProgressHandler = makeResumableHandler("fileProgress");
+var fileAddedHandler = makeResumableHandler("fileAdded");
+var filesAddedHandler = makeResumableHandler("filesAdded");
+var filesRetryHandler = makeResumableHandler("filesRetry");
+var filesErrorHandler = makeResumableHandler("filesError");
+var uploadStartHandler = makeResumableHandler("uploadStart");
+var completeHandler = makeResumableHandler("complete");
+var progressHandler = makeResumableHandler("progress");
+var errorHandler = makeResumableHandler("error");
+
+
+var markResumableDragAndDropElement = (resumable, fileinput, droparea, browsebutton) => {
+ if(resumable.support) {
+ //Hide file input element and display drag&drop UI
+ add_class(fileinput, "hidden");
+ remove_class(droparea, "hidden");
+
+ // Define UI elements for browse and drag&drop
+ resumable.assignDrop(droparea);
+ resumable.assignBrowse(browsebutton);
+ }
+
+ return resumable;
+};
+
+
+var makeResumableElement = (targeturi, fileinput, droparea, uploadbutton, filetype) => {
+ var resumable = Resumable({
+ target: targeturi,
+ fileType: filetype,
+ maxFiles: 1,
+ forceChunkSize: true,
+ generateUniqueIdentifier: (file, event) => {
+ return computeFileChecksum(file).then((checksum) => {
+ var _relativePath = (file.webkitRelativePath
+ || file.relativePath
+ || file.fileName
+ || file.name);
+ return checksum + "-" + _relativePath.replace(
+ /[^a-zA-Z0-9_-]/img, "");
+ });
+ }
+ });
+
+ return resumable;
+};
diff --git a/uploader/static/js/misc.js b/uploader/static/js/misc.js
new file mode 100644
index 0000000..cf7b39e
--- /dev/null
+++ b/uploader/static/js/misc.js
@@ -0,0 +1,6 @@
+"Miscellaneous functions and event-handlers"
+
+$(".not-implemented").click((event) => {
+ event.preventDefault();
+ alert("This feature is not implemented yet. Please bear with us.");
+});
diff --git a/uploader/static/js/populations.js b/uploader/static/js/populations.js
new file mode 100644
index 0000000..ded9b10
--- /dev/null
+++ b/uploader/static/js/populations.js
@@ -0,0 +1,40 @@
+var populationDataTable = (populationdata) => {
+ var lengthMenu = [10, 25, 50, 100, 1000];
+ if(populationdata.length > 1000) {
+ lengthMenu.push(populationdata.length)
+ }
+ $("#tbl-select-population").DataTable({
+ responsive: true,
+ lengthMenu: lengthMenu,
+ language: {
+ processing: "Processing… Please wait.",
+ loadingRecords: "Loading population — Please wait.",
+ lengthMenu: "Show _MENU_ populations",
+ info: "Showing _START_ to _END_ of _TOTAL_ populations"
+ },
+ data: populationdata,
+ columns: [
+ {
+ data: (apopulation) => {
+ return `<input type="radio" name="population_id"`
+ + `id="rdo_population_id_${apopulation.InbredSetId}" `
+ + `value="${apopulation.InbredSetId}">`;
+ }
+ },
+ {
+ data: (apopulation) => {
+ return `<label for="rdo_population_id_${apopulation.InbredSetId}" `
+ + `class="control-label" style="font-weight: 1;">`
+ + `${apopulation.FullName} (${apopulation.InbredSetName})`
+ + `</label>`;
+ }
+ }
+ ]
+ });
+};
+
+
+$(() => {
+ populationDataTable(JSON.parse(
+ $("#tbl-select-population").attr("data-populations-list")));
+});
diff --git a/uploader/static/js/select_platform.js b/uploader/static/js/select_platform.js
new file mode 100644
index 0000000..4fdd865
--- /dev/null
+++ b/uploader/static/js/select_platform.js
@@ -0,0 +1,70 @@
+function radio_column(chip) {
+ col = document.createElement("td");
+ radio = document.createElement("input");
+ radio.setAttribute("type", "radio");
+ radio.setAttribute("name", "genechipid");
+ radio.setAttribute("value", chip["GeneChipId"]);
+ radio.setAttribute("required", "required");
+ col.appendChild(radio);
+ return col;
+}
+
+function setup_genechips(genechip_data) {
+ columns = ["GeneChipId", "GeneChipName"]
+ submit_button = document.querySelector(
+ "#select-platform-form button[type='submit']");
+ elt = document.getElementById(
+ "genechips-table").getElementsByTagName("tbody")[0];
+ remove_children(elt);
+ if((genechip_data === undefined) || genechip_data.length === 0) {
+ row = document.createElement("tr");
+ col = document.createElement("td");
+ col.setAttribute("colspan", "3");
+ text = document.createTextNode("No chips found for selected species");
+ col.appendChild(text);
+ row.appendChild(col);
+ elt.appendChild(row);
+ submit_button.setAttribute("disabled", true);
+ return false;
+ }
+
+ submit_button.removeAttribute("disabled")
+ genechip_data.forEach(chip => {
+ row = document.createElement("tr");
+ row.appendChild(radio_column(chip));
+ columns.forEach(column => {
+ col = document.createElement("td");
+ content = document.createTextNode(chip[column]);
+ col.appendChild(content);
+ row.appendChild(col);
+ });
+ elt.appendChild(row);
+ });
+}
+
+function genechips() {
+ return JSON.parse(
+ document.getElementById("select-platform-form").getAttribute(
+ "data-genechips"));
+}
+
+function update_genechips(event) {
+ genec = genechips();
+
+ species_elt = document.getElementById("species");
+
+ if(event.target == species_elt) {
+ setup_genechips(genec[species_elt.value.toLowerCase()]);
+ }
+}
+
+function select_row_radio(row) {
+ radio = row.getElementsByTagName(
+ "td")[0].getElementsByTagName(
+ "input")[0];
+ if(radio === undefined) {
+ return false;
+ }
+ radio.setAttribute("checked", "checked");
+ return true;
+}
diff --git a/uploader/static/js/species.js b/uploader/static/js/species.js
new file mode 100644
index 0000000..b070725
--- /dev/null
+++ b/uploader/static/js/species.js
@@ -0,0 +1,39 @@
+var speciesDataTable = (speciesdata) => {
+ var lengthMenu = [10, 25, 50, 100, 1000];
+ if(speciesdata.length > 1000) {
+ lengthMenu.push(speciesdata.length)
+ }
+ $("#tbl-select-species").DataTable({
+ responsive: true,
+ lengthMenu: lengthMenu,
+ language: {
+ processing: "Processing… Please wait.",
+ loadingRecords: "Loading species — Please wait.",
+ lengthMenu: "Show _MENU_ species",
+ info: "Showing _START_ to _END_ of _TOTAL_ species"
+ },
+ data: speciesdata,
+ columns: [
+ {
+ data: (aspecies) => {
+ return `<input type="radio" name="species_id"`
+ + `id="rdo_species_id_${aspecies.SpeciesId}" `
+ + `value="${aspecies.SpeciesId}">`;
+ }
+ },
+ {
+ data: (aspecies) => {
+ return `<label for="rdo_species_id_${aspecies.SpeciesId}" `
+ + `class="control-label" style="font-weight: 1;">`
+ + `${aspecies.FullName} (${aspecies.SpeciesName})`
+ + `</label>`;
+ }
+ }
+ ]
+ });
+};
+
+$(() => {
+ speciesDataTable(JSON.parse(
+ $("#tbl-select-species").attr("data-species-list")));
+});
diff --git a/uploader/static/js/upload_progress.js b/uploader/static/js/upload_progress.js
new file mode 100644
index 0000000..9638b36
--- /dev/null
+++ b/uploader/static/js/upload_progress.js
@@ -0,0 +1,97 @@
+function make_processing_indicator(elt) {
+ var count = 0;
+ return function() {
+ var message = "Finalising upload and saving file: "
+ if(count > 5) {
+ count = 1;
+ }
+ for(i = 0; i < count; i++) {
+ message = message + ".";
+ }
+ elt.innerHTML = message
+ count = count + 1
+ };
+}
+
+function make_progress_updater(file, indicator_elt) {
+ var progress_bar = indicator_elt.querySelector("#progress-bar");
+ var progress_text = indicator_elt.querySelector("#progress-text");
+ var extra_text = indicator_elt.querySelector("#progress-extra-text");
+ return function(event) {
+ if(event.loaded <= file.size) {
+ var percent = Math.round((event.loaded / file.size) * 100);
+ progress_bar.value = percent
+ progress_text.innerHTML = "Uploading: " + percent + "%";
+ extra_text.setAttribute("class", "hidden")
+ }
+
+ if(event.loaded == event.total) {
+ progress_bar.value = 100;
+ progress_text.innerHTML = "Uploaded: 100%";
+ extra_text.setAttribute("class", null);
+ intv = setInterval(make_processing_indicator(extra_text), 400);
+ setTimeout(function() {clearTimeout(intv);}, 20000);
+ }
+ };
+}
+
+function setup_cancel_upload(request, indicator_elt) {
+ document.getElementById("btn-cancel-upload").addEventListener(
+ "click", function(event) {
+ event.preventDefault();
+ request.abort();
+ });
+}
+
+function setup_request(file, progress_indicator_elt) {
+ var request = new XMLHttpRequest();
+ var updater = make_progress_updater(file, progress_indicator_elt);
+ request.upload.addEventListener("progress", updater);
+ request.onload = function(event) {
+ document.location.assign(request.responseURL);
+ };
+ setup_cancel_upload(request, progress_indicator_elt)
+ return request;
+}
+
+function selected_filetype(radios) {
+ for(idx = 0; idx < radios.length; idx++) {
+ if(radios[idx].checked) {
+ return radios[idx].value;
+ }
+ }
+}
+
+function make_data_uploader(setup_formdata) {
+ return function(event) {
+ event.preventDefault();
+
+ var pindicator = document.getElementById("upload-progress-indicator");
+
+ var form = event.target;
+ var the_file = form.querySelector("input[type='file']").files[0];
+ if(the_file === undefined) {
+ form.querySelector("input[type='file']").parentElement.setAttribute(
+ "class", "invalid-input");
+ error_elt = form.querySelector("#no-file-error");
+ if(error_elt !== undefined) {
+ error_elt.setAttribute("style", "display: block;");
+ }
+ return false;
+ }
+ var formdata = setup_formdata(form);
+
+ document.getElementById("progress-filename").innerHTML = the_file.name;
+ var request = setup_request(the_file, pindicator);
+ request.open(form.getAttribute("method"), form.getAttribute("action"));
+ request.send(formdata);
+ return false;
+ }
+}
+
+
+function setup_upload_handlers(formid, datauploader) {
+ console.info("Setting up the upload handlers.")
+ upload_form = document.getElementById(formid);
+ upload_form.addEventListener("submit", datauploader);
+}
diff --git a/uploader/static/js/upload_samples.js b/uploader/static/js/upload_samples.js
new file mode 100644
index 0000000..aed536f
--- /dev/null
+++ b/uploader/static/js/upload_samples.js
@@ -0,0 +1,132 @@
+/*
+ * Read the file content and set the `data-preview-content` attribute on the
+ * file element
+ */
+function read_first_n_lines(event,
+ fileelement,
+ numlines,
+ firstlineheading = true) {
+ var thefile = fileelement.files[0];
+ var reader = new FileReader();
+ reader.addEventListener("load", (event) => {
+ var filecontent = event.target.result.split(
+ "\n").slice(
+ 0, (numlines + (firstlineheading ? 1 : 0))).map(
+ (line) => {return line.trim("\r");});
+ fileelement.setAttribute(
+ "data-preview-content", JSON.stringify(filecontent));
+ display_preview(event);
+ })
+ reader.readAsText(thefile);
+}
+
+function remove_rows(preview_table) {
+ var table_body = preview_table.getElementsByTagName("tbody")[0];
+ while(table_body.children.length > 0) {
+ table_body.removeChild(table_body.children.item(0));
+ }
+}
+
+/*
+ * Display error row
+ */
+function display_error_row(preview_table, error_message) {
+ remove_rows(preview_table);
+ row = document.createElement("tr");
+ cell = document.createElement("td");
+ cell.setAttribute("colspan", 4);
+ cell.innerHTML = error_message;
+ row.appendChild(cell);
+ preview_table.getElementsByTagName("tbody")[0].appendChild(row);
+}
+
+function strip(str, chars) {
+ var end = str.length;
+ var start = 0
+ for(var j = str.length; j > 0; j--) {
+ if(!chars.includes(str[j - 1])) {
+ break;
+ }
+ end = end - 1;
+ }
+ for(var i = 0; i < end; i++) {
+ if(!chars.includes(str[i])) {
+ break;
+ }
+ start = start + 1;
+ }
+ return str.slice(start, end);
+}
+
+function process_preview_data(preview_data, separator, delimiter) {
+ return preview_data.map((line) => {
+ return line.split(separator).map((field) => {
+ return strip(field, delimiter);
+ });
+ });
+}
+
+function render_preview(preview_table, preview_data) {
+ remove_rows(preview_table);
+ var table_body = preview_table.getElementsByTagName("tbody")[0];
+ preview_data.forEach((line) => {
+ var row = document.createElement("tr");
+ line.forEach((field) => {
+ var cell = document.createElement("td");
+ cell.innerHTML = field;
+ row.appendChild(cell);
+ });
+ table_body.appendChild(row);
+ });
+}
+
+/*
+ * Display a preview of the data, relying on the user's selection.
+ */
+function display_preview(event) {
+ var data_preview_table = document.getElementById("tbl:samples-preview");
+ remove_rows(data_preview_table);
+
+ var separator = document.getElementById("select:separator").value;
+ if(separator === "other") {
+ separator = document.getElementById("txt:separator").value;
+ }
+ if(separator == "") {
+ display_error_row(data_preview_table, "Please provide a separator.");
+ return false;
+ }
+
+ var delimiter = document.getElementById("txt:delimiter").value;
+
+ var firstlineheading = document.getElementById("chk:heading").checked;
+
+ var fileelement = document.getElementById("file:samples");
+ var preview_data = JSON.parse(
+ fileelement.getAttribute("data-preview-content") || "[]");
+ if(preview_data.length == 0) {
+ display_error_row(
+ data_preview_table,
+ "No file data to preview. Check that file is provided.");
+ }
+
+ render_preview(data_preview_table, process_preview_data(
+ preview_data.slice(0 + (firstlineheading ? 1 : 0)),
+ separator,
+ delimiter));
+}
+
+document.getElementById("chk:heading").addEventListener(
+ "change", display_preview);
+document.getElementById("select:separator").addEventListener(
+ "change", display_preview);
+document.getElementById("txt:separator").addEventListener(
+ "keyup", display_preview);
+document.getElementById("txt:delimiter").addEventListener(
+ "keyup", display_preview);
+document.getElementById("file:samples").addEventListener(
+ "change", (event) => {
+ read_first_n_lines(event,
+ document.getElementById("file:samples"),
+ 30,
+ document.getElementById("chk:heading").checked);
+ });
diff --git a/uploader/static/js/utils.js b/uploader/static/js/utils.js
new file mode 100644
index 0000000..045dd47
--- /dev/null
+++ b/uploader/static/js/utils.js
@@ -0,0 +1,10 @@
+function remove_children(element) {
+ Array.from(element.children).forEach(child => {
+ element.removeChild(child);
+ });
+}
+
+function trigger_change_event(element) {
+ evt = new Event("change");
+ element.dispatchEvent(evt);
+}
diff --git a/uploader/templates/base.html b/uploader/templates/base.html
new file mode 100644
index 0000000..5eca445
--- /dev/null
+++ b/uploader/templates/base.html
@@ -0,0 +1,139 @@
+<!DOCTYPE html>
+<html lang="en">
+
+ <head>
+
+ <meta charset="UTF-8" />
+ <meta application-name="GeneNetwork Quality-Control Application" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ {%block extrameta%}{%endblock%}
+
+ <title>Data Upload and Quality Control: {%block title%}{%endblock%}</title>
+
+ <link rel="stylesheet" type="text/css"
+ href="{{url_for('base.bootstrap',
+ filename='css/bootstrap.min.css')}}" />
+ <link rel="stylesheet" type="text/css"
+ href="{{url_for('base.bootstrap',
+ filename='css/bootstrap-theme.min.css')}}" />
+ <link rel="stylesheet" type="text/css" href="/static/css/styles.css" />
+ <link rel="stylesheet"
+ href="{{url_for('base.datatables', filename='css/jquery.dataTables.css')}}" />
+
+ {%block css%}{%endblock%}
+
+ </head>
+
+ <body>
+ <header id="header">
+ <span id="header-text">GeneNetwork</span>
+ <nav id="header-nav">
+ <ul class="nav justify-content-end">
+ <li>
+ {%if user_logged_in()%}
+ <a href="{{url_for('oauth2.logout')}}"
+ title="Log out of the system">
+ <span class="glyphicon glyphicon-user"></span>
+ Sign Out</a>
+ {%else%}
+ <a href="{{authserver_authorise_uri()}}"
+ title="Log in to the system">Sign In</a>
+ {%endif%}
+ </li>
+ </ul>
+ </nav>
+ </header>
+
+ <aside id="nav-sidebar">
+ <ul class="nav flex-column">
+ <li {%if activemenu=="home"%}class="activemenu"{%endif%}>
+ <a href="/" >Home</a></li>
+ <li {%if activemenu=="species"%}class="activemenu"{%endif%}>
+ <a href="{{url_for('species.list_species')}}"
+ title="View and manage species information.">Species</a></li>
+ <li {%if activemenu=="platforms"%}class="activemenu"{%endif%}>
+ <a href="{{url_for('species.platforms.index')}}"
+ title="View and manage species platforms.">Sequencing Platforms</a></li>
+ <li {%if activemenu=="populations"%}class="activemenu"{%endif%}>
+ <a href="{{url_for('species.populations.index')}}"
+ title="View and manage species populations.">Populations</a></li>
+ <li {%if activemenu=="samples"%}class="activemenu"{%endif%}>
+ <a href="{{url_for('species.populations.samples.index')}}"
+ title="Upload population samples.">Samples</a></li>
+ <li {%if activemenu=="genotypes"%}class="activemenu"{%endif%}>
+ <a href="{{url_for('species.populations.genotypes.index')}}"
+ title="Upload Genotype data.">Genotype Data</a></li>
+ <!--
+ TODO: Maybe include menus here for managing studies and dataset or
+ maybe have the studies/datasets managed under their respective
+ sections, e.g. "Publish*" studies/datasets under the "Phenotypes"
+ section, "ProbeSet*" studies/datasets under the "Expression Data"
+ sections, etc.
+ -->
+ <li {%if activemenu=="phenotypes"%}class="activemenu"{%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.index')}}"
+ title="Upload phenotype data.">Phenotype Data</a></li>
+ <!--
+ <li {%if activemenu=="expression-data"%}class="activemenu"{%endif%}>
+ <a href="{{url_for('species.populations.expression-data.index')}}"
+ title="Upload expression data."
+ class="not-implemented">Expression Data</a></li>
+ <li {%if activemenu=="individuals"%}class="activemenu"{%endif%}>
+ <a href="#"
+ class="not-implemented"
+ title="Upload individual data.">Individual Data</a></li>
+ <li {%if activemenu=="rna-seq"%}class="activemenu"{%endif%}>
+ <a href="#"
+ class="not-implemented"
+ title="Upload RNA-Seq data.">RNA-Seq Data</a></li>
+ <li {%if activemenu=="async-jobs"%}class="activemenu"{%endif%}>
+ <a href="#"
+ class="not-implemented"
+ title="View and manage the backgroud jobs you have running">
+ Background Jobs</a></li>
+ -->
+ </ul>
+ </aside>
+
+ <main id="main" class="main">
+
+ <div id="pagetitle" class="pagetitle">
+ <span class="title">Data Upload and Quality Control: {%block pagetitle%}{%endblock%}</span>
+ <!--
+ <nav>
+ <ol class="breadcrumb">
+ <li {%if activelink is not defined or activelink=="home"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('base.index')}}">Home</a>
+ </li>
+ {%block lvl1_breadcrumbs%}{%endblock%}
+ </ol>
+ </nav>
+ -->
+ </div>
+
+ <div id="all-content">
+ <div id="main-content">
+ {%block contents%}{%endblock%}
+ </div>
+ <div id="sidebar-content">
+ {%block sidebarcontents%}{%endblock%}
+ </div>
+ </div>
+ </main>
+
+
+ <script src="{{url_for('base.jquery',
+ filename='jquery.min.js')}}"></script>
+ <script src="{{url_for('base.bootstrap',
+ filename='js/bootstrap.min.js')}}"></script>
+ <script type="text/javascript" src="/static/js/misc.js"></script>
+ <script type="text/javascript"
+ src="{{url_for('base.datatables',
+ filename='js/jquery.dataTables.js')}}"></script>
+ {%block javascript%}{%endblock%}
+ </body>
+</html>
diff --git a/uploader/templates/cli-output.html b/uploader/templates/cli-output.html
new file mode 100644
index 0000000..64b1a9a
--- /dev/null
+++ b/uploader/templates/cli-output.html
@@ -0,0 +1,8 @@
+{%macro cli_output(job, stream)%}
+
+<h4 class="subheading">{{stream | upper}} Output</h4>
+<div class="cli-output" style="max-height: 10em; overflow: auto;">
+ <pre>{{job.get(stream, "")}}</pre>
+</div>
+
+{%endmacro%}
diff --git a/uploader/templates/continue_from_create_dataset.html b/uploader/templates/continue_from_create_dataset.html
new file mode 100644
index 0000000..03bb49c
--- /dev/null
+++ b/uploader/templates/continue_from_create_dataset.html
@@ -0,0 +1,52 @@
+{%extends "base.html"%}
+{%from "dbupdate_hidden_fields.html" import hidden_fields%}
+
+{%block title%}Create Study{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
+{%endblock%}
+
+{%block contents%}
+<h2 class="heading">{{filename}}: create study</h2>
+
+{%with messages = get_flashed_messages(with_categories=true)%}
+{%if messages:%}
+<ul>
+ {%for category, message in messages:%}
+ <li class="{{category}}">{{message}}</li>
+ {%endfor%}
+</ul>
+{%endif%}
+{%endwith%}
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.final_confirmation')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col1">
+ <legend>continue with new dataset</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, datasetid=datasetid, totallines=totallines)}}
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+<div class="row">
+ <p class="two-col-sep-separator">OR</p>
+</div>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.select_dataset')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col2">
+ <legend>Select from existing dataset</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, datasetid=datasetid, totallines=totallines)}}
+
+ <button type="submit" class="btn btn-primary">go back</button>
+ </form>
+</div>
+{%endblock%}
diff --git a/uploader/templates/continue_from_create_study.html b/uploader/templates/continue_from_create_study.html
new file mode 100644
index 0000000..34e6e5e
--- /dev/null
+++ b/uploader/templates/continue_from_create_study.html
@@ -0,0 +1,52 @@
+{%extends "base.html"%}
+{%from "dbupdate_hidden_fields.html" import hidden_fields%}
+
+{%block title%}Create Study{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
+{%endblock%}
+
+{%block contents%}
+<h2 class="heading">{{filename}}: create study</h2>
+
+{%with messages = get_flashed_messages(with_categories=true)%}
+{%if messages:%}
+<ul>
+ {%for category, message in messages:%}
+ <li class="{{category}}">{{message}}</li>
+ {%endfor%}
+</ul>
+{%endif%}
+{%endwith%}
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.select_dataset')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col1">
+ <legend>continue with new study</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, totallines=totallines)}}
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+<div class="row">
+ <p class="two-col-sep-separator">OR</p>
+</div>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.select_study')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col2">
+ <legend>Select from existing study</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, totallines=totallines)}}
+
+ <button type="submit" class="btn btn-primary">go back</button>
+ </form>
+</div>
+{%endblock%}
diff --git a/uploader/templates/dbupdate_error.html b/uploader/templates/dbupdate_error.html
new file mode 100644
index 0000000..e1359d2
--- /dev/null
+++ b/uploader/templates/dbupdate_error.html
@@ -0,0 +1,12 @@
+{%extends "base.html"%}
+
+{%block title%}DB Update Error{%endblock%}
+
+{%block contents%}
+<h1 class="heading">database update error</h2>
+
+<p class="alert-danger">
+ <strong>Database Update Error</strong>: {{error_message}}
+</p>
+
+{%endblock%}
diff --git a/uploader/templates/dbupdate_hidden_fields.html b/uploader/templates/dbupdate_hidden_fields.html
new file mode 100644
index 0000000..ccbc299
--- /dev/null
+++ b/uploader/templates/dbupdate_hidden_fields.html
@@ -0,0 +1,29 @@
+{%macro hidden_fields(filename, filetype):%}
+
+<!-- {{kwargs}}: mostly for accessing the kwargs in macro -->
+
+<input type="hidden" name="filename" value="{{filename}}" />
+<input type="hidden" name="filetype" value="{{filetype}}" />
+{%if kwargs.get("totallines")%}
+<input type="hidden" name="totallines" value="{{kwargs['totallines']}}" />
+{%endif%}
+{%if kwargs.get("species"):%}
+<input type="hidden" name="species" value="{{kwargs['species']}}" />
+{%endif%}
+{%if kwargs.get("genechipid"):%}
+<input type="hidden" name="genechipid" value="{{kwargs['genechipid']}}" />
+{%endif%}
+{%if kwargs.get("inbredsetid"):%}
+<input type="hidden" name="inbredsetid" value="{{kwargs['inbredsetid']}}" />
+{%endif%}
+{%if kwargs.get("tissueid"):%}
+<input type="hidden" name="tissueid" value="{{kwargs['tissueid']}}" />
+{%endif%}
+{%if kwargs.get("studyid"):%}
+<input type="hidden" name="studyid" value="{{kwargs['studyid']}}" />
+{%endif%}
+{%if kwargs.get("datasetid"):%}
+<input type="hidden" name="datasetid" value="{{kwargs['datasetid']}}" />
+{%endif%}
+
+{%endmacro%}
diff --git a/uploader/templates/errors_display.html b/uploader/templates/errors_display.html
new file mode 100644
index 0000000..715cfcf
--- /dev/null
+++ b/uploader/templates/errors_display.html
@@ -0,0 +1,47 @@
+{%macro errors_display(errors, no_error_msg, error_message, complete)%}
+
+{%if errors | length == 0 %}
+<span {%if complete%}class="alert-success"{%endif%}>{{no_error_msg}}</span>
+{%else %}
+<p class="alert-danger">{{error_message}}</p>
+
+<table class="table reports-table">
+ <thead>
+ <tr>
+ <th>line number</th>
+ <th>column(s)</th>
+ <th>error</th>
+ <th>error message</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for error in errors%}
+ <tr>
+ <td>{{error["line"]}}</td>
+ <td>
+ {%if isinvalidvalue(error):%}
+ {{error.column}}
+ {%elif isduplicateheading(error): %}
+ {{error.columns}}
+ {%else: %}
+ -
+ {%endif %}
+ </td>
+ <td>
+ {%if isinvalidvalue(error):%}
+ Invalid Value
+ {%elif isduplicateheading(error): %}
+ Duplicate Header
+ {%else%}
+ Inconsistent Columns
+ {%endif %}
+ </td>
+ <td>{{error["message"]}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+</table>
+{%endif%}
+
+{%endmacro%}
diff --git a/uploader/templates/expression-data/base.html b/uploader/templates/expression-data/base.html
new file mode 100644
index 0000000..d63fd7e
--- /dev/null
+++ b/uploader/templates/expression-data/base.html
@@ -0,0 +1,13 @@
+{%extends "populations/base.html"%}
+
+{%block lvl3_breadcrumbs%}
+<li {%if activelink=="expression-data"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.expression-data.index')}}">
+ Expression Data</a>
+</li>
+{%block lvl4_breadcrumbs%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/expression-data/data-review.html b/uploader/templates/expression-data/data-review.html
new file mode 100644
index 0000000..c985b03
--- /dev/null
+++ b/uploader/templates/expression-data/data-review.html
@@ -0,0 +1,85 @@
+{%extends "base.html"%}
+
+{%block title%}Data Review{%endblock%}
+
+{%block contents%}
+<h1 class="heading">data review</h1>
+
+<div class="row">
+ <h2 id="data-concerns">Data Concerns</h2>
+ <p>The following are some of the requirements that the data in your file
+ <strong>MUST</strong> fulfil before it is considered valid for this system:
+ </p>
+
+ <ol>
+ <li>File headings
+ <ul>
+ <li>The first row in the file should contains the headings. The number of
+ headings in this first row determines the number of columns expected for
+ all other lines in the file.</li>
+ <li>Each heading value in the first row MUST appear in the first row
+ <strong>ONE AND ONLY ONE</strong> time</li>
+ <li>The sample/cases (previously 'strains') headers in your first row will be
+ against those in the <a href="https://genenetwork.org"
+ title="Link to the GeneNetwork service">
+ GeneNetwork</a> database.<br />
+ <small class="text-muted">
+ If you encounter an error saying your sample(s)/case(s) do not exist
+ in the GeneNetwork database, then you will have to use the
+ <a href="{{url_for('species.populations.samples.index')}}"
+ title="Upload samples/cases feature">Upload Samples/Cases</a>
+ option on this system to upload them.
+ </small>
+ </ul>
+ </li>
+
+ <li>Data
+ <ol>
+ <li><strong>NONE</strong> of the data cells/fields is allowed to be empty.
+ All fields/cells <strong>MUST</strong> contain a value.</li>
+ <li>The first column of the data rows will be considered a textual field,
+ holding the "identifier" for that row<li>
+ <li>Except for the first column/field for each data row,
+ <strong>NONE</strong> of the data columns/cells/fields should contain
+ spurious characters like `eeeee`, `5.555iloveguix`, etc...<br />
+ All of them should be decimal values</li>
+ <li>decimal numbers must conform to the following criteria:
+ <ul>
+ <li>when checking an average file decimal numbers must have exactly three
+ decimal places to the right of the decimal point.</li>
+ <li>when checking a standard error file decimal numbers must have six or
+ greater decimal places to the right of the decimal point.</li>
+ <li>there must be a number to the left side of the decimal place
+ (e.g. 0.55555 is allowed but .55555 is not).</li>
+ </ul>
+ </li>
+ </ol>
+ </li>
+ </ol>
+</div>
+
+
+<div class="row">
+ <h2 id="file-types">Supported File Types</h2>
+ We support the following file types:
+
+ <ul>
+ <li>Tab-Separated value files (.tsv)
+ <ul>
+ <li>The <strong>TAB</strong> character is used to separate the fields of each
+ column</li>
+ <li>The values of each field <strong>ARE NOT</strong> quoted.</li>
+ <li>Here is an
+ <a href="https://gitlab.com/fredmanglis/gnqc_py/-/blob/main/tests/test_data/no_data_errors.tsv"
+ target="_blank">example file</a> with a single data row.</li>
+ </ul>
+ </li>
+ <li>.txt files: Content has the same format as .tsv file above</li>
+ <li>.zip files: each zip file should contain
+ <strong>ONE AND ONLY ONE</strong> file of the .tsv or .txt type above.
+ <br />Any zip file with more than one file is invalid, and so is an empty
+ zip file.</li>
+ </ul>
+
+</div>
+{%endblock%}
diff --git a/uploader/templates/expression-data/index.html b/uploader/templates/expression-data/index.html
new file mode 100644
index 0000000..9ba3582
--- /dev/null
+++ b/uploader/templates/expression-data/index.html
@@ -0,0 +1,33 @@
+{%extends "expression-data/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-select-species.html" import select_species_form%}
+
+{%block title%}Expression Data{%endblock%}
+
+{%block pagetitle%}Expression Data{%endblock%}
+
+{%block breadcrumb%}
+<li class="breadcrumb-item">
+ <a href="{{url_for('base.index')}}">Home</a>
+</li>
+<li class="breadcrumb-item active">
+ <a href="{{url_for('species.populations.expression-data.index')}}"
+ title="Upload expression data.">
+ Expression Data</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+<div class="row">
+ <h2 class="heading">Expression Data</h2>
+ {{flash_all_messages()}}
+
+ <p>This section allows you to enter the expression data for your experiment.
+ You will need to select the species that your data concerns below.</p>
+</div>
+
+<div class="row">
+ {{select_species_form(url_for("species.populations.expression-data.index"),
+ species)}}
+</div>
+{%endblock%}
diff --git a/uploader/templates/expression-data/job-progress.html b/uploader/templates/expression-data/job-progress.html
new file mode 100644
index 0000000..ef264e1
--- /dev/null
+++ b/uploader/templates/expression-data/job-progress.html
@@ -0,0 +1,47 @@
+{%extends "base.html"%}
+{%from "errors_display.html" import errors_display%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="5">
+{%endblock%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job_name}}</h2>
+
+<div class="row">
+ <form action="{{url_for('species.populations.expression-data.abort',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}" method="POST">
+ <legend class="heading">Status</legend>
+ <div class="form-group">
+ <label for="job_status" class="form-label">status:</label>
+ <span class="form-text">{{job_status}}: {{message}}</span><br />
+ </div>
+
+ <div class="form-group">
+ <label for="job_{{job_id}}" class="form-label">parsing: </label>
+ <progress id="job_{{job_id}}"
+ value="{{progress/100}}"
+ class="form-control">
+ {{progress}}</progress>
+ <span class="form-text text-muted">{{"%.2f" | format(progress)}}%</span>
+ </div>
+
+ <input type="hidden" name="job_id" value="{{job_id}}" />
+
+ <button type="submit" class="btn btn-danger">Abort</button>
+ </form>
+</div>
+
+<div class="row">
+ {{errors_display(errors, "No errors found so far", "We have found the following errors so far", False)}}
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/expression-data/no-such-job.html b/uploader/templates/expression-data/no-such-job.html
new file mode 100644
index 0000000..d22c429
--- /dev/null
+++ b/uploader/templates/expression-data/no-such-job.html
@@ -0,0 +1,15 @@
+{%extends "base.html"%}
+
+{%block extrameta%}
+<meta http-equiv="refresh"
+ content="5;url={{url_for('species.populations.expression-data.index.upload_file')}}">
+{%endblock%}
+
+{%block title%}No Such Job{%endblock%}
+
+{%block contents%}
+<h1 class="heading">No Such Job: {{job_id}}</h2>
+
+<p>No job, with the id '<em>{{job_id}}</em>' was found!</p>
+
+{%endblock%}
diff --git a/uploader/templates/expression-data/parse-failure.html b/uploader/templates/expression-data/parse-failure.html
new file mode 100644
index 0000000..31f6be8
--- /dev/null
+++ b/uploader/templates/expression-data/parse-failure.html
@@ -0,0 +1,26 @@
+{%extends "base.html"%}
+
+{%block title%}Worker Failure{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Worker Failure</h1>
+
+<p>
+ There was an error while parsing your file.
+</p>
+
+<p>
+ Please notify the developers of this issue when you encounter it,
+ providing the information below.
+</p>
+
+<h4>Debugging Information</h4>
+
+<ul>
+ <li><strong>job id</strong>: {{job["job_id"]}}</li>
+ <li><strong>filename</strong>: {{job["filename"]}}</li>
+ <li><strong>line number</strong>: {{job["line_number"]}}</li>
+ <li><strong>Progress</strong>: {{job["percent"]}} %</li>
+</ul>
+
+{%endblock%}
diff --git a/uploader/templates/expression-data/parse-results.html b/uploader/templates/expression-data/parse-results.html
new file mode 100644
index 0000000..03a23e2
--- /dev/null
+++ b/uploader/templates/expression-data/parse-results.html
@@ -0,0 +1,39 @@
+{%extends "base.html"%}
+{%from "errors_display.html" import errors_display%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Parse Results{%endblock%}
+
+{%block contents%}
+
+<div class="row">
+ <h2 class="heading">{{job_name}}: parse results</h2>
+
+ {%if user_aborted%}
+ <span class="alert-warning">Job aborted by the user</span>
+ {%endif%}
+
+ {{errors_display(errors, "No errors found in the file", "We found the following errors", True)}}
+
+ {%if errors | length == 0 and not user_aborted %}
+ <form method="post" action="{{url_for('dbinsert.select_platform')}}">
+ <input type="hidden" name="job_id" value="{{job_id}}" />
+ <input type="submit" value="update database" class="btn btn-primary" />
+ </form>
+ {%endif%}
+
+ {%if errors | length > 0 or user_aborted %}
+ <br />
+ <a href="{{url_for('species.populations.expression-data.upload_file',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ title="Back to index page."
+ class="btn btn-primary">Go back</a>
+
+ {%endif%}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/expression-data/select-file.html b/uploader/templates/expression-data/select-file.html
new file mode 100644
index 0000000..4ca461e
--- /dev/null
+++ b/uploader/templates/expression-data/select-file.html
@@ -0,0 +1,115 @@
+{%extends "expression-data/base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+{%from "upload_progress_indicator.html" import upload_progress_indicator%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Expression Data &mdash; Upload Data{%endblock%}
+
+{%block pagetitle%}Expression Data &mdash; Upload Data{%endblock%}
+
+{%block contents%}
+{{upload_progress_indicator()}}
+
+<div class="row">
+ <h2 class="heading">Upload Expression Data</h2>
+
+ <p>This feature enables you to upload expression data. It expects the data to
+ be in <strong>tab-separated values (TSV)</strong> files. The data should be
+ a simple matrix of <em>phenotype × sample</em>, i.e. The first column is a
+ list of the <em>phenotypes</em> and the first row is a list of
+ <em>samples/cases</em>.</p>
+
+ <p>If you haven't done so please go to this page to learn the requirements for
+ file formats and helpful suggestions to enter your data in a fast and easy
+ way.</p>
+
+ <ol>
+ <li><strong>PLEASE REVIEW YOUR DATA.</strong>Make sure your data complies
+ with our system requirements. (
+ <a href="{{url_for('species.populations.expression-data.data_review')}}#data-concerns"
+ title="Details for the data expectations.">Help</a>
+ )</li>
+ <li><strong>UPLOAD YOUR DATA FOR DATA VERIFICATION.</strong> We accept
+ <strong>.csv</strong>, <strong>.txt</strong> and <strong>.zip</strong>
+ files (<a href="{{url_for('species.populations.expression-data.data_review')}}#file-types"
+ title="Details for the data expectations.">Help</a>)</li>
+ </ol>
+</div>
+
+<div class="row">
+ <form action="{{url_for(
+ 'species.populations.expression-data.upload_file',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ method="POST"
+ enctype="multipart/form-data"
+ id="frm-upload-expression-data">
+ {{flash_messages("error-expr-data")}}
+
+ <div class="form-group">
+ <legend class="heading">File Type</legend>
+
+ <div class="radio">
+ <label for="filetype_average" class="form-check-label">
+ <input type="radio" name="filetype" value="average" id="filetype_average"
+ required="required" class="form-check-input" />
+ Average</label>
+ <p class="form-text text-muted">
+ <small>The averages data …</small></p>
+ </div>
+
+ <div class="radio">
+ <label for="filetype_standard_error" class="form-check-label">
+ <input type="radio" name="filetype" value="standard-error"
+ id="filetype_standard_error" required="required"
+ class="form-check-input" />
+ Standard Error
+ </label>
+ <p class="form-text text-muted">
+ <small>The standard errors computed from the averages …</small></p>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <span id="no-file-error" class="alert-danger" style="display: none;">
+ No file selected
+ </span>
+ <label for="file_upload" class="form-label">Select File</label>
+ <input type="file" name="qc_text_file" id="file_upload"
+ accept="text/plain, text/tab-separated-values, application/zip"
+ class="form-control"/>
+ <p class="form-text text-muted">
+ <small>Select the file to upload.</small></p>
+ </div>
+
+ <button type="submit"
+ class="btn btn-primary"
+ data-toggle="modal"
+ data-target="#upload-progress-indicator">upload file</button>
+ </form>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/upload_progress.js"></script>
+<script type="text/javascript">
+ function setup_formdata(form) {
+ var formdata = new FormData();
+ formdata.append(
+ "qc_text_file",
+ form.querySelector("input[type='file']").files[0]);
+ formdata.append(
+ "filetype",
+ selected_filetype(
+ Array.from(form.querySelectorAll("input[type='radio']"))));
+ return formdata;
+ }
+
+ setup_upload_handlers(
+ "frm-upload-expression-data", make_data_uploader(setup_formdata));
+</script>
+{%endblock%}
diff --git a/uploader/templates/expression-data/select-population.html b/uploader/templates/expression-data/select-population.html
new file mode 100644
index 0000000..8555e27
--- /dev/null
+++ b/uploader/templates/expression-data/select-population.html
@@ -0,0 +1,29 @@
+{%extends "expression-data/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+{%from "populations/macro-select-population.html" import select_population_form%}
+
+{%block title%}Expression Data{%endblock%}
+
+{%block pagetitle%}Expression Data{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <p>You have selected the species. Now you need to select the population that
+ the expression data belongs to.</p>
+</div>
+
+<div class="row">
+ {{select_population_form(url_for(
+ "species.populations.expression-data.select_population",
+ species_id=species.SpeciesId),
+ populations)}}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
diff --git a/uploader/templates/final_confirmation.html b/uploader/templates/final_confirmation.html
new file mode 100644
index 0000000..0727fc8
--- /dev/null
+++ b/uploader/templates/final_confirmation.html
@@ -0,0 +1,47 @@
+{%extends "base.html"%}
+{%from "dbupdate_hidden_fields.html" import hidden_fields%}
+
+{%block title%}Confirmation{%endblock%}
+
+{%macro display_item(item_name, item_data):%}
+<li>
+ <strong>{{item_name}}</strong>
+ {%if item_data%}
+ <ul>
+ {%for term,value in item_data.items():%}
+ <li><strong>{{term}}:</strong> {{value}}</li>
+ {%endfor%}
+ </ul>
+ {%endif%}
+</li>
+{%endmacro%}
+
+{%block contents%}
+<h2 class="heading">Final Confirmation</h2>
+
+<div class="two-col-sep-col1">
+ <p><strong>Selected Data</strong></p>
+ <ul>
+ <li><strong>File</strong>
+ <ul>
+ <li><strong>Filename</strong>: {{filename}}</li>
+ <li><strong>File Type</strong>: {{filetype}}</li>
+ </ul>
+ </li>
+ {{display_item("Species", the_species)}}
+ {{display_item("Platform", platform)}}
+ {{display_item("Study", study)}}
+ {{display_item("Dataset", dataset)}}
+ </ul>
+</div>
+
+<form method="POST" action="{{url_for('dbinsert.insert_data')}}">
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid,datasetid=datasetid, totallines=totallines)}}
+ <fieldset>
+ <input type="submit" class="btn btn-primary" value="confirm" />
+ </fieldset>
+</form>
+</div>
+{%endblock%}
diff --git a/uploader/templates/flash_messages.html b/uploader/templates/flash_messages.html
new file mode 100644
index 0000000..b7af178
--- /dev/null
+++ b/uploader/templates/flash_messages.html
@@ -0,0 +1,25 @@
+{%macro flash_all_messages()%}
+{%with messages = get_flashed_messages(with_categories=true)%}
+{%if messages:%}
+<ul>
+ {%for category, message in messages:%}
+ <li class="{{category}}">{{message}}</li>
+ {%endfor%}
+</ul>
+{%endif%}
+{%endwith%}
+{%endmacro%}
+
+{%macro flash_messages(filter_class)%}
+{%with messages = get_flashed_messages(with_categories=true)%}
+{%if messages:%}
+<ul>
+ {%for category, message in messages:%}
+ {%if filter_class in category%}
+ <li class="{{category}}">{{message}}</li>
+ {%endif%}
+ {%endfor%}
+</ul>
+{%endif%}
+{%endwith%}
+{%endmacro%}
diff --git a/uploader/templates/genotypes/base.html b/uploader/templates/genotypes/base.html
new file mode 100644
index 0000000..7d61312
--- /dev/null
+++ b/uploader/templates/genotypes/base.html
@@ -0,0 +1,23 @@
+{%extends "populations/base.html"%}
+
+{%block lvl3_breadcrumbs%}
+<li {%if activelink=="genotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ {%if population is mapping%}
+ <a href="{{url_for('species.populations.genotypes.list_genotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">
+ {%if dataset is defined and dataset is mapping%}
+ {{dataset.Name}}
+ {%else%}
+ Genotypes
+ {%endif%}</a>
+ {%else%}
+ <a href="{{url_for('species.populations.genotypes.index')}}">Genotypes</a>
+ {%endif%}
+</li>
+{%block lvl4_breadcrumbs%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/genotypes/create-dataset.html b/uploader/templates/genotypes/create-dataset.html
new file mode 100644
index 0000000..10331c1
--- /dev/null
+++ b/uploader/templates/genotypes/create-dataset.html
@@ -0,0 +1,82 @@
+{%extends "genotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Genotypes — Create Dataset{%endblock%}
+
+{%block pagetitle%}Genotypes — Create Dataset{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="create-dataset"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.genotypes.create_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">Create Dataset</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <form id="frm-geno-create-dataset"
+ method="POST"
+ action="{{url_for('species.populations.genotypes.create_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">
+ <legend>Create a new Genotype Dataset</legend>
+
+ <div class="form-group">
+ <label for="txt-geno-dataset-name" class="form-label">Name</label>
+ <input type="text"
+ id="txt-geno-dataset-name"
+ name="geno-dataset-name"
+ required="required"
+ class="form-control" />
+ <small class="form-text text-muted">
+ <p>This is a short representative, but constrained name for the genotype
+ dataset.<br />
+ The field will only accept letters ('A-Za-z'), numbers (0-9), hyphens
+ and underscores. Any other character will cause the name to be
+ rejected.</p></small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-geno-dataset-fullname" class="form-label">Full Name</label>
+ <input type="text"
+ id="txt-geno-dataset-fullname"
+ name="geno-dataset-fullname"
+ required="required"
+ class="form-control" />
+ <small class="form-text text-muted">
+ <p>This is a longer, more descriptive name for your dataset.</p></small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-geno-dataset-shortname"
+ class="form-label">Short Name</label>
+ <input type="text"
+ id="txt-geno-dataset-shortname"
+ name="geno-dataset-shortname"
+ class="form-control" />
+ <small class="form-text text-muted">
+ <p>A short name for your dataset. If you leave this field blank, the
+ short name will be set to the same value as the
+ "<strong>Name</strong>" field above.</p></small>
+ </div>
+
+ <div class="form-group">
+ <input type="submit"
+ class="btn btn-primary"
+ value="create dataset" />
+ </div>
+ </form>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/genotypes/index.html b/uploader/templates/genotypes/index.html
new file mode 100644
index 0000000..b50ebc5
--- /dev/null
+++ b/uploader/templates/genotypes/index.html
@@ -0,0 +1,32 @@
+{%extends "genotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-select-species.html" import select_species_form%}
+
+{%block title%}Genotypes{%endblock%}
+
+{%block pagetitle%}Genotypes{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <p>
+ This section allows you to upload genotype information for your experiments,
+ in the case that you have not previously done so.
+ </p>
+ <p>
+ We'll need to link the genotypes to the species and population, so do please
+ go ahead and select those in the next two steps.
+ </p>
+</div>
+
+<div class="row">
+ {{select_species_form(url_for("species.populations.genotypes.index"),
+ species)}}
+</div>
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/species.js"></script>
+{%endblock%}
diff --git a/uploader/templates/genotypes/list-genotypes.html b/uploader/templates/genotypes/list-genotypes.html
new file mode 100644
index 0000000..0f074fd
--- /dev/null
+++ b/uploader/templates/genotypes/list-genotypes.html
@@ -0,0 +1,149 @@
+{%extends "genotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Genotypes{%endblock%}
+
+{%block pagetitle%}Genotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="list-genotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.genotypes.list_genotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">List genotypes</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <h2>Genetic Markers</h2>
+ <p>There are a total of {{total_markers}} currently registered genetic markers
+ for the "{{species.FullName}}" species. You can click
+ <a href="{{url_for('species.populations.genotypes.list_markers',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ title="View genetic markers for species '{{species.FullName}}">
+ this link to view the genetic markers
+ </a>.
+ </p>
+</div>
+
+<div class="row">
+ <h2>Genotype Encoding</h2>
+ <p>
+ The genotype encoding used for the "{{population.FullName}}" population from
+ the "{{species.FullName}}" species is as shown in the table below.
+ </p>
+ <table class="table">
+
+ <thead>
+ <tr>
+ <th>Allele Type</th>
+ <th>Allele Symbol</th>
+ <th>Allele Value</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for row in genocode%}
+ <tr>
+ <td>{{row.AlleleType}}</td>
+ <td>{{row.AlleleSymbol}}</td>
+ <td>{{row.DatabaseValue if row.DatabaseValue is not none else "NULL"}}</td>
+ </tr>
+ {%else%}
+ <tr>
+ <td colspan="7" class="text-info">
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ There is no explicit genotype encoding defined for this population.
+ </td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+
+ {%if genocode | length < 1%}
+ <a href="#add-genotype-encoding"
+ title="Add a genotype encoding system for this population"
+ class="btn btn-primary not-implemented">
+ add genotype encoding
+ </a>
+ {%endif%}
+</div>
+
+<div class="row text-danger">
+ <h3>Some Important Concepts to Consider/Remember</h3>
+ <ul>
+ <li>Reference vs. Non-reference alleles</li>
+ <li>In <em>GenoCode</em> table, items are ordered by <strong>InbredSet</strong></li>
+ </ul>
+ <h3>Possible references</h3>
+ <ul>
+ <li>https://mr-dictionary.mrcieu.ac.uk/term/genotype/</li>
+ <li>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7363099/</li>
+ </ul>
+</div>
+
+<div class="row">
+ <h2>Genotype Datasets</h2>
+
+ <p>The genotype data is organised under various genotype datasets. You can
+ click on the link for the relevant dataset to view a little more information
+ about it.</p>
+
+ {%if dataset is not none%}
+ <table class="table">
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Full Name</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>{{dataset.Name}}</td>
+ <td><a href="{{url_for('species.populations.genotypes.view_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}"
+ title="View details regarding and manage dataset '{{dataset.FullName}}'">
+ {{dataset.FullName}}</a></td>
+ </tr>
+ </tbody>
+ </table>
+ {%else%}
+ <p class="text-warning">
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ There is no genotype dataset defined for this population.
+ </p>
+ <p>
+ <a href="{{url_for('species.populations.genotypes.create_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ title="Create a new genotype dataset for the '{{population.FullName}}' population for the '{{species.FullName}}' species."
+ class="btn btn-primary">
+ create new genotype dataset</a></p>
+ {%endif%}
+</div>
+<div class="row text-warning">
+ <p>
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ <strong>NOTE</strong>: Currently the GN2 (and related) system(s) expect a
+ single genotype dataset. If there is more than one, the system apparently
+ fails in unpredictable ways.
+ </p>
+ <p>Fix this to allow multiple datasets, each with a different assembly from
+ all the rest.</p>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/genotypes/list-markers.html b/uploader/templates/genotypes/list-markers.html
new file mode 100644
index 0000000..a705ae3
--- /dev/null
+++ b/uploader/templates/genotypes/list-markers.html
@@ -0,0 +1,105 @@
+{%extends "genotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+
+{%block title%}Genotypes: List Markers{%endblock%}
+
+{%block pagetitle%}Genotypes: List Markers{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="list-markers"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.genotypes.list_markers',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">List markers</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+{%if markers | length > 0%}
+<div class="row">
+ <p>
+ There are a total of {{total_markers}} genotype markers for this species.
+ </p>
+ <div class="row">
+ <div class="col-md-2" style="text-align: start;">
+ {%if start_from > 0%}
+ <a href="{{url_for('species.populations.genotypes.list_markers',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ start_from=start_from-count,
+ count=count)}}">
+ <span class="glyphicon glyphicon-backward"></span>
+ Previous
+ </a>
+ {%endif%}
+ </div>
+ <div class="col-md-8" style="text-align: center;">
+ Displaying markers {{start_from+1}} to {{start_from+count if start_from+count < total_markers else total_markers}} of
+ {{total_markers}}
+ </div>
+ <div class="col-md-2" style="text-align: end;">
+ {%if start_from + count < total_markers%}
+ <a href="{{url_for('species.populations.genotypes.list_markers',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ start_from=start_from+count,
+ count=count)}}">
+ Next
+ <span class="glyphicon glyphicon-forward"></span>
+ </a>
+ {%endif%}
+ </div>
+ </div>
+ <table class="table">
+ <thead>
+ <tr>
+ <th title="">#</th>
+ <th title="">Marker Name</th>
+ <th title="Chromosome">Chr</th>
+ <th title="Physical location of the marker in megabasepairs">
+ Location (Mb)</th>
+ <th title="">Source</th>
+ <th title="">Source2</th>
+ </thead>
+
+ <tbody>
+ {%for marker in markers%}
+ <tr>
+ <td>{{marker.sequence_number}}</td>
+ <td>{{marker.Marker_Name}}</td>
+ <td>{{marker.Chr}}</td>
+ <td>{{marker.Mb}}</td>
+ <td>{{marker.Source}}</td>
+ <td>{{marker.Source2}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+</div>
+{%else%}
+<div class="row">
+ <p class="text-warning">
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ This species does not currently have any genetic markers uploaded, therefore,
+ there is nothing to display here.
+ </p>
+ <p>
+ <a href="#add-genetic-markers-for-species-{{species.SpeciesId}}"
+ title="Add genetic markers for this species"
+ class="btn btn-primary">
+ add genetic markers
+ </a>
+ </p>
+</div>
+{%endif%}
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
diff --git a/uploader/templates/genotypes/select-population.html b/uploader/templates/genotypes/select-population.html
new file mode 100644
index 0000000..acdd063
--- /dev/null
+++ b/uploader/templates/genotypes/select-population.html
@@ -0,0 +1,25 @@
+{%extends "genotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+{%from "populations/macro-select-population.html" import select_population_form%}
+
+{%block title%}Genotypes{%endblock%}
+
+{%block pagetitle%}Genotypes{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ {{select_population_form(url_for("species.populations.genotypes.select_population", species_id=species.SpeciesId), species, populations)}}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/populations.js"></script>
+{%endblock%}
diff --git a/uploader/templates/genotypes/view-dataset.html b/uploader/templates/genotypes/view-dataset.html
new file mode 100644
index 0000000..e7ceb36
--- /dev/null
+++ b/uploader/templates/genotypes/view-dataset.html
@@ -0,0 +1,61 @@
+{%extends "genotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Genotypes: View Dataset{%endblock%}
+
+{%block pagetitle%}Genotypes: View Dataset{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="view-dataset"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.genotypes.view_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">view dataset</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <h2>Genotype Dataset Details</h2>
+ <table class="table">
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Full Name</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>{{dataset.Name}}</td>
+ <td>{{dataset.FullName}}</td>
+ </tr>
+ </tbody>
+ </table>
+</div>
+
+<div class="row text-warning">
+ <h2>Assembly Details</h2>
+
+ <p>Maybe include the assembly details here if found to be necessary.</p>
+</div>
+
+<div class="row">
+ <h2>Genotype Data</h2>
+
+ <p class="text-danger">
+ Provide link to enable uploading of genotype data here.</p>
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/http-error.html b/uploader/templates/http-error.html
new file mode 100644
index 0000000..374fb86
--- /dev/null
+++ b/uploader/templates/http-error.html
@@ -0,0 +1,18 @@
+{%extends "base.html"%}
+
+{%block title%}HTTP Error: {{exc.code}}{%endblock%}
+
+{%block contents%}
+<h1>{{exc.code}}: {{exc.description}}</h1>
+
+<div class="row">
+ <p>
+ You attempted to access {{request_url}} which failed with the following
+ error:
+ </p>
+</div>
+
+<div class="row">
+ <pre>{{"\n".join(trace)}}</pre>
+</div>
+{%endblock%}
diff --git a/uploader/templates/index.html b/uploader/templates/index.html
new file mode 100644
index 0000000..aa1414e
--- /dev/null
+++ b/uploader/templates/index.html
@@ -0,0 +1,107 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Home{%endblock%}
+
+{%block pagetitle%}Home{%endblock%}
+
+{%block contents%}
+
+<div class="row">
+ {{flash_all_messages()}}
+ <div class="explainer">
+ <p>Welcome to the <strong>GeneNetwork Data Upload and Quality Control
+ System</strong>.</p>
+ <p>This tool helps you prepare and upload research data to GeneNetwork for
+ analysis.</p>
+
+ <h2 class="heading">Getting Started</h2>
+ <p>The sections below explain the features of the system. Review this guide
+ to learn how to use the system.</p>
+
+ {%block extrapageinfo%}{%endblock%}
+
+ <h3 class="subheading">Species</h3>
+
+ <p>GeneNetwork supports genetic studies across multiple species (e.g. mice
+ [Mus musculus], human [homo sapiens], rats [Rattus norvegicus], etc.) .
+ Here you can:</p>
+ <ul>
+ <li>View all species that are currently supported</li>
+ <li>Add new species not yet in the system</li>
+ </ul>
+
+ <h3 class="subheading">Populations</h3>
+
+ <p>A "population" refers to a specific subgroup within a species that you’re
+ studying (e.g., BXD mice). Here you can:</p>
+ <ul>
+ <li>View the populations that exist for a selected species</li>
+ <li>Add new populations of study for a selected species</li>
+ </ul>
+
+ <h3 class="subheading">Samples</h3>
+
+ <p>Manage individual specimens or cases used in your experiments. These
+ include:</p>
+
+ <ul>
+ <li>Experimental subjects</li>
+ <li>Data sources (e.g., tissue samples, clinical cases)</li>
+ <li>Strain means (instead of entering multiple BXD1 individuals, for
+ example, the mean would be entered for a single BXD1 strain)</li>
+ </ul>
+
+
+ <h3 class="subheading">Genotype Data</h3>
+
+ <p>Upload and review genetic markers and allele encodings for your
+ population. Key details:</p>
+
+ <ul>
+ <li>Markers are species-level (e.g., mouse SNP databases).</li>
+ <li>Allele data is population-specific (tied to your experimental
+ samples).</li>
+ </ul>
+
+ <p><strong>Requirement</strong>: Samples must already have been registered
+ in the system before uploading genotype data.</p>
+
+ <h3 class="subheading">Phenotype Data</h3>
+
+ <p>Phenotypes are the visible traits or features of a living thing. For
+ example, phenotypes include:</p>
+
+ <ul>
+ <li>Weight</li>
+ <li>Height</li>
+ <li>Color (such as the color of fur or eyes)</li>
+ </ul>
+
+ <p>This part of the system will allow you to upload and manage the values
+ for different phenotypes from various samples in your studies.</p>
+
+ <!--
+
+ <h3 class="subheading">Expression Data</h3>
+
+ <p class="text-danger">
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ <strong>TODO</strong>: Document this &hellip;</p>
+
+ <h3 class="subheading">Individual Data</h3>
+
+ <p class="text-danger">
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ <strong>TODO</strong>: Document this &hellip;</p>
+
+ <h3 class="subheading">RNA-Seq Data</h3>
+
+ <p class="text-danger">
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ <strong>TODO</strong>: Document this &hellip;</p>
+ </div>
+ -->
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/insert_error.html b/uploader/templates/insert_error.html
new file mode 100644
index 0000000..5301288
--- /dev/null
+++ b/uploader/templates/insert_error.html
@@ -0,0 +1,32 @@
+{%extends "base.html"%}
+
+{%block title%}Data Insertion Failure{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Insertion Failure</h1>
+
+<div class="row">
+ <p>
+ There was an error inserting data into the database
+ </p>
+
+ <p>
+ Please notify the developers of this issue when you encounter it,
+ providing the information below.
+ </p>
+
+ <h4>Debugging Information</h4>
+
+ <ul>
+ <li><strong>job id</strong>: {{job["jobid"]}}</li>
+ </ul>
+</div>
+
+<div class="row">
+ <h4>STDERR Output</h4>
+ <pre class="cli-output">
+ {{job["stderr"]}}
+ </pre>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/insert_progress.html b/uploader/templates/insert_progress.html
new file mode 100644
index 0000000..52177d6
--- /dev/null
+++ b/uploader/templates/insert_progress.html
@@ -0,0 +1,46 @@
+{%extends "base.html"%}
+{%from "stdout_output.html" import stdout_output%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="5">
+{%endblock%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job_name}}</h1>
+
+<div class="row">
+ <form>
+ <div class="form-group">
+ <label for="job_status" class="form-label">status:</label>
+ <span class="form-text">{{job_status}}: {{message}}</span>
+ </div>
+
+{%if job.get("stdout", "").split("\n\n") | length < 3 %}
+{%set lines = 0%}
+{%else%}
+{%set lines = (job.get("stdout", "").split("\n\n") | length / 3) %}
+{%endif%}
+{%set totallines = job.get("totallines", lines+3) | int %}
+{%if totallines > 1000 %}
+{%set fraction = ((lines*1000)/totallines) %}
+{%else%}
+{%set fraction = (lines/totallines)%}
+{%endif%}
+
+ <div class="form-group">
+ <label for="job_{{job_id}}" class="form-label">inserting: </label>
+ <progress id="jobs_{{job_id}}"
+ value="{{(fraction)}}"
+ class="form-control">{{fraction*100}}</progress>
+ <span class="form-text text-muted">
+ {{"%.2f" | format(fraction * 100 | float)}}%</span>
+ </div>
+ </form>
+</div>
+
+
+{{stdout_output(job)}}
+
+{%endblock%}
diff --git a/uploader/templates/insert_success.html b/uploader/templates/insert_success.html
new file mode 100644
index 0000000..7e1fa8d
--- /dev/null
+++ b/uploader/templates/insert_success.html
@@ -0,0 +1,19 @@
+{%extends "base.html"%}
+{%from "stdout_output.html" import stdout_output%}
+
+{%block title%}Insertion Success{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Insertion Success</h1>
+
+<div class="row">
+<p>Data inserted successfully!</p>
+
+<p>The following queries were run:</p>
+</div>
+
+<div class="row">
+ {{stdout_output(job)}}
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/login.html b/uploader/templates/login.html
new file mode 100644
index 0000000..e76c644
--- /dev/null
+++ b/uploader/templates/login.html
@@ -0,0 +1,12 @@
+{%extends "index.html"%}
+
+{%block title%}Data Upload{%endblock%}
+
+{%block pagetitle%}log in{%endblock%}
+
+{%block extrapageinfo%}
+<p class="text-dark">
+ You <strong>need to
+ <a href="{{authserver_authorise_uri()}}"
+ title="Sign in to the system">sign in</a></strong> to use this system.</p>
+{%endblock%}
diff --git a/uploader/templates/macro-table-pagination.html b/uploader/templates/macro-table-pagination.html
new file mode 100644
index 0000000..292c531
--- /dev/null
+++ b/uploader/templates/macro-table-pagination.html
@@ -0,0 +1,26 @@
+{%macro table_pagination(start_at, page_count, total_count, base_uri, name)%}
+{%set ns = namespace(forward_uri=base_uri, back_uri=base_uri)%}
+{%set ns.forward_uri="brr"%}
+ <div class="row">
+ <div class="col-md-2" style="text-align: start;">
+ {%if start_at > 0%}
+ <a href="{{base_uri +
+ '?start_at='+((start_at-page_count)|string) +
+ '&count='+(page_count|string)}}">
+ <span class="glyphicon glyphicon-backward"></span>
+ Previous
+ </a>
+ {%endif%}
+ </div>
+ <div class="col-md-8" style="text-align: center;">
+ Displaying {{name}} {{start_at+1}} to {{start_at+page_count if start_at+page_count < total_count else total_count}} of {{total_count}}</div>
+ <div class="col-md-2" style="text-align: end;">
+ {%if start_at + page_count < total_count%}
+ <a href="{{base_uri +
+ '?start_at='+((start_at+page_count)|string) +
+ '&count='+(page_count|string)}}">
+ Next<span class="glyphicon glyphicon-forward"></span></a>
+ {%endif%}
+ </div>
+ </div>
+{%endmacro%}
diff --git a/uploader/templates/phenotypes/add-phenotypes-base.html b/uploader/templates/phenotypes/add-phenotypes-base.html
new file mode 100644
index 0000000..97b55f2
--- /dev/null
+++ b/uploader/templates/phenotypes/add-phenotypes-base.html
@@ -0,0 +1,331 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="add-phenotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">Add Phenotypes</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <form id="frm-add-phenotypes"
+ method="POST"
+ enctype="multipart/form-data"
+ action="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ use_bundle=use_bundle)}}"
+ data-resumable-target="{{url_for('files.resumable_upload_post')}}">
+ <legend>Add New Phenotypes</legend>
+
+ <div class="form-text help-block">
+ {%block frm_add_phenotypes_documentation%}{%endblock%}
+ <p><strong class="text-warning">This will not update any existing phenotypes!</strong></p>
+ </div>
+
+ {%block frm_add_phenotypes_elements%}{%endblock%}
+
+ <div class="checkbox">
+ <label>
+ <input id="chk-published" type="checkbox" name="published?" />
+ These phenotypes are published</label>
+ </div>
+
+ <fieldset id="fldset-publication-info" class="hidden">
+ <legend>Publication Information</legend>
+ <div class="form-group">
+ <label for="txt-pubmed-id" class="form-label">Pubmed ID</label>
+ <div class="input-group">
+ <input id="txt-pubmed-id" name="pubmed-id" type="text"
+ class="form-control" />
+ <span class="input-group-btn">
+ <button id="btn-search-pubmed-id" class="btn btn-info">Search</button>
+ </span>
+ </div>
+ <span id="search-pubmed-id-error"
+ class="form-text text-muted text-danger hidden">
+ </span><br />
+ <span class="form-text text-muted">
+ Enter your publication's PubMed ID above and click "Search" to search
+ for some (or all) of the publication details requested below.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-authors" class="form-label">Authors</label>
+ <input id="txt-publication-authors" name="publication-authors"
+ type="text" class="form-control" />
+ <span class="form-text text-muted">
+ Enter the authors in the following format &hellip;</span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-title" class="form-label">
+ Publication Title</label>
+ <input id="txt-publication-title" name="publication-title" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter your publication's title.</span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-abstract" class="form-label">
+ Publication Abstract</label>
+ <textarea id="txt-publication-abstract" name="publication-abstract"
+ class="form-control" rows="10"></textarea>
+ <span class="form-text text-muted">
+ Enter the abstract for your publication.</span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-journal" class="form-label">Journal</label>
+ <input id="txt-publication-journal" name="journal" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter the name of the journal where your work was published.</span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-volume" class="form-label">Volume</label>
+ <input id="txt-publication-volume" name="publication-volume" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter the volume in the following format &hellip;</span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-pages" class="form-label">Pages</label>
+ <input id="txt-publication-pages" name="publication-pages" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter the journal volume where your work was published.</span>
+ </div>
+
+ <div class="form-group">
+ <label for="select-publication-month" class="form-label">
+ Publication Month</label>
+ <select id="select-publication-month" name="publication-month"
+ class="form-control">
+ {%for month in monthnames%}
+ <option value="{{month | lower}}"
+ {%if current_month | lower == month | lower%}
+ selected="selected"
+ {%endif%}>{{month | capitalize}}</option>
+ {%endfor%}
+ </select>
+ <span class="form-text text-muted">
+ Select the month when the work was published.
+ <span class="text-danger">
+ This cannot be before, say 1600 and cannot be in the future!</span></span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-year" class="form-label">Publication Year</label>
+ <input id="txt-publication-year" name="publication-year" type="text"
+ class="form-control" value="{{current_year}}" />
+ <span class="form-text text-muted">
+ Enter the year your work was published.
+ <span class="text-danger">
+ This cannot be before, say 1600 and cannot be in the future!</span>
+ </span>
+ </div>
+ </fieldset>
+
+ <div class="form-group">
+ <input type="submit"
+ value="upload phenotypes"
+ class="btn btn-primary" />
+ </div>
+ </form>
+</div>
+
+<div class="row">
+ {%block page_documentation%}{%endblock%}
+</div>
+
+{%endblock%}
+
+
+{%block javascript%}
+<script type="text/javascript">
+ var remove_class = (element, classvalue) => {
+ new_classes = (element.attr("class") || "").split(" ").map((val) => {
+ return val.trim();
+ }).filter((val) => {
+ return ((val !== classvalue) &&
+ (val !== ""))
+ }).join(" ");
+
+ if(new_classes === "") {
+ element.removeAttr("class");
+ } else {
+ element.attr("class", new_classes);
+ }
+ };
+
+ var add_class = (element, classvalue) => {
+ remove_class(element, classvalue);
+ element.attr("class", (element.attr("class") || "") + " " + classvalue);
+ };
+
+ $("#chk-published").on("click", (event) => {
+ pub_details = $("#fldset-publication-info")
+ if(event.target.checked) {
+ // display the publication details
+ remove_class(pub_details, "hidden");
+ } else {
+ // hide the publication details
+ add_class(pub_details, "hidden");
+ }
+ });
+
+ var extract_details = (pubmed_id, details) => {
+ var months = {
+ "jan": "January",
+ "feb": "February",
+ "mar": "March",
+ "apr": "April",
+ "may": "May",
+ "jun": "June",
+ "jul": "July",
+ "aug": "August",
+ "sep": "September",
+ "oct": "October",
+ "nov": "November",
+ "dec": "December"
+ };
+ var _date = details[pubmed_id].pubdate.split(" ");
+ return {
+ "authors": details[pubmed_id].authors.map((authobj) => {
+ return authobj.name;
+ }),
+ "title": details[pubmed_id].title,
+ "journal": details[pubmed_id].fulljournalname,
+ "volume": details[pubmed_id].volume,
+ "pages": details[pubmed_id].pages,
+ "month": _date.length > 1 ? months[_date[1].toLowerCase()] : "jan",
+ "year": _date[0],
+ };
+ };
+
+ var update_publication_details = (details) => {
+ Object.entries(details).forEach((entry) => {;
+ switch(entry[0]) {
+ case "authors":
+ $("#txt-publication-authors").val(entry[1].join(", "));
+ break;
+ case "month":
+ $("#select-publication-month")
+ .children("option")
+ .each((index, child) => {
+ child.selected = child.value == entry[1].toLowerCase();
+ });
+ default:
+ $("#txt-publication-" + entry[0]).val(entry[1]);
+ break;
+ }
+ });
+ };
+
+ var fetch_publication_abstract = (pubmed_id, pub_details) => {
+ $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi",
+ {
+ "method": "GET",
+ "data": {
+ "db": "pubmed",
+ "id": pubmed_id,
+ "rettype": "abstract",
+ "retmode": "xml"
+ },
+ "success": (data, textStatus, jqXHR) => {
+ update_publication_details({
+ ...pub_details,
+ ...{
+ "abstract": Array.from(data
+ .getElementsByTagName(
+ "Abstract")[0]
+ .children)
+ .map((elt) => {return elt.textContent.trim();})
+ .join("\r\n")
+ }});
+ },
+ "error": (jqXHR, textStatus, errorThrown) => {},
+ "complete": (jqXHR, textStatus) => {},
+ "dataType": "xml"
+ });
+ };
+
+ var fetch_publication_details = (pubmed_id, complete_thunks) => {
+ error_display = $("#search-pubmed-id-error");
+ error_display.text("");
+ add_class(error_display, "hidden");
+ $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi",
+ {
+ "method": "GET",
+ "data": {"db": "pubmed", "id": pubmed_id, "format": "json"},
+ "success": (data, textStatus, jqXHR) => {
+ // process and update publication details
+ hasError = (
+ Object.hasOwn(data, "error") ||
+ Object.hasOwn(data.result[pubmed_id], "error"));
+ if(hasError) {
+ error_display.text(
+ "There was an error fetching a publication with " +
+ "the given PubMed ID! The error received " +
+ "was: '" + (
+ data.error ||
+ data.result[pubmed_id].error) +
+ "'. Please check ID you provided and try " +
+ "again.");
+ remove_class(error_display, "hidden");
+ } else {
+ fetch_publication_abstract(
+ pubmed_id,
+ extract_details(pubmed_id, data.result));
+ }
+ },
+ "error": (jqXHR, textStatus, errorThrown) => {},
+ "complete": () => {
+ complete_thunks.forEach((thunk) => {thunk()});
+ },
+ "dataType": "json"
+ });
+ };
+
+ $("#btn-search-pubmed-id").on("click", (event) => {
+ event.preventDefault();
+ var search_button = event.target;
+ var pubmed_id = $("#txt-pubmed-id").val().trim();
+ remove_class($("#txt-pubmed-id").parent(), "has-error");
+ if(pubmed_id == "") {
+ add_class($("#txt-pubmed-id").parent(), "has-error");
+ return false;
+ }
+
+ search_button.disabled = true;
+ // Fetch publication details
+ fetch_publication_details(pubmed_id,
+ [() => {search_button.disabled = false;}]);
+ return false;
+ });
+</script>
+
+{%block more_javascript%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/add-phenotypes-raw-files.html b/uploader/templates/phenotypes/add-phenotypes-raw-files.html
new file mode 100644
index 0000000..7f8d8b0
--- /dev/null
+++ b/uploader/templates/phenotypes/add-phenotypes-raw-files.html
@@ -0,0 +1,732 @@
+{%extends "phenotypes/add-phenotypes-base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
+{%from "phenotypes/macro-display-preview-table.html" import display_preview_table%}
+{%from "phenotypes/macro-display-resumable-elements.html" import display_resumable_elements%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="add-phenotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">Add Phenotypes</a>
+</li>
+{%endblock%}
+
+{%block frm_add_phenotypes_documentation%}
+<p>This page will allow you to upload all the separate files that make up your
+ phenotypes. Here, you will have to upload each separate file individually. If
+ you want instead to upload all your files as a single ZIP file,
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ use_bundle=true)}}"
+ title="">click here</a>.</p>
+{%endblock%}
+
+{%block frm_add_phenotypes_elements%}
+<fieldset id="fldset-file-metadata">
+ <legend>File(s) Metadata</legend>
+ <div class="form-group">
+ <label for="txt-file-separator" class="form-label">File Separator</label>
+ <div class="input-group">
+ <input id="txt-file-separator"
+ name="file-separator"
+ type="text"
+ value="&#9;"
+ class="form-control"
+ maxlength="1" />
+ <span class="input-group-btn">
+ <button id="btn-reset-file-separator" class="btn btn-info">Reset Default</button>
+ </span>
+ </div>
+ <span class="form-text text-muted">
+ Provide the character that separates the fields in your file(s). It should
+ be the same character for all files (if more than one is provided).<br />
+ A tab character will be assumed if you leave this field blank. See
+ <a href="#docs-file-separator"
+ title="Documentation for file-separator characters">
+ documentation for more information</a>.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-file-comment-character" class="form-label">File Comment-Character</label>
+ <div class="input-group">
+ <input id="txt-file-comment-character"
+ name="file-comment-character"
+ type="text"
+ value="#"
+ class="form-control"
+ maxlength="1" />
+ <span class="input-group-btn">
+ <button id="btn-reset-file-comment-character" class="btn btn-info">
+ Reset Default</button>
+ </span>
+ </div>
+ <span class="form-text text-muted">
+ This specifies that lines that begin with the character provided will be
+ considered comment lines and ignored in their entirety. See
+ <a href="#docs-file-comment-character"
+ title="Documentation for comment characters">
+ documentation for more information</a>.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-file-na" class="form-label">File "No-Value" Indicators</label>
+ <div class="input-group">
+ <input id="txt-file-na"
+ name="file-na"
+ type="text"
+ value="- NA N/A"
+ class="form-control" />
+ <span class="input-group-btn">
+ <button id="btn-reset-file-na" class="btn btn-info">Reset Default</button>
+ </span>
+ </div>
+ <span class="form-text text-muted">
+ This specifies strings in your file indicate that there is no value for a
+ particular cell (a cell is where a column and row intersect). Provide a
+ space-separated list of strings if you have more than one way of
+ indicating no values. See
+ <a href="#docs-file-na" title="Documentation for no-value fields">
+ documentation for more information</a>.</span>
+ </div>
+</fieldset>
+
+<fieldset id="fldset-data-files">
+ <legend>Data File(s)</legend>
+
+ <div class="form-group non-resumable-elements">
+ <label for="finput-phenotype-descriptions" class="form-label">
+ Phenotype Descriptions</label>
+ <input id="finput-phenotype-descriptions"
+ name="phenotype-descriptions"
+ class="form-control"
+ type="file"
+ data-preview-table="tbl-preview-pheno-desc"
+ required="required" />
+ <span class="form-text text-muted">
+ Provide a file that contains only the phenotype descriptions,
+ <a href="#docs-file-phenotype-description"
+ title="Documentation of the phenotype data file format.">
+ the documentation for the expected format of the file</a>.</span>
+ </div>
+
+ {{display_resumable_elements(
+ "resumable-phenotype-descriptions",
+ "phenotype descriptions",
+ '<p>You can drop a CSV file that contains the phenotype descriptions here,
+ or you can click the "Browse" button (below and to the right) to select it
+ from your computer.</p>
+ <p>The CSV file must conform to some standards, as documented in the
+ <a href="#docs-file-phenotype-description"
+ title="Documentation of the phenotype data file format.">
+ "Phenotypes Descriptions" documentation</a> section below.</p>')}}
+
+
+ <div class="form-group non-resumable-elements">
+ <label for="finput-phenotype-data" class="form-label">Phenotype Data</label>
+ <input id="finput-phenotype-data"
+ name="phenotype-data"
+ class="form-control"
+ type="file"
+ data-preview-table="tbl-preview-pheno-data"
+ required="required" />
+ <span class="form-text text-muted">
+ Provide a file that contains only the phenotype data. See
+ <a href="#docs-file-phenotype-data"
+ title="Documentation of the phenotype data file format.">
+ the documentation for the expected format of the file</a>.</span>
+ </div>
+
+ {{display_resumable_elements(
+ "resumable-phenotype-data",
+ "phenotype data",
+ '<p>You can drop a CSV file that contains the phenotype data here,
+ or you can click the "Browse" button (below and to the right) to select it
+ from your computer.</p>
+ <p>The CSV file must conform to some standards, as documented in the
+ <a href="#docs-file-phenotype-data"
+ title="Documentation of the phenotype data file format.">
+ "Phenotypes Data" documentation</a> section below.</p>')}}
+
+ {%if population.Family in families_with_se_and_n%}
+ <div class="form-group non-resumable-elements">
+ <label for="finput-phenotype-se" class="form-label">Phenotype: Standard Errors</label>
+ <input id="finput-phenotype-se"
+ name="phenotype-se"
+ class="form-control"
+ type="file"
+ data-preview-table="tbl-preview-pheno-se"
+ required="required" />
+ <span class="form-text text-muted">
+ Provide a file that contains only the standard errors for the phenotypes,
+ computed from the data above.</span>
+ </div>
+ {{display_resumable_elements(
+ "resumable-phenotype-se",
+ "standard errors",
+ '<p>You can drop a CSV file that contains the computed standard-errors data
+ here, or you can click the "Browse" button (below and to the right) to
+ select it from your computer.</p>
+ <p>The CSV file must conform to some standards, as documented in the
+ <a href="#docs-file-phenotype-se"
+ title="Documentation of the phenotype data file format.">
+ "Phenotypes Data" documentation</a> section below.</p>')}}
+
+
+ <div class="form-group non-resumable-elements">
+ <label for="finput-phenotype-n" class="form-label">Phenotype: Number of Samples/Individuals</label>
+ <input id="finput-phenotype-n"
+ name="phenotype-n"
+ class="form-control"
+ type="file"
+ data-preview-table="tbl-preview-pheno-n"
+ required="required" />
+ <span class="form-text text-muted">
+ Provide a file that contains only the number of samples/individuals used in
+ the computation of the standard errors above.</span>
+ </div>
+ {{display_resumable_elements(
+ "resumable-phenotype-n",
+ "number of samples/individuals",
+ '<p>You can drop a CSV file that contains the number of samples/individuals
+ used in computation of the standard-errors here, or you can click the
+ "Browse" button (below and to the right) to select it from your computer.
+ </p>
+ <p>The CSV file must conform to some standards, as documented in the
+ <a href="#docs-file-phenotype-n"
+ title="Documentation of the phenotype data file format.">
+ "Phenotypes Data" documentation</a> section below.</p>')}}
+</fieldset>
+{%endif%}
+{%endblock%}
+
+
+{%block page_documentation%}
+<div class="row">
+ <h2 class="heading" id="docs-help">Help</h2>
+ <h3 class="subheading">Common Features</h3>
+ <p>The following are the common expectations for <strong>ALL</strong> the
+ files provided in the form above:
+ <ul>
+ <li>The file <strong>MUST</strong> be character-separated values (CSV)
+ text file</li>
+ <li>The first row in the file <strong>MUST</strong> be a heading row, and
+ will be composed of the list identifiers for all of
+ samples/individuals/cases involved in your study.</li>
+ <li>The first column of data in the file <strong>MUST</strong> be the
+ identifiers for all of the phenotypes you wish to upload.</li>
+ </ul>
+ </p>
+
+ <p>If you do not specify the separator character, then we will assume a
+ <strong>TAB</strong> character was used as your separator.</p>
+
+ <p>We also assume you might include comments lines in your files. In that
+ case, if you do not specify what character denotes that a line in your files
+ is a comment line, we will assume the <strong>#</strong> character.<br />
+ A comment <strong>MUST ALWAYS</strong> begin at the start of the line marked
+ with the comment character specified.</p>
+
+ <h3 class="subheading" id="docs-file-metadata">File Metadata</h3>
+ <p>We request some details about your files to help us parse and process the
+ files correctly. The details we collect are:</p>
+ <dl>
+ <dt id="docs-file-separator">File separator</dt>
+ <dd>The files you provide should be character-separated value (CSV) files.
+ We need to know what character you used to separate the values in your
+ file. Some common ones are the Tab character, the comma, etc.<br />
+ Providing that information makes it possible for the system to parse and
+ process your files correctly.<br>
+ <strong>NOTE:</strong> All the files you upload MUST use the same
+ separator.</dd>
+
+ <dt id="docs-file-comment-character">Comment character</dt>
+ <dd>We support use of comment lines in your files. We only support one type
+ of comment style, the <em>line comment</em>.<br />
+ This mean the comment begins at the start of the line, and the end of that
+ line indicates the end of that comment. If you have a really long comment,
+ then you need to break it across multiple lines, marking each line a
+ comment line.<br />
+ The "comment character" is the character at the start of the line that
+ indicates that the line is a line comment.</dd>
+
+ <dt id="docs-file-na">No-Value indicator(s)</dt>
+ <dd>Data in the real world is messy, and in some cases, entirely absent. You
+ need to indicate, in your files, that a particular field did not have a
+ value, and once you do that, you then need to let the system know how you
+ mark such fields. Common ways of indicating "empty values" are, leaving
+ the field blank, using a character such as '-', or using strings like
+ "NA", "N/A", "NULL", etc.<br />
+ Providing this information will help with parsing and processing such
+ no-value fields the correct way.</dd>
+ </dl>
+
+ <h3 class="subheading" id="docs-file-phenotype-description">
+ file: Phenotypes Descriptions</h3>
+ <p>The data in this file is a matrix of <em>phenotypes × metadata-fields</em>.
+ Please note we use the term "metadata-fields" above loosely, due to lack of
+ a good word for this.</p>
+ <p>The file <strong>MUST</strong> have columns in this order:
+ <dl>
+ <dt>Phenotype Identifiers</dt>
+ <dd>These are the names/identifiers for your phenotypes. These
+ names/identifiers are the same ones you will have in all the other files you are
+ uploading.</dd>
+
+ <dt>Descriptions</dt>
+ <dd>Each phenotype will need a description. Good description are necessary
+ to inform other people of what the data is about. Good description are
+ hard to construct, so we provide
+ <a href="https://info.genenetwork.org/faq.php#q-22"
+ title="How to write phenotype descriptions">
+ advice on describing your phenotypes.</a></dd>
+
+ <dt>Units</dt>
+ <dd>Each phenotype will need units for the measurements taken. If there are
+ none, then indicate the field is a no-value field.</dd>
+ </dl></p>
+ <p>You can add more columns after those three if you want to, but these 3
+ <strong>MUST</strong> be present.</p>
+ <p>The file would, for example, look like the following:</p>
+ <code>id,description,units,…<br />
+ pheno10001|Central nervous system, behavior, cognition; …|mg|…<br />
+ pheno10002|Aging, metabolism, central nervous system: …|mg|…<br />
+ â‹®<br /></code>
+
+ <p><strong>Note 01</strong>: The first usable row is the heading row.</p>
+ <p><strong>Note 02: </strong>This example demonstrates a subtle issue that
+ could make your CSV file invalid &mdash; the choice of your field separator
+ character.<br >
+ In the example above, we use the pipe character (<code>|</code>) as our
+ field separator. This is because, if we follow the advice on how to write
+ good descriptions, then we cannot use the comma as our separator &ndash; if
+ we did, then our CSV file would be invalid because the system would have no
+ way to tell the difference between the comma as a field separator, and the
+ comma as a way to separate the "general category and ontology terms".</p>
+
+ <h3 class="subheading">file: Phenotype Data, Standard Errors and/or Sample Counts</h3>
+ <span id="docs-file-phenotype-data"></span>
+ <span id="docs-file-phenotype-se"></span>
+ <span id="docs-file-phenotype-n"></span>
+ <p>The data is a matrix of <em>phenotypes × individuals</em>, e.g.</p>
+ <code>
+ # num-cases: 2549
+ # num-phenos: 13
+ id,IND001,IND002,IND003,IND004,…<br />
+ pheno10001,61.400002,54.099998,483,49.799999,…<br />
+ pheno10002,49,50.099998,403,45.5,…<br />
+ pheno10003,62.5,53.299999,501,62.900002,…<br />
+ pheno10004,53.099998,55.099998,403,NA,…<br />
+ â‹®<br /></code>
+
+ <p>where <code>IND001,IND002,IND003,IND004,…</code> are the
+ samples/individuals/cases in your study, and
+ <code>pheno10001,pheno10002,pheno10004,pheno10004,…</code> are the
+ identifiers for your phenotypes.</p>
+ <p>The lines beginning with the "<em>#</em>" symbol (i.e.
+ <code># num-cases: 2549</code> and <code># num-phenos: 13</code> are comment
+ lines and will be ignored</p>
+ <p>In this example, the comma (,) is used as the file separator.</p>
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_preview_table("tbl-preview-pheno-desc", "descriptions")}}
+{{display_preview_table("tbl-preview-pheno-data", "data")}}
+{%if population.Family in families_with_se_and_n%}
+{{display_preview_table("tbl-preview-pheno-se", "standard errors")}}
+{{display_preview_table("tbl-preview-pheno-n", "number of samples")}}
+{%endif%}
+{{display_pheno_dataset_card(species, population, dataset)}}
+{%endblock%}
+
+
+{%block more_javascript%}
+<script src="{{url_for('base.node_modules',
+ filename='resumablejs/resumable.js')}}"></script>
+<script type="text/javascript" src="/static/js/files.js"></script>
+
+<script type="text/javascript">
+ $("#btn-reset-file-separator").on("click", (event) => {
+ event.preventDefault();
+ $("#txt-file-separator").val("\t");
+ $("#txt-file-separator").trigger("change");
+ });
+ $("#btn-reset-file-comment-character").on("click", (event) => {
+ event.preventDefault();
+ $("#txt-file-comment-character").val("#");
+ $("#txt-file-comment-character").trigger("change");
+ });
+ $("#btn-reset-file-na").on("click", (event) => {
+ event.preventDefault();
+ $("#txt-file-na").val("- NA N/A");
+ $("#txt-file-na").trigger("change");
+ });
+
+ var update_preview = (table, filedata, formdata, numrows) => {
+ table.find("thead tr").remove()
+ table.find(".data-row").remove();
+ var linenum = 0;
+ var tableheader = table.find("thead");
+ var tablebody = table.find("tbody");
+ var numheadings = 0;
+ var navalues = formdata
+ .na_strings
+ .split(" ")
+ .map((v) => {return v.trim();})
+ .filter((v) => {return Boolean(v);});
+ filedata.forEach((line) => {
+ if(line.startsWith(formdata.comment_char) || linenum >= numrows) {
+ return false;
+ }
+ var row = $("<tr></tr>");
+ line.split(formdata.separator)
+ .map((field) => {
+ var value = field.trim();
+ if(navalues.includes(value)) {
+ return "⋘NUL⋙";
+ }
+ return value;
+ })
+ .filter((field) => {
+ return (field !== "" && field != undefined && field != null);
+ })
+ .forEach((field) => {
+ if(linenum == 0) {
+ numheadings += 1;
+ var tablefield = $("<th></th>");
+ tablefield.text(field);
+ row.append(tablefield);
+ } else {
+ add_class(row, "data-row");
+ var tablefield = $("<td></td>");
+ tablefield.text(field);
+ row.append(tablefield);
+ }
+ });
+
+ if(linenum == 0) {
+ tableheader.append(row);
+ } else {
+ tablebody.append(row);
+ }
+ linenum += 1;
+ });
+
+ if(table.find("tbody tr.data-row").length > 0) {
+ add_class(table.find(".data-row-template"), "hidden");
+ } else {
+ remove_class(table.find(".data-row-template"), "hidden");
+ }
+ };
+
+ var makePreviewUpdater = (preview_table) => {
+ return (data) => {
+ update_preview(
+ preview_table,
+ data,
+ filesMetadata(),
+ PREVIEW_ROWS);
+ };
+ };
+
+ var preview_tables_to_elements_map = {
+ "#tbl-preview-pheno-desc": "#finput-phenotype-descriptions",
+ "#tbl-preview-pheno-data": "#finput-phenotype-data",
+ "#tbl-preview-pheno-se": "#finput-phenotype-se",
+ "#tbl-preview-pheno-n": "#finput-phenotype-n"
+ };
+
+ var filesMetadata = () => {
+ return {
+ "separator": $("#txt-file-separator").val(),
+ "comment_char": $(
+ "#txt-file-comment-character").val(),
+ "na_strings": $("#txt-file-na").val()
+ }
+ };
+
+ var PREVIEW_ROWS = 5;
+
+ var handler_update_previews = (event) => {
+ Object.entries(preview_tables_to_elements_map).forEach((mapentry) => {
+ var preview_table = $(mapentry[0]);
+ var file_input = $(mapentry[1]);
+ if(file_input.length === 1) {
+ readFirstNLines(
+ file_input[0].files[0],
+ 10,
+ [makePreviewUpdater(preview_table)]);
+ }
+ });
+ };
+
+ [
+ "#txt-file-separator",
+ "#txt-file-comment-character",
+ "#txt-file-na"
+ ].forEach((elementid) => {
+ $(elementid).on("change", handler_update_previews);
+ });
+
+ [
+ "#finput-phenotype-descriptions",
+ "#finput-phenotype-data",
+ "#finput-phenotype-se",
+ "#finput-phenotype-n"
+ ].forEach((elementid) => {
+ $(elementid).on("change", (event) => {
+ readFirstNLines(
+ event.target.files[0],
+ 10,
+ [makePreviewUpdater(
+ $("#" + event.target.getAttribute("data-preview-table")))]);
+ });
+ });
+
+
+ var resumableDisplayFiles = (display_area, files) => {
+ files.forEach((file) => {
+ display_area.find(".file-display").remove();
+ var display_element = display_area
+ .find(".file-display-template")
+ .clone();
+ remove_class(display_element, "hidden");
+ remove_class(display_element, "file-display-template");
+ add_class(display_element, "file-display");
+ display_element.find(".filename").text(file.name
+ || file.fileName
+ || file.relativePath
+ || file.webkitRelativePath);
+ display_element.find(".filesize").text(
+ (file.size / (1024*1024)).toFixed(2) + "MB");
+ display_element.find(".fileuniqueid").text(file.uniqueIdentifier);
+ display_element.find(".filemimetype").text(file.file.type);
+ display_area.append(display_element);
+ });
+ };
+
+
+ var indicateProgress = (resumable, progress_bar) => {
+ return () => {/*Has no event!*/
+ var progress = (resumable.progress() * 100).toFixed(2);
+ var pbar = progress_bar.find(".progress-bar");
+ remove_class(progress_bar, "hidden");
+ pbar.css("width", progress+"%");
+ pbar.attr("aria-valuenow", progress);
+ pbar.text("Uploading: " + progress + "%");
+ };
+ };
+
+ var retryUpload = (retry_button, cancel_button) => {
+ retry_button.on("click", (event) => {
+ resumable.files.forEach((file) => {file.retry();});
+ add_class(retry_button, "hidden");
+ remove_class(cancel_button, "hidden");
+ add_class(browse_button, "hidden");
+ });
+ };
+
+ var cancelUpload = (cancel_button, retry_button) => {
+ cancel_button.on("click", (event) => {
+ resumable.files.forEach((file) => {
+ if(file.isUploading()) {
+ file.abort();
+ }
+ });
+ add_class(cancel_button, "hidden");
+ remove_class(retry_button, "hidden");
+ remove_class(browse_button, "hidden");
+ });
+ };
+
+
+ var startUpload = (browse_button, retry_button, cancel_button) => {
+ return (event) => {
+ remove_class(cancel_button, "hidden");
+ add_class(retry_button, "hidden");
+ add_class(browse_button, "hidden");
+ };
+ };
+
+ var processForm = (form) => {
+ var formdata = new FormData(form);
+ uploaded_files.forEach((msg) => {
+ formdata.delete(msg["file-input-name"]);
+ formdata.append(msg["file-input-name"], JSON.stringify({
+ "uploaded-file": msg["uploaded-file"],
+ "original-name": msg["original-name"]
+ }));
+ });
+ formdata.append("resumable-upload", "true");
+ return formdata;
+ }
+
+ var uploaded_files = new Set();
+ var submitForm = (new_file) => {
+ uploaded_files.add(new_file);
+ if(uploaded_files.size === resumables.length) {
+ var form = $("#frm-add-phenotypes");
+ if(form.length !== 1) {
+ // TODO: Handle error somehow?
+ alert("Could not find form!!!");
+ return false;
+ }
+
+ $.ajax({
+ "url": form.attr("action"),
+ "type": "POST",
+ "data": processForm(form[0]),
+ "processData": false,
+ "contentType": false,
+ "success": (data, textstatus, jqxhr) => {
+ // TODO: Redirect to endpoint that should come as part of the
+ // success/error message.
+ console.log("SUCCESS DATA: ", data);
+ console.log("SUCCESS STATUS: ", textstatus);
+ console.log("SUCCESS jqXHR: ", jqxhr);
+ window.location.assign(window.location.origin + data["redirect-to"]);
+ },
+ });
+ return false;
+ }
+ return false;
+ };
+
+ var uploadSuccess = (file_input_name) => {
+ return (file, message) => {
+ submitForm({...JSON.parse(message), "file-input-name": file_input_name});
+ };
+ };
+
+
+ var uploadError = () => {
+ return (message, file) => {
+ $("#frm-add-phenotypes input[type=submit]").removeAttr("disabled");
+ console.log("THE FILE:", file);
+ console.log("THE ERROR MESSAGE:", message);
+ };
+ };
+
+
+
+ var makeResumableObject = (form_id, file_input_id, resumable_element_id, preview_table_id) => {
+ var the_form = $("#" + form_id);
+ var file_input = $("#" + file_input_id);
+ var submit_button = the_form.find("input[type=submit]");
+ if(file_input.length != 1) {
+ return false;
+ }
+ var r = errorHandler(
+ fileSuccessHandler(
+ uploadStartHandler(
+ filesAddedHandler(
+ markResumableDragAndDropElement(
+ makeResumableElement(
+ the_form.attr("data-resumable-target"),
+ file_input.parent(),
+ $("#" + resumable_element_id),
+ submit_button,
+ ["csv", "tsv"]),
+ file_input.parent(),
+ $("#" + resumable_element_id),
+ $("#" + resumable_element_id + "-browse-button")),
+ (files) => {
+ // TODO: Also trigger preview!
+ resumableDisplayFiles(
+ $("#" + resumable_element_id + "-selected-files"), files);
+ files.forEach((file) => {
+ readFirstNLines(
+ file.file,
+ 10,
+ [makePreviewUpdater(
+ $("#" + preview_table_id))])
+ });
+ }),
+ startUpload($("#" + resumable_element_id + "-browse-button"),
+ $("#" + resumable_element_id + "-retry-button"),
+ $("#" + resumable_element_id + "-cancel-button"))),
+ uploadSuccess(file_input.attr("name"))),
+ uploadError());
+
+ /** Setup progress indicator **/
+ progressHandler(
+ r,
+ indicateProgress(r, $("#" + resumable_element_id + "-progress-bar")));
+
+ return r;
+ };
+
+ var resumables = [
+ ["frm-add-phenotypes", "finput-phenotype-descriptions", "resumable-phenotype-descriptions", "tbl-preview-pheno-desc"],
+ ["frm-add-phenotypes", "finput-phenotype-data", "resumable-phenotype-data", "tbl-preview-pheno-data"],
+ ["frm-add-phenotypes", "finput-phenotype-se", "resumable-phenotype-se", "tbl-preview-pheno-se"],
+ ["frm-add-phenotypes", "finput-phenotype-n", "resumable-phenotype-n", "tbl-preview-pheno-n"],
+ ].map((row) => {
+ return makeResumableObject(row[0], row[1], row[2], row[3]);
+ }).filter((val) => {
+ return Boolean(val);
+ });
+
+ $("#frm-add-phenotypes input[type=submit]").on("click", (event) => {
+ event.preventDefault();
+ // TODO: Check all the relevant files exist
+ // TODO: Verify that files are not duplicated
+ var filenames = [];
+ var nondupfiles = [];
+ resumables.forEach((r) => {
+ var fname = r.files[0].file.name;
+ filenames.push(fname);
+ if(!nondupfiles.includes(fname)) {
+ nondupfiles.push(fname);
+ }
+ });
+
+ // Check that all files were provided
+ if(resumables.length !== filenames.length) {
+ window.alert("You MUST provide all the files requested.");
+ event.target.removeAttribute("disabled");
+ return false;
+ }
+
+ // Check that there are no duplicate files
+ var duplicates = Object.entries(filenames.reduce(
+ (acc, curr, idx, arr) => {
+ acc[curr] = (acc[curr] || 0) + 1;
+ return acc;
+ },
+ {})).filter((entry) => {return entry[1] !== 1;});
+ if(duplicates.length > 0) {
+ var msg = "The file(s):\r\n";
+ msg = msg + duplicates.reduce(
+ (msgstr, afile) => {
+ return msgstr + " • " + afile[0] + "\r\n";
+ },
+ "");
+ msg = msg + "is(are) duplicated. Please fix and try again.";
+ window.alert(msg);
+ event.target.removeAttribute("disabled");
+ return false;
+ }
+ // TODO: Check all fields
+ // Start the uploads.
+ event.target.setAttribute("disabled", "disabled");
+ resumables.forEach((r) => {r.upload();});
+ });
+</script>
+{%endblock%}
diff --git a/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html b/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html
new file mode 100644
index 0000000..898fc0c
--- /dev/null
+++ b/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html
@@ -0,0 +1,207 @@
+{%extends "phenotypes/add-phenotypes-base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="add-phenotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">Add Phenotypes</a>
+</li>
+{%endblock%}
+
+{%block frm_add_phenotypes_documentation%}
+<p>Select the zip file bundle containing information on the phenotypes you
+ wish to upload, then click the "Upload Phenotypes" button below to
+ upload the data.</p>
+<p>If you wish to upload the files individually instead,
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}"
+ title="">click here</a>.</p>
+<p>See the <a href="#section-file-formats">File Formats</a> section below
+ to get an understanding of what is expected of the bundle files you
+ upload.</p>
+{%endblock%}
+
+{%block frm_add_phenotypes_elements%}
+<div class="form-group">
+ <label for="finput-phenotypes-bundle" class="form-label">
+ Phenotypes Bundle</label>
+ <input type="file"
+ id="finput-phenotypes-bundle"
+ name="phenotypes-bundle"
+ accept="application/zip, .zip"
+ required="required"
+ class="form-control" />
+</div>
+{%endblock%}
+
+{%block page_documentation%}
+<div class="row">
+ <h2 class="heading" id="section-file-formats">File Formats</h2>
+ <p>We accept an extended form of the
+ <a href="https://kbroman.org/qtl2/assets/vignettes/input_files.html#format-of-the-data-files"
+ title="R/qtl2 software input file format documentation">
+ input files' format used with the R/qtl2 software</a> as a single ZIP
+ file</p>
+ <p>The files that are used for this feature are:
+ <ul>
+ <li>the <em>control</em> file</li>
+ <li><em>pheno</em> file(s)</li>
+ <li><em>phenocovar</em> file(s)</li>
+ <li><em>phenose</em> files(s)</li>
+ </ul>
+ </p>
+ <p>Other files within the bundle will be ignored, for this feature.</p>
+ <p>The following section will detail the expectations for each of the
+ different file types within the uploaded ZIP file bundle for phenotypes:</p>
+
+ <h3 class="subheading">Control File</h3>
+ <p>There <strong>MUST be <em>one, and only one</em></strong> file that acts
+ as the control file. This file can be:
+ <ul>
+ <li>a <em>JSON</em> file, or</li>
+ <li>a <em>YAML</em> file.</li>
+ </ul>
+ </p>
+
+ <p>The control file is useful for defining things about the bundle such as:</p>
+ <ul>
+ <li>The field separator value (default: <code>sep: ','</code>). There can
+ only ever be one field separator and it <strong>MUST</strong> be the same
+ one for <strong>ALL</strong> files in the bundle.</li>
+ <li>The comment character (default: <code>comment.char: '#'</code>). Any
+ line that starts with this character will be considered a comment line and
+ be ignored in its entirety.</li>
+ <li>Code for missing values (default: <code>na.strings: 'NA'</code>). You
+ can specify more than one code to indicate missing values, e.g.
+ <code>{…, "na.strings": ["NA", "N/A", "-"], …}</code></li>
+ </ul>
+
+ <h3 class="subheading"><em>pheno</em> File(s)</h3>
+ <p>These files are the main data files. You must have at least one of these
+ files in your bundle for it to be valid for this step.</p>
+ <p>The data is a matrix of <em>individuals × phenotypes</em> by default, as
+ below:<br />
+ <code>
+ id,10001,10002,10003,10004,…<br />
+ BXD1,61.400002,54.099998,483,49.799999,…<br />
+ BXD2,49,50.099998,403,45.5,…<br />
+ BXD5,62.5,53.299999,501,62.900002,…<br />
+ BXD6,53.099998,55.099998,403,NA,…<br />
+ â‹®<br /></code>
+ </p>
+ <p>If the <code>pheno_transposed</code> value is set to <code>True</code>,
+ then the data will be a <em>phenotypes × individuals</em> matrix as in the
+ example below:<br />
+ <code>
+ id,BXD1,BXD2,BXD5,BXD6,…<br />
+ 10001,61.400002,49,62.5,53.099998,…<br />
+ 10002,54.099998,50.099998,53.299999,55.099998,…<br />
+ 10003,483,403,501,403,…<br />
+ 10004,49.799999,45.5,62.900002,NA,…<br />
+ â‹®
+ </code>
+ </p>
+
+
+ <h3 class="subheading"><em>phenocovar</em> File(s)</h3>
+ <p>At least one phenotypes metadata file with the metadata values such as
+ descriptions, PubMed Identifier, publication titles (if present), etc.</p>
+ <p>The data in this/these file(s) is a matrix of
+ <em>phenotypes × phenotypes-covariates</em>. The first column is always the
+ phenotype names/identifiers — same as in the R/qtl2 format.</p>
+ <p><em>phenocovar</em> files <strong>should never be transposed</strong>!</p>
+ <p>This file <strong>MUST</strong> be present in the bundle, and have data for
+ the bundle to be considered valid by our system for this step.<br />
+ In addition to that, the following are the fields that <strong>must be
+ present</strong>, and
+ have values, in the file before the file is considered valid:
+ <ul>
+ <li><em>description</em>: A description for each phenotype. Useful
+ for users to know what the phenotype is about.</li>
+ <li><em>units</em>: The units of measurement for the phenotype,
+ e.g. milligrams for brain weight, centimetres/millimetres for
+ tail-length, etc.</li>
+ </ul></p>
+
+ <p>The following <em>optional</em> fields can also be provided:
+ <ul>
+ <li><em>pubmedid</em>: A PubMed Identifier for the publication where
+ the phenotype is published. If this field is not provided, the system will
+ assume your phenotype is not published.</li>
+ </ul>
+ </p>
+ <p>These files will be marked up in the control file with the
+ <code>phenocovar</code> key, as in the examples below:
+ <ol>
+ <li>JSON: single file<br />
+ <code>{<br />
+ &nbsp;&nbsp;â‹®,<br />
+ &nbsp;&nbsp;"phenocovar": "your_covariates_file.csv",<br />
+ &nbsp;&nbsp;â‹®<br />
+ }
+ </code>
+ </li>
+ <li>JSON: multiple files<br />
+ <code>{<br />
+ &nbsp;&nbsp;â‹®,<br />
+ &nbsp;&nbsp;"phenocovar": [<br />
+ &nbsp;&nbsp;&nbsp;&nbsp;"covariates_file_01.csv",<br />
+ &nbsp;&nbsp;&nbsp;&nbsp;"covariates_file_01.csv",<br />
+ &nbsp;&nbsp;&nbsp;&nbsp;â‹®<br />
+ &nbsp;&nbsp;],<br />
+ &nbsp;&nbsp;â‹®<br />
+ }
+ </code>
+ </li>
+ <li>YAML: single file or<br />
+ <code>
+ â‹®<br />
+ phenocovar: your_covariates_file.csv<br />
+ â‹®
+ </code>
+ </li>
+ <li>YAML: multiple files<br />
+ <code>
+ â‹®<br />
+ phenocovar:<br />
+ - covariates_file_01.csv<br />
+ - covariates_file_02.csv<br />
+ - covariates_file_03.csv<br />
+ …<br />
+ â‹®
+ </code>
+ </li>
+ </ol>
+ </p>
+
+ <h3 class="subheading"><em>phenose</em> and <em>phenonum</em> File(s)</h3>
+ <p>These are extensions to the R/qtl2 standard, i.e. these types ofs file are
+ not supported by the original R/qtl2 file format</p>
+ <p>We use these files to upload the standard errors (<em>phenose</em>) when
+ the data file (<em>pheno</em>) is average data. In that case, the
+ <em>phenonum</em> file(s) contains the number of individuals that were
+ involved when computing the averages.</p>
+ <p>Both types of files are matrices of <em>individuals × phenotypes</em> by
+ default. Like the related <em>pheno</em> files, if
+ <code>pheno_transposed: True</code>, then the file will be a matrix of
+ <em>phenotypes × individuals</em>.</p>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_pheno_dataset_card(species, population, dataset)}}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/base.html b/uploader/templates/phenotypes/base.html
new file mode 100644
index 0000000..adbc012
--- /dev/null
+++ b/uploader/templates/phenotypes/base.html
@@ -0,0 +1,19 @@
+{%extends "populations/base.html"%}
+
+{%block lvl3_breadcrumbs%}
+<li {%if activelink=="phenotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ {%if dataset is mapping%}
+ <a href="{{url_for('species.populations.phenotypes.view_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">{{dataset.Name}}</a>
+ {%else%}
+ <a href="{{url_for('species.populations.phenotypes.index')}}">Phenotypes</a>
+ {%endif%}
+</li>
+{%block lvl4_breadcrumbs%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/create-dataset.html b/uploader/templates/phenotypes/create-dataset.html
new file mode 100644
index 0000000..8e45491
--- /dev/null
+++ b/uploader/templates/phenotypes/create-dataset.html
@@ -0,0 +1,108 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="create-dataset"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.create_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">Create Datasets</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <p>Create a new phenotype dataset.</p>
+</div>
+
+<div class="row">
+ <form id="frm-create-pheno-dataset"
+ action="{{url_for('species.populations.phenotypes.create_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ method="POST">
+
+ <div class="form-group">
+ <label class="form-label" for="txt-dataset-name">Name</label>
+ {%if errors["dataset-name"] is defined%}
+ <small class="form-text text-muted danger">
+ <p>{{errors["dataset-name"]}}</p></small>
+ {%endif%}
+ <input type="text"
+ name="dataset-name"
+ id="txt-dataset-name"
+ value="{{original_formdata.get('dataset-name') or (population.InbredSetCode + 'Publish')}}"
+ {%if errors["dataset-name"] is defined%}
+ class="form-control danger"
+ {%else%}
+ class="form-control"
+ {%endif%}
+ required="required" />
+ <small class="form-text text-muted">
+ <p>A short representative name for the dataset.</p>
+ <p>Recommended: Use the population code and append "Publish" at the end.
+ <br />This field will only accept names composed of
+ letters ('A-Za-z'), numbers (0-9), hyphens and underscores.</p>
+ </small>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="txt-dataset-fullname">Full Name</label>
+ {%if errors["dataset-fullname"] is defined%}
+ <small class="form-text text-muted danger">
+ <p>{{errors["dataset-fullname"]}}</p></small>
+ {%endif%}
+ <input id="txt-dataset-fullname"
+ name="dataset-fullname"
+ type="text"
+ value="{{original_formdata.get('dataset-fullname', '')}}"
+ {%if errors["dataset-fullname"] is defined%}
+ class="form-control danger"
+ {%else%}
+ class="form-control"
+ {%endif%}
+ required="required" />
+ <small class="form-text text-muted">
+ <p>A longer, descriptive name for the dataset. The name is meant for use
+ by humans, and therefore, it should be clear what the dataset contains
+ from the name.</p>
+ </small>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="txt-dataset-shortname">Short Name</label>
+ <input id="txt-dataset-shortname"
+ name="dataset-shortname"
+ type="text"
+ class="form-control"
+ value="{{original_formdata.get('dataset-shortname') or (population.InbredSetCode + ' Publish')}}" />
+ <small class="form-text text-muted">
+ <p>An optional, short name for the dataset. <br />
+ If this is not provided, it will default to the value provided for the
+ <strong>Name</strong> field above.</p></small>
+ </div>
+
+ <div class="form-group">
+ <input type="submit"
+ class="btn btn-primary"
+ value="create phenotype dataset" />
+ </div>
+
+ </form>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/edit-phenotype.html b/uploader/templates/phenotypes/edit-phenotype.html
new file mode 100644
index 0000000..32c903f
--- /dev/null
+++ b/uploader/templates/phenotypes/edit-phenotype.html
@@ -0,0 +1,332 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="edit-phenotype"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.edit_phenotype_data',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ xref_id=xref_id)}}">View Datasets</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <h2 class="heading">edit phenotype data</h2>
+ <p>The forms provided in this page help you update the data for the
+ phenotypes, and the publication information for the phenotype,
+ respectively.</p>
+</div>
+
+<div class="row">
+ <h3 class="subheading">Basic metadata</h3>
+ <form name="frm-phenotype-basic-metadata"
+ class="form-horizontal"
+ method="POST"
+ action="{{url_for(
+ 'species.populations.phenotypes.edit_phenotype_data',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ xref_id=xref_id)}}">
+ <input type="hidden" name="phenotype-id" value="{{phenotype.Id}}" />
+ <div class="form-group">
+ <label for="txt-pre-publication-description"
+ class="control-label col-sm-2">Pre-Publication Description</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-pre-publication-description"
+ name="pre-publication-description"
+ class="form-control"
+ value="{{phenotype['Pre_publication_description'] or ''}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-pre-publication-abbreviation"
+ class="control-label col-sm-2">Pre-Publication Abbreviation</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-pre-publication-abbreviation"
+ name="pre-publication-abbreviation"
+ class="form-control"
+ value="{{phenotype['Pre_publication_abbreviation'] or ''}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-post-publication-description"
+ class="control-label col-sm-2">Post-Publication Description</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-post-publication-description"
+ name="post-publication-description"
+ class="form-control"
+ value="{{phenotype['Post_publication_description'] or ''}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-post-publication-abbreviation"
+ class="control-label col-sm-2">Post-Publication Abbreviation</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-post-publication-abbreviation"
+ name="post-publication-abbreviation"
+ class="form-control"
+ value="{{phenotype['Post_publication_abbreviation'] or ''}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-original-description"
+ class="control-label col-sm-2">Original Description</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-original-description"
+ name="original-description"
+ class="form-control"
+ value="{{phenotype['Original_description'] or ''}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-units"
+ class="control-label col-sm-2">units</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-units"
+ name="units"
+ class="form-control"
+ required="required"
+ value="{{phenotype['Units']}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <div class="col-sm-offset-2 col-sm-10">
+ <input type="submit"
+ name="submit"
+ class="btn btn-primary"
+ value="update basic metadata">
+ </div>
+ </div>
+ </form>
+</div>
+
+
+<div class="row">
+ <h3 class="subheading">phenotype data</h3>
+ <form id="frm-edit-phenotype-data"
+ class="form-horizontal"
+ method="POST"
+ action="{{url_for(
+ 'species.populations.phenotypes.edit_phenotype_data',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ xref_id=xref_id)}}">
+ <div style="max-height: 23.37em;overflow-y: scroll;">
+ <table class="table table-striped table-responsive table-form-table">
+ <thead style="position: sticky; top: 0;">
+ <tr>
+ <th>#</th>
+ <th>Sample</th>
+ <th>Value</th>
+ {%if population.Family in families_with_se_and_n%}
+ <th>Standard-Error</th>
+ <th>Number of Samples</th>
+ {%endif%}
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for item in phenotype.data%}
+ <tr>
+ <td>{{loop.index}}</td>
+ <td>{{item.StrainName}}</td>
+ <td>
+ <input type="text"
+ name="value-new::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.value}}"
+ class="form-control" />
+ <input type="hidden"
+ name="value-original::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.value}}" /></td>
+ {%if population.Family in families_with_se_and_n%}
+ <td>
+ <input type="text"
+ name="se-new::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.error or ''}}"
+ data-original-value="{{item.error or ''}}"
+ class="form-control" />
+ <input type="hidden"
+ name="se-original::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.error or ''}}" /></td>
+ <td>
+ <input type="text"
+ name="n-new::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.count or ''}}"
+ data-original-value="{{item.count or "-"}}"
+ class="form-control" />
+ <input type="hidden"
+ name="n-original::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.count or ''}}" /></td>
+ {%endif%}
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+ </div>
+ <div class="form-group">
+ <div class="col-sm-offset-2 col-sm-10">
+ <input type="submit"
+ name="submit"
+ class="btn btn-primary"
+ value="update data" />
+ </div>
+ </div>
+ </form>
+</div>
+
+
+<div class="row">
+ <h3 class="subheading">publication information</h3>
+ <p>Use the form below to update the publication information for this
+ phenotype.</p>
+ <form id="frm-edit-phenotype-pub-data"
+ class="form-horizontal"
+ method="POST"
+ action="#">
+ <div class="form-group">
+ <label for="txt-pubmed-id" class="control-label col-sm-2">Pubmed ID</label>
+ <div class="col-sm-10">
+ <input id="txt-pubmed-id" name="pubmed-id" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter your publication's PubMed ID.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-authors" class="control-label col-sm-2">Authors</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-authors" name="publication-authors"
+ type="text" class="form-control" />
+ <span class="form-text text-muted">
+ Enter the authors.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-title" class="control-label col-sm-2">
+ Publication Title</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-title" name="publication-title" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter your publication's title.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-abstract" class="control-label col-sm-2">
+ Publication Abstract</label>
+ <div class="col-sm-10">
+ <textarea id="txt-publication-abstract" name="publication-abstract"
+ class="form-control" rows="10"></textarea>
+ <span class="form-text text-muted">
+ Enter the abstract for your publication.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-journal" class="control-label col-sm-2">Journal</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-journal" name="journal" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter the name of the journal where your work was published.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-volume" class="control-label col-sm-2">Volume</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-volume" name="publication-volume" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter the volume in the following format &hellip;</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-pages" class="control-label col-sm-2">Pages</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-pages" name="publication-pages" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter the journal volume where your work was published.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="select-publication-month" class="control-label col-sm-2">
+ Publication Month</label>
+ <div class="col-sm-10">
+ <select id="select-publication-month" name="publication-month"
+ class="form-control">
+ {%for month in monthnames%}
+ <option value="{{month | lower}}"
+ {%if current_month | lower == month | lower%}
+ selected="selected"
+ {%endif%}>{{month | capitalize}}</option>
+ {%endfor%}
+ </select>
+ <span class="form-text text-muted">
+ Select the month when the work was published.
+ <span class="text-danger">
+ This cannot be before, say 1600 and cannot be in the future!</span></span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-year" class="control-label col-sm-2">Publication Year</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-year" name="publication-year" type="text"
+ class="form-control" value="{{current_year}}" />
+ <span class="form-text text-muted">
+ Enter the year your work was published.
+ <span class="text-danger">
+ This cannot be before, say 1600 and cannot be in the future!</span>
+ </span>
+ </div>
+ </div>
+ <div class="form-group">
+ <div class="col-sm-offset-2 col-sm-10">
+ <input type="submit"
+ name="submit"
+ class="btn btn-primary not-implemented"
+ value="update publication" />
+ </div>
+ </div>
+ </form>
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/index.html b/uploader/templates/phenotypes/index.html
new file mode 100644
index 0000000..689c28e
--- /dev/null
+++ b/uploader/templates/phenotypes/index.html
@@ -0,0 +1,21 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-select-species.html" import select_species_form%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ {{select_species_form(url_for("species.populations.phenotypes.index"), species)}}
+</div>
+{%endblock%}
+
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/species.js"></script>
+{%endblock%}
diff --git a/uploader/templates/phenotypes/job-status.html b/uploader/templates/phenotypes/job-status.html
new file mode 100644
index 0000000..12963c1
--- /dev/null
+++ b/uploader/templates/phenotypes/job-status.html
@@ -0,0 +1,155 @@
+{%extends "phenotypes/base.html"%}
+{%from "cli-output.html" import cli_output%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
+
+{%block extrameta%}
+{%if job and job.status not in ("success", "completed:success", "error", "completed:error")%}
+<meta http-equiv="refresh" content="5" />
+{%endif%}
+{%endblock%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="add-phenotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">View Datasets</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+
+{%if job%}
+<h4 class="subheading">Progress</h4>
+<div class="row" style="overflow:scroll;">
+ <p><strong>Process Status:</strong> {{job.status}}</p>
+ {%if metadata%}
+ <table class="table table-responsive">
+ <thead>
+ <tr>
+ <th>File</th>
+ <th>Status</th>
+ <th>Lines Processed</th>
+ <th>Total Errors</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for file,meta in metadata.items()%}
+ <tr>
+ <td>{{file}}</td>
+ <td>{{meta.status}}</td>
+ <td>{{meta.linecount}}</td>
+ <td>{{meta["total-errors"]}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+ {%endif%}
+</div>
+
+<div class="row">
+ {%if job.status in ("completed:success", "success")%}
+ <p>
+ {%if errors | length == 0%}
+ <a href="{{url_for('species.populations.phenotypes.review_job_data',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ job_id=job_id)}}"
+ class="btn btn-primary"
+ title="Continue to process data">Continue</a>
+ {%else%}
+ <span class="text-muted"
+ disabled="disabled"
+ style="border: solid 2px;border-radius: 5px;padding: 0.3em;">
+ Cannot continue due to errors. Please fix the errors first.
+ </span>
+ {%endif%}
+ </p>
+ {%endif%}
+</div>
+
+<h4 class="subheading">Errors</h4>
+<div class="row" style="max-height: 20em; overflow: scroll;">
+ {%if errors | length == 0 %}
+ <p class="text-info">
+ <span class="glyphicon glyphicon-info-sign"></span>
+ No errors found so far
+ </p>
+ {%else%}
+ <table class="table table-responsive">
+ <thead style="position: sticky; top: 0; background: white;">
+ <tr>
+ <th>File</th>
+ <th>Row</th>
+ <th>Column</th>
+ <th>Value</th>
+ <th>Message</th>
+ </tr>
+ </thead>
+
+ <tbody style="font-size: 0.9em;">
+ {%for error in errors%}
+ <tr>
+ <td>{{error.filename}}</td>
+ <td>{{error.rowtitle}}</td>
+ <td>{{error.coltitle}}</td>
+ <td>{%if error.cellvalue | length > 25%}
+ {{error.cellvalue[0:24]}}&hellip;
+ {%else%}
+ {{error.cellvalue}}
+ {%endif%}
+ </td>
+ <td>
+ {%if error.message | length > 250 %}
+ {{error.message[0:249]}}&hellip;
+ {%else%}
+ {{error.message}}
+ {%endif%}
+ </td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+ {%endif%}
+</div>
+
+<div class="row">
+ {{cli_output(job, "stdout")}}
+</div>
+
+<div class="row">
+ {{cli_output(job, "stderr")}}
+</div>
+
+{%else%}
+<div class="row">
+ <h3 class="text-danger">No Such Job</h3>
+ <p>Could not find a job with the ID: {{job_id}}</p>
+ <p>
+ Please go back to
+ <a href="{{url_for('species.populations.phenotypes.view_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}"
+ title="'{{dataset.Name}}' dataset page">
+ the '{{dataset.Name}}' dataset page</a>
+ to upload new phenotypes or edit existing ones.</p>
+</div>
+{%endif%}
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_pheno_dataset_card(species, population, dataset)}}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/list-datasets.html b/uploader/templates/phenotypes/list-datasets.html
new file mode 100644
index 0000000..2cf2c7f
--- /dev/null
+++ b/uploader/templates/phenotypes/list-datasets.html
@@ -0,0 +1,68 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="list-datasets"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.list_datasets',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">List Datasets</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ {%if datasets | length > 0%}
+ <p>The dataset(s) available for this population is/are:</p>
+
+ <table class="table">
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Full Name</th>
+ <th>Short Name</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for dataset in datasets%}
+ <tr>
+ <td><a href="{{url_for('species.populations.phenotypes.view_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">{{dataset.Name}}</a></td>
+ <td>{{dataset.FullName}}</td>
+ <td>{{dataset.ShortName}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+ {%else%}
+ <p>Phenotypes need to go into a dataset. We do not currently have a dataset
+ for species <strong>'{{species["FullName"]}} ({{species["Name"]}})'</strong>
+ phenotypes.</p>
+
+ <p>Do, please, create a new dataset by clicking on the "Create Dataset" button
+ below and following the prompts/instructions.</p>
+ <p><a href="{{url_for('species.populations.phenotypes.create_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ class="btn btn-primary"
+ title="Create a new phenotype dataset.">create dataset</a></p>
+ {%endif%}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html
new file mode 100644
index 0000000..11b108b
--- /dev/null
+++ b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html
@@ -0,0 +1,31 @@
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%macro display_pheno_dataset_card(species, population, dataset)%}
+{{display_population_card(species, population)}}
+
+<div class="card">
+ <div class="card-body">
+ <h5 class="card-title">Phenotypes' Dataset</h5>
+ <div class="card-text">
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Name</td>
+ <td>{{dataset.Name}}</td>
+ </tr>
+
+ <tr>
+ <td>Full Name</td>
+ <td>{{dataset.FullName}}</td>
+ </tr>
+
+ <tr>
+ <td>Short Name</td>
+ <td>{{dataset.ShortName}}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+</div>
+{%endmacro%}
diff --git a/uploader/templates/phenotypes/macro-display-preview-table.html b/uploader/templates/phenotypes/macro-display-preview-table.html
new file mode 100644
index 0000000..f54c53e
--- /dev/null
+++ b/uploader/templates/phenotypes/macro-display-preview-table.html
@@ -0,0 +1,21 @@
+{%macro display_preview_table(tableid, filetype)%}
+<div class="card" style="max-width: 676px;">
+ <div class="card-body">
+ <h5 class="card-title">Phenotypes '{{filetype | title}}' File Preview</h5>
+ <div class="card-text" style="overflow: scroll;">
+ <table id="{{tableid}}" class="table table-condensed table-responsive">
+ <thead>
+ <tr>
+ </tr>
+ <tbody>
+ <tr>
+ <td class="data-row-template text-info">
+ Provide a phenotype '{{filetype | lower}}' file to preview.
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+</div>
+{%endmacro%}
diff --git a/uploader/templates/phenotypes/macro-display-resumable-elements.html b/uploader/templates/phenotypes/macro-display-resumable-elements.html
new file mode 100644
index 0000000..b0bf1b5
--- /dev/null
+++ b/uploader/templates/phenotypes/macro-display-resumable-elements.html
@@ -0,0 +1,60 @@
+{%macro display_resumable_elements(id, title, help)%}
+<div id="{{id}}"
+ class="resumable-elements hidden"
+ style="background:#D4D4EE;border-radius: 5px;;padding: 1em;border-left: solid #B2B2CC 1px;border-bottom: solid #B2B2CC 2px;margin-top:0.3em;">
+ <strong style="line-height: 1.2em;">{{title | title}}</strong>
+
+ <span class="form-text text-muted">{{help | safe}}</span>
+
+ <div id="{{id}}-selected-files"
+ class="resumable-selected-files"
+ style="display:flex;flex-direction:row;flex-wrap: wrap;justify-content:space-around;gap:10px 20px;">
+ <div class="panel panel-info file-display-template hidden">
+ <div class="panel-heading filename">The Filename Goes Here!</div>
+ <div class="panel-body">
+ <ul>
+ <li>
+ <strong>Name</strong>:
+ <span class="filename">the file's name</span></li>
+
+ <li><strong>Size</strong>: <span class="filesize">0 MB</span></li>
+
+ <li>
+ <strong>Unique Identifier</strong>:
+ <span class="fileuniqueid">brrr</span></li>
+
+ <li>
+ <strong>Mime</strong>:
+ <span class="filemimetype">text/csv</span></li>
+ </ul>
+ </div>
+ </div>
+ </div>
+
+ <a id="{{id}}-browse-button"
+ class="resumable-browse-button btn btn-info"
+ href="#"
+ style="margin-left: 80%;">Browse</a>
+
+ <div id="{{id}}-progress-bar" class="progress hidden">
+ <div class="progress-bar"
+ role="progress-bar"
+ aria-valuenow="60"
+ aria-valuemin="0"
+ aria-valuemax="100"
+ style="width: 0%;">
+ Uploading: 60%
+ </div>
+ </div>
+
+ <div id="{{id}}-cancel-resume-buttons">
+ <a id="{{id}}-resume-button"
+ class="resumable-resume-button btn btn-info hidden"
+ href="#">resume upload</a>
+
+ <a id="{{id}}-cancel-button"
+ class="resumable-cancel-button btn btn-danger hidden"
+ href="#">cancel upload</a>
+ </div>
+</div>
+{%endmacro%}
diff --git a/uploader/templates/phenotypes/review-job-data.html b/uploader/templates/phenotypes/review-job-data.html
new file mode 100644
index 0000000..7bc8c62
--- /dev/null
+++ b/uploader/templates/phenotypes/review-job-data.html
@@ -0,0 +1,101 @@
+{%extends "phenotypes/base.html"%}
+{%from "cli-output.html" import cli_output%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
+
+{%block extrameta%}
+{%if not job%}
+<meta http-equiv="refresh"
+ content="20; url={{url_for('species.populations.phenotypes.view_dataset', species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}" />
+{%endif%}
+{%endblock%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="add-phenotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">View Datasets</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+
+{%if job%}
+<div class="row">
+ <h3 class="heading">Data Review</h3>
+ <p>The &#x201C;<strong>{{dataset.FullName}}</strong>&#x201D; dataset from the
+ &#x201C;<strong>{{population.FullName}}</strong>&#x201D; population of the
+ species &#x201C;<strong>{{species.SpeciesName}} ({{species.FullName}})</strong>&#x201D;
+ will be updated as follows:</p>
+
+ {%for ftype in ("phenocovar", "pheno", "phenose", "phenonum")%}
+ {%if summary.get(ftype, False)%}
+ <ul>
+ <li>A total of {{summary[ftype]["number-of-files"]}} files will be processed
+ adding {%if ftype == "phenocovar"%}(possibly){%endif%}
+ {{summary[ftype]["total-data-rows"]}} new
+ {%if ftype == "phenocovar"%}
+ phenotypes
+ {%else%}
+ {{summary[ftype]["description"]}} rows
+ {%endif%}
+ to the database.
+ </li>
+ </ul>
+ {%endif%}
+ {%endfor%}
+
+ <a href="#" class="not-implemented btn btn-primary">continue</a>
+</div>
+{%else%}
+<div class="row">
+ <h4 class="subheading">Invalid Job</h3>
+ <p class="text-danger">
+ Could not find a job with the ID: <strong>{{job_id}}.</p>
+ <p>You will be redirected in
+ <span id="countdown-element" class="text-info">20</span> second(s)</p>
+ <p class="text-muted">
+ <small>
+ If you are not redirected, please
+ <a href="{{url_for(
+ 'species.populations.phenotypes.view_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">click here</a> to continue
+ </small>
+ </p>
+</div>
+{%endif%}
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_pheno_dataset_card(species, population, dataset)}}
+{%endblock%}
+
+
+{%block javascript%}
+<script type="text/javascript">
+ $(document).ready(function() {
+ var countdown = 20;
+ var countdown_element = $("#countdown-element");
+ if(countdown_element.length === 1) {
+ intv = window.setInterval(function() {
+ countdown = countdown - 1;
+ countdown_element.html(countdown);
+ }, 1000);
+ }
+ });
+</script>
+{%endblock%}
diff --git a/uploader/templates/phenotypes/select-population.html b/uploader/templates/phenotypes/select-population.html
new file mode 100644
index 0000000..cea0806
--- /dev/null
+++ b/uploader/templates/phenotypes/select-population.html
@@ -0,0 +1,31 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+{%from "populations/macro-select-population.html" import select_population_form%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+
+<div class="row">
+ {{select_population_form(url_for("species.populations.phenotypes.select_population", species_id=species.SpeciesId), species, populations)}}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/populations.js"></script>
+<script type="text/javascript">
+ $(function() {
+ populationDataTable(JSON.parse($("#tbl-select-population").attr("data-populations-list")));
+ });
+ </script>
+{%endblock%}
diff --git a/uploader/templates/phenotypes/view-dataset.html b/uploader/templates/phenotypes/view-dataset.html
new file mode 100644
index 0000000..c896214
--- /dev/null
+++ b/uploader/templates/phenotypes/view-dataset.html
@@ -0,0 +1,123 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="view-dataset"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.view_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">View</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <p>The basic dataset details are:</p>
+
+ <table class="table">
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Full Name</th>
+ <th>Short Name</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>{{dataset.Name}}</td>
+ <td>{{dataset.FullName}}</td>
+ <td>{{dataset.ShortName}}</td>
+ </tr>
+ </tbody>
+ </table>
+</div>
+
+<div class="row">
+ <p><a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}"
+ title="Add a bunch of phenotypes"
+ class="btn btn-primary">Add phenotypes</a></p>
+</div>
+
+<div class="row">
+ <h2>Phenotype Data</h2>
+
+ <p>Click on any of the phenotypes in the table below to view and edit that
+ phenotype's data.</p>
+ <p>Use the search to filter through all the phenotypes and find specific
+ phenotypes of interest.</p>
+
+ <table id="tbl-phenotypes-list" class="table compact stripe">
+ <thead>
+ <tr>
+ <th></th>
+ <th>Record</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+
+ <tbody></tbody>
+ </table>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
+
+
+{%block javascript%}
+<script type="text/javascript">
+ $(function() {
+ var data = {{phenotypes | tojson}};
+ $("#tbl-phenotypes-list").DataTable({
+ responsive: true,
+ lengthMenu: [10, 25, 50, 100, 1000, data.length],
+ language: {
+ processing: "Processing results… Please wait.",
+ loadingRecord: "Loading phenotypes — Please wait.",
+ info: "_START_ to _END_ of _TOTAL_ phenotypes",
+ lengthMenu: "Show _MENU_ phenotypes",
+ },
+ data: data,
+ columns: [
+ {data: "sequence_number"},
+ {
+ data: function(pheno) {
+ var spcs_id = {{species.SpeciesId}};
+ var pop_id = {{population.Id}};
+ var dtst_id = {{dataset.Id}};
+ return `<a href="/species/${spcs_id}` +
+ `/populations/${pop_id}` +
+ `/phenotypes/datasets/${dtst_id}` +
+ `/phenotype/${pheno.xref_id}` +
+ `" target="_blank">` +
+ `${pheno.InbredSetCode}_${pheno.xref_id}` +
+ `</a>`;
+ }
+ },
+ {data: function(pheno) {
+ return (pheno.Post_publication_description ||
+ pheno.Original_description ||
+ pheno.Pre_publication_description);
+ }}
+ ]
+ });
+ });
+</script>
+{%endblock%}
diff --git a/uploader/templates/phenotypes/view-phenotype.html b/uploader/templates/phenotypes/view-phenotype.html
new file mode 100644
index 0000000..21ac501
--- /dev/null
+++ b/uploader/templates/phenotypes/view-phenotype.html
@@ -0,0 +1,135 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="view-phenotype"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.view_phenotype',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ xref_id=xref_id)}}">View Phenotype</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <div class="panel panel-default">
+ <div class="panel-heading"><strong>Basic Phenotype Details</strong></div>
+
+ <table class="table">
+ <tbody>
+ <tr>
+ <td><strong>Phenotype</strong></td>
+ <td>{{phenotype.Post_publication_description or phenotype.Pre_publication_abbreviation or phenotype.Original_description}}
+ </tr>
+ <tr>
+ <td><strong>Database</strong></td>
+ <td>{{dataset.FullName}}</td>
+ </tr>
+ <tr>
+ <td><strong>Units</strong></td>
+ <td>{{phenotype.Units}}</td>
+ </tr>
+ {%for key,value in publish_data.items()%}
+ <tr>
+ <td><strong>{{key}}</strong></td>
+ <td>{{value}}</td>
+ </tr>
+ {%else%}
+ <tr>
+ <td colspan="2" class="text-muted">
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ No publication data found.
+ </td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+ </div>
+</div>
+
+{%if "group:resource:edit-resource" in privileges
+or "group:resource:delete-resource" in privileges%}
+<div class="row">
+ <div class="btn-group btn-group-justified">
+ <div class="btn-group">
+ {%if "group:resource:edit-resource" in privileges%}
+ <a href="{{url_for('species.populations.phenotypes.edit_phenotype_data',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ xref_id=xref_id)}}"
+ title="Edit the values for the phenotype. This is meant to be used when you need to update only a few values."
+ class="btn btn-primary">Edit</a>
+ {%endif%}
+ </div>
+ <div class="btn-group"></div>
+ <div class="btn-group">
+ {%if "group:resource:delete-resource" in privileges%}
+ <a href="#"
+ title="Delete the entire phenotype. This is useful when you need to change data for most or all of the fields for this phenotype."
+ class="btn btn-danger not-implemented"
+ disabled="disabled">delete</a>
+ {%endif%}
+ </div>
+ </div>
+</div>
+{%endif%}
+
+<div class="row">
+ <div class="panel panel-default">
+ <div class="panel-heading"><strong>Phenotype Data</strong></div>
+ {%if "group:resource:view-resource" in privileges%}
+ <table class="table">
+ <thead>
+ <tr>
+ <th>#</th>
+ <th>Sample</th>
+ <th>Value</th>
+ {%if has_se%}
+ <th>SE</th>
+ <th>N</th>
+ {%endif%}
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for item in phenotype.data%}
+ <tr>
+ <td>{{loop.index}}</td>
+ <td>{{item.StrainName}}</td>
+ <td>{{item.value}}</td>
+ {%if has_se%}
+ <td>{{item.error or "-"}}</td>
+ <td>{{item.count or "-"}}</td>
+ {%endif%}
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+ {%else%}
+ <p class="text-danger">
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ You do not currently have privileges to view this phenotype in greater
+ detail.
+ </p>
+ {%endif%}
+ </div>
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/platforms/base.html b/uploader/templates/platforms/base.html
new file mode 100644
index 0000000..dac965f
--- /dev/null
+++ b/uploader/templates/platforms/base.html
@@ -0,0 +1,13 @@
+{%extends "species/base.html"%}
+
+{%block lvl3_breadcrumbs%}
+<li {%if activelink=="platforms"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.platforms.index')}}">
+ Sequencing Platforms</a>
+</li>
+{%block lvl4_breadcrumbs%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/platforms/create-platform.html b/uploader/templates/platforms/create-platform.html
new file mode 100644
index 0000000..0866d5e
--- /dev/null
+++ b/uploader/templates/platforms/create-platform.html
@@ -0,0 +1,124 @@
+{%extends "platforms/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+
+{%block title%}Platforms &mdash; Create Platforms{%endblock%}
+
+{%block pagetitle%}Platforms &mdash; Create Platforms{%endblock%}
+
+{%block lvl3_breadcrumbs%}
+<li {%if activelink=="create-platform"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.platforms.create_platform',
+ species_id=species.SpeciesId)}}">create platform</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <h2>Create New Platform</h2>
+
+ <p>You can create a new genetic sequencing platform below.</p>
+</div>
+
+<div class="row">
+ <form id="frm-create-platform"
+ method="POST"
+ action="{{url_for('species.platforms.create_platform',
+ species_id=species.SpeciesId)}}">
+
+ <div class="form-group">
+ <label for="txt-geo-platform" class="form-label">GEO Platform</label>
+ <input type="text"
+ id="txt-geo-platform"
+ name="geo-platform"
+ required="required"
+ class="form-control" />
+ <small class="form-text text-muted">
+ <p>This is the platform's
+ <a href="https://www.ncbi.nlm.nih.gov/geo/browse/?view=platforms&tax={{species.TaxonomyId}}"
+ title="Platforms for '{{species.FullName}}' on NCBI">
+ accession value on NCBI</a>. If you do not know the value, click the
+ link and search on NCBI for species '{{species.FullName}}'.</p></small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-platform-name" class="form-label">Platform Name</label>
+ <input type="text"
+ id="txt-platform-name"
+ name="platform-name"
+ required="required"
+ class="form-control" />
+ <small class="form-text text-muted">
+ <p>This is name of the genetic sequencing platform.</p></small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-platform-shortname" class="form-label">
+ Platform Short Name</label>
+ <input type="text"
+ id="txt-platform-shortname"
+ name="platform-shortname"
+ required="required"
+ class="form-control" />
+ <small class="form-text text-muted">
+ <p>Use the following conventions for this field:
+ <ol>
+ <li>Start with a 4-letter vendor code, e.g. "Affy" for "Affymetrix", "Illu" for "Illumina", etc.</li>
+ <li>Append an underscore to the 4-letter vendor code</li>
+ <li>Use the name of the array given by the vendor, e.g. U74AV2, MOE430A, etc.</li>
+ </ol>
+ </p>
+ </small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-platform-title" class="form-label">Platform Title</label>
+ <input type="text"
+ id="txt-platform-title"
+ name="platform-title"
+ required="required"
+ class="form-control" />
+ <small class="form-text text-muted">
+ <p>The full platform title. Sometimes, this is the same as the Platform
+ Name above.</p></small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-go-tree-value" class="form-label">GO Tree Value</label>
+ <input type="text"
+ id="txt-go-tree-value"
+ name="go-tree-value"
+ class="form-control" />
+ <small class="form-text text-muted">
+ <p>This is a Chip identification value useful for analysis with the
+ <strong>
+ <a href="https://www.geneweaver.org/"
+ title="Go to the GeneWeaver site."
+ target="_blank">GeneWeaver</a></strong>
+ and
+ <strong>
+ <a href="https://www.webgestalt.org/"
+ title="Go to the WEB-based GEne SeT AnaLysis Toolkit site."
+ target="_blank">WebGestalt</a></strong>
+ tools.<br />
+ This can be left blank for custom platforms.</p></small>
+ </div>
+
+ <div class="form-group">
+ <input type="submit"
+ value="create new platform"
+ class="btn btn-primary" />
+ </div>
+ </form>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
diff --git a/uploader/templates/platforms/index.html b/uploader/templates/platforms/index.html
new file mode 100644
index 0000000..555b444
--- /dev/null
+++ b/uploader/templates/platforms/index.html
@@ -0,0 +1,25 @@
+{%extends "platforms/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-select-species.html" import select_species_form%}
+
+{%block title%}Platforms{%endblock%}
+
+{%block pagetitle%}Platforms{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <p>In this section, you will be able to view and manage the sequencing
+ platforms that are currently supported by GeneNetwork.</p>
+</div>
+
+<div class="row">
+ {{select_species_form(url_for("species.platforms.index"), species)}}
+</div>
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/species.js"></script>
+{%endblock%}
diff --git a/uploader/templates/platforms/list-platforms.html b/uploader/templates/platforms/list-platforms.html
new file mode 100644
index 0000000..a6bcfdc
--- /dev/null
+++ b/uploader/templates/platforms/list-platforms.html
@@ -0,0 +1,93 @@
+{%extends "platforms/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+
+{%block title%}Platforms &mdash; List Platforms{%endblock%}
+
+{%block pagetitle%}Platforms &mdash; List Platforms{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <p>View the list of the genetic sequencing platforms that are currently
+ supported by GeneNetwork.</p>
+ <p>If you cannot find the platform you wish to use, you can add it by clicking
+ the "New Platform" button below.</p>
+ <p><a href="{{url_for('species.platforms.create_platform',
+ species_id=species.SpeciesId)}}"
+ title="Create a new genetic sequencing platform for species {{species.FullName}}"
+ class="btn btn-primary">Create Platform</a></p>
+</div>
+
+<div class="row">
+ <h2>Supported Platforms</h2>
+ {%if platforms is defined and platforms | length > 0%}
+ <p>There are {{total_platforms}} platforms supported by GeneNetwork</p>
+
+ <div class="row">
+ <div class="col-md-2" style="text-align: start;">
+ {%if start_from > 0%}
+ <a href="{{url_for('species.platforms.list_platforms',
+ species_id=species.SpeciesId,
+ start_from=start_from-count,
+ count=count)}}">
+ <span class="glyphicon glyphicon-backward"></span>
+ Previous
+ </a>
+ {%endif%}
+ </div>
+ <div class="col-md-8" style="text-align: center;">
+ Displaying platforms {{start_from+1}} to {{start_from+count if start_from+count < total_platforms else total_platforms}} of
+ {{total_platforms}}
+ </div>
+ <div class="col-md-2" style="text-align: end;">
+ {%if start_from + count < total_platforms%}
+ <a href="{{url_for('species.platforms.list_platforms',
+ species_id=species.SpeciesId,
+ start_from=start_from+count,
+ count=count)}}">
+ Next
+ <span class="glyphicon glyphicon-forward"></span>
+ </a>
+ {%endif%}
+ </div>
+ </div>
+
+ <table class="table">
+ <thead>
+ <tr>
+ <th></th>
+ <th>Platform Name</th>
+ <th><a href="https://www.ncbi.nlm.nih.gov/geo/browse/?view=platforms&tax={{species.TaxonomyId}}"
+ title="Gene Expression Omnibus: Platforms section"
+ target="_blank">GEO Platform</a></th>
+ <th>Title</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for platform in platforms%}
+ <tr>
+ <td>{{platform.sequence_number}}</td>
+ <td>{{platform.GeneChipName}}</td>
+ <td><a href="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc={{platform.GeoPlatform}}"
+ title="View platform on the Gene Expression Omnibus"
+ target="_blank">{{platform.GeoPlatform}}</a></td>
+ <td>{{platform.Title}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+ {%else%}
+ <p class="text-warning">
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ There are no platforms supported at this time!</p>
+ {%endif%}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
diff --git a/uploader/templates/populations/base.html b/uploader/templates/populations/base.html
new file mode 100644
index 0000000..9db8083
--- /dev/null
+++ b/uploader/templates/populations/base.html
@@ -0,0 +1,18 @@
+{%extends "species/base.html"%}
+
+{%block lvl2_breadcrumbs%}
+<li {%if activelink=="populations"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ {%if population is mapping%}
+ <a href="{{url_for('species.populations.view_population',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">{{population.Name}}</a>
+ {%else%}
+ <a href="{{url_for('species.populations.index')}}">Populations</a>
+ {%endif%}
+</li>
+{%block lvl3_breadcrumbs%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/populations/create-population.html b/uploader/templates/populations/create-population.html
new file mode 100644
index 0000000..c0c4f45
--- /dev/null
+++ b/uploader/templates/populations/create-population.html
@@ -0,0 +1,258 @@
+{%extends "populations/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-select-species.html" import select_species_form%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+
+{%block title%}Create Population{%endblock%}
+
+{%block pagetitle%}Create Population{%endblock%}
+
+{%block lvl3_breadcrumbs%}
+<li {%if activelink=="create-population"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.create_population',
+ species_id=species.SpeciesId)}}">create population</a>
+</li>
+{%endblock%}
+
+
+{%block contents%}
+<div class="row">
+ <p>The population is the next hierarchical node under Species. Data is grouped under a specific population, under a particular species.</p>
+ <p>
+ This page enables you to create a new population, in the case that you
+ cannot find the population you want in the
+ <a
+ href="{{url_for('species.populations.list_species_populations',
+ species_id=species.SpeciesId)}}"
+ title="Population for species '{{species.FullName}}'.">
+ list of species populations
+ </a>
+ </p>
+</div>
+
+<div class="row">
+ <form method="POST"
+ action="{{url_for('species.populations.create_population',
+ species_id=species.SpeciesId,
+ return_to=return_to)}}">
+
+ <legend>Create Population</legend>
+
+ {{flash_all_messages()}}
+
+ <input type="hidden" name="return_to" value="{{return_to}}">
+
+ <div {%if errors.population_fullname%}
+ class="form-group has-error"
+ {%else%}
+ class="form-group"
+ {%endif%}>
+ <label for="txt-population-fullname" class="form-label">Full Name</label>
+ {%if errors.population_fullname%}
+ <small class="form-text text-danger">{{errors.population_fullname}}</small>
+ {%endif%}
+ <input type="text"
+ id="txt-population-fullname"
+ name="population_fullname"
+ required="required"
+ minLength="3"
+ maxLength="100"
+ value="{{error_values.population_fullname or ''}}"
+ class="form-control" />
+ <small class="form-text text-muted">
+ <p>
+ This is a descriptive name for your population &mdash; useful for
+ humans.
+ </p>
+ </small>
+ </div>
+
+ <div {%if errors.population_name%}
+ class="form-group has-error"
+ {%else%}
+ class="form-group"
+ {%endif%}>
+ <label for="txt-population-name" class="form-label">Name</label>
+ {%if errors.population_name%}
+ <small class="form-text text-danger">{{errors.population_name}}</small>
+ {%endif%}
+ <input type="text"
+ id="txt-population-name"
+ name="population_name"
+ required="required"
+ minLength="3"
+ maxLength="30"
+ value="{{error_values.population_name or ''}}"
+ class="form-control" />
+ <small class="form-text text-muted">
+ <p>
+ This is a short representative, but constrained name for your
+ population.
+ <br />
+ The field will only accept letters ('A-Za-z'), numbers (0-9), hyphens
+ and underscores. Any other character will cause the name to be
+ rejected.
+ </p>
+ </small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-population-code" class="form-label">Population Code</label>
+ <input type="text"
+ id="txt-population-code"
+ name="population_code"
+ maxLength="5"
+ minLength="3"
+ value="{{error_values.population_code or ''}}"
+ class="form-control" />
+ <small class="form-text text-muted">
+ <p class="form-text text-muted">
+ This is a 3-character code for your population, that is prepended to
+ the phenotype identifiers. e.g. For the "BXD Family" population, the
+ code is "BXD" and therefore, the phenotype identifiers for the
+ population look like the following examples: <em>BXD_10148</em>,
+ <em>BXD_10180</em>, <em>BXD_10197</em>, etc.
+ </p>
+ </small>
+ </div>
+
+ <div {%if errors.population_description%}
+ class="form-group has-error"
+ {%else%}
+ class="form-group"
+ {%endif%}>
+ <label for="txt-population-description" class="form-label">
+ Description
+ </label>
+ {%if errors.population_description%}
+ <small class="form-text text-danger">{{errors.population_description}}</small>
+ {%endif%}
+ <textarea
+ id="txt-population-description"
+ name="population_description"
+ required="required"
+ class="form-control"
+ rows="5">{{error_values.population_description or ''}}</textarea>
+ <small class="form-text text-muted">
+ <p>
+ This is a more detailed description for your population. This is
+ useful to communicate with other researchers some details regarding
+ your population, and what its about.
+ <br />
+ Put, here, anything that describes your population but does not go
+ cleanly under metadata.
+ </p>
+ </small>
+ </div>
+
+ <div {%if errors.population_family%}
+ class="form-group has-error"
+ {%else%}
+ class="form-group"
+ {%endif%}>
+ <label for="select-population-family" class="form-label">Family</label>
+ <select id="select-population-family"
+ name="population_family"
+ class="form-control"
+ required="required">
+ <option value="">Please select a family</option>
+ {%for family in families%}
+ <option value="{{family}}"
+ {%if error_values.population_family == family%}
+ selected="selected"
+ {%endif%}>{{family}}</option>
+ {%endfor%}
+ </select>
+ <small class="form-text text-muted">
+ <p>
+ This is a rough grouping of the populations in GeneNetwork into lists
+ of common types of populations.
+ </p>
+ </small>
+ </div>
+
+ <div {%if errors.population_mapping_method_id%}
+ class="form-group has-error"
+ {%else%}
+ class="form-group"
+ {%endif%}>
+ <label for="select-population-mapping-methods"
+ class="form-label">Mapping Methods</label>
+
+ <select id="select-population-mapping-methods"
+ name="population_mapping_method_id"
+ class="form-control"
+ required="required">
+ <option value="">Select appropriate mapping methods</option>
+ {%for mmethod in mapping_methods%}
+ <option value="{{mmethod.id}}"
+ {%if error_values.population_mapping_method_id == mmethod.id%}
+ selected="selected"
+ {%endif%}>{{mmethod.value}}</option>
+ {%endfor%}
+ </select>
+
+ <small class="form-text text-muted">
+ <p>Select the mapping methods that your population will support.</p>
+ </small>
+ </div>
+
+ <div {%if errors.population_genetic_type%}
+ class="form-group has-error"
+ {%else%}
+ class="form-group"
+ {%endif%}>
+ <label for="select-population-genetic-type"
+ class="form-label">Genetic Type</label>
+ <select id="select-population-genetic-type"
+ name="population_genetic_type"
+ class="form-control">
+ <option value="">Select proper genetic type</option>
+ {%for gtype in genetic_types%}
+ <option value="{{gtype}}"
+ {%if error_values.population_genetic_type == gtype%}
+ selected="selected"
+ {%endif%}>{{gtype}}</option>
+ {%endfor%}
+ </select>
+ <small class="form-text text-muted text-danger">
+ <p>
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ This might be a poorly named field.
+ </p>
+ <p>
+ It probably has more to do with the mating crosses/crossings used to
+ produce the individuals in the population. I am no biologist, however,
+ and I'm leaving this here to remind myself to confirm this.
+ </p>
+ <p>
+ I still don't know what riset is.<br />
+ … probably something to do with Recombinant Inbred Strains
+ </p>
+ <p>
+ Possible resources for this:
+ <ul>
+ <li>https://www.informatics.jax.org/silver/chapters/3-2.shtml</li>
+ <li>https://www.informatics.jax.org/silver/chapters/9-2.shtml</li>
+ </ul>
+ </p>
+ </small>
+ </div>
+
+ <div class="form-group">
+ <input type="submit"
+ value="create population"
+ class="btn btn-primary" />
+ </div>
+
+ </form>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
diff --git a/uploader/templates/populations/index.html b/uploader/templates/populations/index.html
new file mode 100644
index 0000000..d2bee77
--- /dev/null
+++ b/uploader/templates/populations/index.html
@@ -0,0 +1,28 @@
+{%extends "populations/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-select-species.html" import select_species_form%}
+
+{%block title%}Populations{%endblock%}
+
+{%block pagetitle%}Populations{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <p>
+ Your experiment data will relate to a particular population from a
+ particular species. Let us know what species it is you want to work with
+ below.
+ </p>
+</div>
+
+<div class="row">
+ {{select_species_form(url_for("species.populations.index"), species)}}
+</div>
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/species.js"></script>
+{%endblock%}
diff --git a/uploader/templates/populations/list-populations.html b/uploader/templates/populations/list-populations.html
new file mode 100644
index 0000000..f780e94
--- /dev/null
+++ b/uploader/templates/populations/list-populations.html
@@ -0,0 +1,93 @@
+{%extends "populations/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-select-species.html" import select_species_form%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+
+{%block title%}Populations{%endblock%}
+
+{%block pagetitle%}Populations{%endblock%}
+
+{%block lvl3_breadcrumbs%}
+<li {%if activelink=="list-populations"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.list_species_populations',
+ species_id=species.SpeciesId)}}">List populations</a>
+</li>
+{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+<div class="row">
+ <p>
+ The following populations/groups exist for the '{{species.FullName}}'
+ species.
+ </p>
+ <p>
+ Click on the population's name to select and continue using the population.
+ </p>
+</div>
+
+<div class="row">
+ <p>
+ If the population you need for the species '{{species.FullName}}' does not
+ exist, click on the "Create Population" button below to create a new one.
+ </p>
+ <p>
+ <a href="{{url_for('species.populations.create_population',
+ species_id=species.SpeciesId)}}"
+ title="Create a new population for species '{{species.FullName}}'."
+ class="btn btn-danger">
+ Create Population
+ </a>
+ </p>
+</div>
+
+<div class="row">
+ <table class="table">
+ <caption>Populations for {{species.FullName}}</caption>
+ <thead>
+ <tr>
+ <th></th>
+ <th>Name</th>
+ <th>Full Name</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for population in populations%}
+ <tr>
+ <td>{{population["sequence_number"]}}</td>
+ <td>
+ <a href="{{url_for('species.populations.view_population',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ title="Population '{{population.FullName}}' for species '{{species.FullName}}'.">
+ {{population.Name}}
+ </a>
+ </td>
+ <td>{{population.FullName}}</td>
+ <td>{{population.Description}}</td>
+ </tr>
+ {%else%}
+ <tr>
+ <td colspan="3">
+ <p class="text-danger">
+ <span class="glyphicon glyphicon-exclamation-mark"></span>
+ There were no populations found for {{species.FullName}}!
+ </p>
+ </td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
diff --git a/uploader/templates/populations/macro-display-population-card.html b/uploader/templates/populations/macro-display-population-card.html
new file mode 100644
index 0000000..16b477f
--- /dev/null
+++ b/uploader/templates/populations/macro-display-population-card.html
@@ -0,0 +1,41 @@
+{%from "species/macro-display-species-card.html" import display_species_card%}
+
+{%macro display_population_card(species, population)%}
+{{display_species_card(species)}}
+
+<div class="card">
+ <div class="card-body">
+ <h5 class="card-title">Population</h5>
+ <div class="card-text">
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Name</td>
+ <td>{{population.Name}}</td>
+ </tr>
+
+ <tr>
+ <td>Full Name</td>
+ <td>{{population.FullName}}</td>
+ </tr>
+
+ <tr>
+ <td>Code</td>
+ <td>{{population.InbredSetCode}}</td>
+ </tr>
+
+ <tr>
+ <td>Genetic Type</td>
+ <td>{{population.GeneticType}}</td>
+ </tr>
+
+ <tr>
+ <td>Family</td>
+ <td>{{population.Family}}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+</div>
+{%endmacro%}
diff --git a/uploader/templates/populations/macro-select-population.html b/uploader/templates/populations/macro-select-population.html
new file mode 100644
index 0000000..14b0510
--- /dev/null
+++ b/uploader/templates/populations/macro-select-population.html
@@ -0,0 +1,52 @@
+{%from "macro-step-indicator.html" import step_indicator%}
+
+{%macro select_population_form(form_action, species, populations)%}
+<form method="GET" action="{{form_action}}" class="form-horizontal">
+
+ <h2>{{step_indicator("2")}} What population do you want to work with?</h2>
+
+ {%if populations | length != 0%}
+
+ <p class="form-text">Search for, and select the population from the table
+ below and click "Continue"</p>
+
+ <div class="radio">
+ <label class="control-label" for="rdo-cant-find-population">
+ <input type="radio" id="rdo-cant-find-population"
+ name="population_id" value="CREATE-POPULATION" />
+ I cannot find the population I want &mdash; create it!
+ </label>
+ </div>
+
+ <div class="col-sm-offset-10 col-sm-2">
+ <input type="submit" value="continue" class="btn btn-primary" />
+ </div>
+
+ <div style="margin-top:3em;">
+ <table id="tbl-select-population" class="table compact stripe"
+ data-populations-list='{{populations | tojson}}'>
+ <thead>
+ <tr>
+ <th></th>
+ <th>Population</th>
+ </tr>
+ </thead>
+
+ <tbody></tbody>
+ </table>
+ </div>
+
+ {%else%}
+ <p class="form-text">
+ There are no populations currently defined for {{species['FullName']}}
+ ({{species['SpeciesName']}}).<br />
+ Click "Continue" to create the first!</p>
+ <input type="hidden" name="population_id" value="CREATE-POPULATION" />
+
+ <div class="col-sm-offset-10 col-sm-2">
+ <input type="submit" value="continue" class="btn btn-primary" />
+ </div>
+ {%endif%}
+
+</form>
+{%endmacro%}
diff --git a/uploader/templates/populations/rqtl2/create-tissue-success.html b/uploader/templates/populations/rqtl2/create-tissue-success.html
new file mode 100644
index 0000000..d6fe154
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/create-tissue-success.html
@@ -0,0 +1,106 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Select Tissue</h2>
+
+<div class="row">
+ <p>You have successfully added a new tissue, organ or biological material with
+ the following details:</p>
+</div>
+
+<div class="row">
+ {{flash_all_messages()}}
+
+ <form id="frm-create-tissue-display"
+ method="POST"
+ action="#">
+ <legend class="heading">Create Tissue</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+
+ <div class="form-group">
+ <label>Name</label>
+ <label>{{tissue.TissueName}}</label>
+ </div>
+
+ <div class="form-group">
+ <label>Short Name</label>
+ <label>{{tissue.Short_Name}}</label>
+ </div>
+
+ {%if tissue.BIRN_lex_ID%}
+ <div class="form-group">
+ <label>BIRN Lex ID</label>
+ <label>{{tissue.BIRN_lex_ID}}</label>
+ </div>
+ {%endif%}
+
+ {%if tissue.BIRN_lex_Name%}
+ <div class="form-group">
+ <label>BIRN Lex Name</label>
+ <label>{{tissue.BIRN_lex_Name}}</label>
+ </div>
+ {%endif%}
+ </form>
+
+ <div id="action-buttons"
+ style="width:65ch;display:inline-grid;column-gap:5px;">
+
+ <form id="frm-create-tissue-success-continue"
+ method="POST"
+ action="{{url_for('expression-data.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ style="display: inline; width: 100%; grid-column: 1 / 2;
+ padding-top: 0.5em; text-align: center; border: none;
+ background-color: inherit;">
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+ </div>
+</div>
+
+<div class="row">
+ <p style="display:inline;width:100%;grid-column:2/3;text-align:center;
+ color:#336699;font-weight:bold;">
+ OR
+ </p>
+</div>
+
+<div class="row">
+ <form id="frm-create-tissue-success-select-existing"
+ method="POST"
+ action="{{url_for('expression-data.rqtl2.select_tissue',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ style="display: inline; width: 100%; grid-column: 3 / 4;
+ padding-top: 0.5em; text-align: center; border: none;
+ background-color: inherit;">
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+
+ <button type="submit" class="btn btn-primary">
+ select from existing tissues</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/index.html b/uploader/templates/populations/rqtl2/index.html
new file mode 100644
index 0000000..ec6ffb8
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/index.html
@@ -0,0 +1,54 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Data Upload{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 data upload</h1>
+
+<h2>R/qtl2 Upload</h2>
+
+<div class="row">
+ <form method="POST" action="{{url_for('expression-data.rqtl2.select_species')}}"
+ id="frm-rqtl2-upload">
+ <legend class="heading">upload R/qtl2 bundle</legend>
+ {{flash_messages("error-rqtl2")}}
+
+ <div class="form-group">
+ <label for="select:species" class="form-label">Species</label>
+ <select id="select:species"
+ name="species_id"
+ required="required"
+ class="form-control">
+ <option value="">Select species</option>
+ {%for spec in species%}
+ <option value="{{spec.SpeciesId}}">{{spec.MenuName}}</option>
+ {%endfor%}
+ </select>
+ <small class="form-text text-muted">
+ Data that you upload to the system should belong to a know species.
+ Here you can select the species that you wish to upload data for.
+ </small>
+ </div>
+
+ <input type="submit" class="btn btn-primary" value="submit" />
+ </form>
+</div>
+
+<div class="row">
+ <h2 class="heading">R/qtl2 Bundles</h2>
+
+ <div class="explainer">
+ <p>This feature combines and extends the two upload methods below. Instead of
+ uploading one item at a time, the R/qtl2 bundle you upload can contain both
+ the genotypes data (samples/individuals/cases and their data) and the
+ expression data.</p>
+ <p>The R/qtl2 bundle, additionally, can contain extra metadata, that neither
+ of the methods below can handle.</p>
+
+ <a href="{{url_for('expression-data.rqtl2.select_species')}}"
+ title="Upload a zip bundle of R/qtl2 files">
+ <button class="btn btn-primary">upload R/qtl2 bundle</button></a>
+ </div>
+</div>
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/no-such-job.html b/uploader/templates/populations/rqtl2/no-such-job.html
new file mode 100644
index 0000000..b17004f
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/no-such-job.html
@@ -0,0 +1,13 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 job status</h1>
+
+<h2>R/qtl2 Upload: No Such Job</h2>
+
+<p class="alert-danger">No job with ID {{jobid}} was found.</p>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/rqtl2-job-error.html b/uploader/templates/populations/rqtl2/rqtl2-job-error.html
new file mode 100644
index 0000000..9817518
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/rqtl2-job-error.html
@@ -0,0 +1,39 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 job status</h1>
+
+<h2>R/qtl2 Upload: Job Status</h2>
+
+<div class="explainer">
+ <p>The processing of the R/qtl2 bundle you uploaded has failed. We have
+ provided some information below to help you figure out what the problem
+ could be.</p>
+ <p>If you find that you cannot figure out what the problem is on your own,
+ please contact the team running the system for assistance, providing the
+ following details:
+ <ul>
+ <li>R/qtl2 bundle you uploaded</li>
+ <li>This URL: <strong>{{request_url()}}</strong></li>
+ <li>(maybe) a screenshot of this page</li>
+ </ul>
+ </p>
+</div>
+
+<h4>stdout</h4>
+{{cli_output(job, "stdout")}}
+
+<h4>stderr</h4>
+{{cli_output(job, "stderr")}}
+
+<h4>Log</h4>
+<div class="cli-output">
+ {%for msg in messages%}
+ {{msg}}<br />
+ {%endfor%}
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/rqtl2-job-results.html b/uploader/templates/populations/rqtl2/rqtl2-job-results.html
new file mode 100644
index 0000000..4ecd415
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/rqtl2-job-results.html
@@ -0,0 +1,24 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 job status</h1>
+
+<h2>R/qtl2 Upload: Job Status</h2>
+
+<div class="explainer">
+ <p>The processing of the R/qtl2 bundle you uploaded has completed
+ successfully.</p>
+ <p>You should now be able to use GeneNetwork to run analyses on your data.</p>
+</div>
+
+<h4>Log</h4>
+<div class="cli-output">
+ {%for msg in messages%}
+ {{msg}}<br />
+ {%endfor%}
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/rqtl2-job-status.html b/uploader/templates/populations/rqtl2/rqtl2-job-status.html
new file mode 100644
index 0000000..e896f88
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/rqtl2-job-status.html
@@ -0,0 +1,20 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="3">
+{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 job status</h1>
+
+<h2>R/qtl2 Upload: Job Status</h2>
+
+<h4>Log</h4>
+<div class="cli-output">
+ <pre>{{"\n".join(messages)}}</pre>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/rqtl2-qc-job-error.html b/uploader/templates/populations/rqtl2/rqtl2-qc-job-error.html
new file mode 100644
index 0000000..90e8887
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/rqtl2-qc-job-error.html
@@ -0,0 +1,120 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}R/qtl2 bundle: QC Job Error{%endblock%}
+
+{%macro errors_table(tableid, errors)%}
+<table id="{{tableid}}" class="table error-table">
+ <caption>{{caption}}</caption>
+ <thead>
+ <tr>
+ <th>Line</th>
+ <th>Column</th>
+ <th>Value</th>
+ <th>Message</th>
+ </tr>
+ </thead>
+ <tbody>
+ {%for error in errors%}
+ <tr>
+ <td>{{error.line}}</td>
+ <td>{{error.column}}</td>
+ <td>{{error.value}}</td>
+ <td>{{error.message}}</td>
+ </tr>
+ {%else%}
+ <tr>
+ <td colspan="4">No errors to display here.</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+</table>
+{%endmacro%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 bundle: QC job Error</h1>
+
+<div class="explainer">
+ <p>The R/qtl2 bundle has failed some <emph>Quality Control</emph> checks.</p>
+ <p>We list below some of the errors that need to be fixed before the data can
+ be uploaded onto GeneNetwork.</p>
+</div>
+
+{%if errorsgeneric | length > 0%}
+<h2 class="heading">Generic Errors ({{errorsgeneric | length}})</h3>
+<div class="explainer">
+ We found the following generic errors in your R/qtl2 bundle:
+</div>
+
+<h3>Missing Files</h3>
+<div class="explainer">
+ <p>These files are listed in the bundle's control file, but do not actually
+ exist in the bundle</p>
+</div>
+<table id="tbl-errors-missing-files" class="table error-table">
+ <thead>
+ <tr>
+ <th>Control File Key</th>
+ <th>Bundle File Name</th>
+ <th>Message</th>
+ </tr>
+ </thead>
+ <tbody>
+ {%for error in (errorsgeneric | selectattr("type", "equalto", "MissingFile"))%}
+ <tr>
+ <td>{{error.controlfilekey}}</td>
+ <td>{{error.filename}}</td>
+ <td>{{error.message}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+</table>
+
+<h3>Other Generic Errors</h3>
+{{errors_table("tbl-errors-generic", errorsgeneric| selectattr("type", "ne", "MissingFile"))}}
+{%endif%}
+
+{%if errorsgeno | length > 0%}
+<h2 class="heading">Geno Errors ({{errorsgeno | length}})</h3>
+<div class="explainer">
+ We found the following errors in the 'geno' file in your R/qtl2 bundle:
+</div>
+{{errors_table("tbl-errors-geno", errorsgeno[0:50])}}
+{%endif%}
+
+{%if errorspheno | length > 0%}
+<h2 class="heading">Pheno Errors ({{errorspheno | length}})</h3>
+<div class="explainer">
+ We found the following errors in the 'pheno' file in your R/qtl2 bundle:
+</div>
+{{errors_table("tbl-errors-pheno", errorspheno[0:50])}}
+{%endif%}
+
+{%if errorsphenose | length > 0%}
+<h2 class="heading">Phenose Errors ({{errorsphenose | length}})</h3>
+<div class="explainer">
+ We found the following errors in the 'phenose' file in your R/qtl2 bundle:
+</div>
+{{errors_table("tbl-errors-phenose", errorsphenose[0:50])}}
+{%endif%}
+
+{%if errorsphenocovar | length > 0%}
+<h2 class="heading">Phenocovar Errors ({{errorsphenocovar | length}})</h3>
+<div class="explainer">
+ We found the following errors in the 'phenocovar' file in your R/qtl2 bundle:
+</div>
+{{errorsphenocovar}}
+{%endif%}
+
+<h4>stdout</h4>
+{{cli_output(job, "stdout")}}
+
+<h4>stderr</h4>
+{{cli_output(job, "stderr")}}
+
+<h4>Log</h4>
+<div class="cli-output">
+ <pre>{{"\n".join(messages)}}</pre>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/rqtl2-qc-job-results.html b/uploader/templates/populations/rqtl2/rqtl2-qc-job-results.html
new file mode 100644
index 0000000..b3c3a8f
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/rqtl2-qc-job-results.html
@@ -0,0 +1,66 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}R/qtl2 bundle: QC job results{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 bundle: QC job results</h1>
+
+<div class="row">
+ <p>The R/qtl2 bundle you uploaded has passed all automated quality-control
+ checks successfully.</p>
+ <p>You may now continue to load the data into GeneNetwork for the bundle, with
+ the following details:</p>
+</div>
+
+<div class="row">
+ <form id="form-qc-job-results"
+ action="{{url_for('expression-data.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ method="POST">
+ <div class="form-group">
+ <legend>Species</legend>
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+
+ <span class="form-label">Name</span>
+ <span class="form-text">{{species.Name | capitalize}}</span>
+
+ <span class="form-label">Scientific</span>
+ <span class="form-text">{{species.FullName | capitalize}}</span>
+ </div>
+
+ <div class="form-group">
+ <legend>population</legend>
+ <input type="hidden" name="population_id" value="{{population.Id}}" />
+
+ <span class="form-label">Name</span>
+ <span class="form-text">{{population.InbredSetName}}</span>
+
+ <span class="form-label">Full Name</span>
+ <span class="form-text">{{population.FullName}}</span>
+
+ <span class="form-label">Genetic Type</span>
+ <span class="form-text">{{population.GeneticType}}</span>
+
+ <span class="form-label">Description</span>
+ <span class="form-text">{{population.Description or "-"}}</span>
+ </div>
+
+ <div class="form-group">
+ <legend>R/qtl2 Bundle File</legend>
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2bundle}}" />
+ <input type="hidden" name="original-filename" value="{{rqtl2bundleorig}}" />
+
+ <span class="form-label">Original Name</span>
+ <span class="form-text">{{rqtl2bundleorig}}</span>
+
+ <span class="form-label">Internal Name</span>
+ <span class="form-text">{{rqtl2bundle[0:25]}}&hellip;</span>
+ </div>
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/rqtl2-qc-job-status.html b/uploader/templates/populations/rqtl2/rqtl2-qc-job-status.html
new file mode 100644
index 0000000..f4a6266
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/rqtl2-qc-job-status.html
@@ -0,0 +1,41 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="3">
+{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 bundle: QC job status</h1>
+
+{%if geno_percent%}
+<p>
+ <h2>Checking 'geno' file:</h2>
+ <progress id="prg-geno-checking" value="{{geno_percent}}" max="100">
+ {{geno_percent}}%</progress>
+ {{geno_percent}}%</p>
+{%endif%}
+
+{%if pheno_percent%}
+<p>
+ <h2>Checking 'pheno' file:</h2>
+ <progress id="prg-pheno-checking" value="{{pheno_percent}}" max="100">
+ {{pheno_percent}}%</progress>
+ {{pheno_percent}}%</p>
+{%endif%}
+
+{%if phenose_percent%}
+<p>
+ <h2>Checking 'phenose' file:</h2>
+ <progress id="prg-phenose-checking" value="{{phenose_percent}}" max="100">
+ {{phenose_percent}}%</progress>
+ {{phenose_percent}}%</p>
+{%endif%}
+
+<h4>Log</h4>
+<div class="cli-output">
+ <pre>{{"\n".join(messages)}}</pre>
+</div>
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/rqtl2-qc-job-success.html b/uploader/templates/populations/rqtl2/rqtl2-qc-job-success.html
new file mode 100644
index 0000000..f126835
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/rqtl2-qc-job-success.html
@@ -0,0 +1,37 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}R/qtl2 Bundle: Quality Control Successful{%endblock%}
+
+{%block contents%}
+<h2 class="heading">R/qtl2 Bundle: Quality Control Successful</h2>
+
+<div class="row">
+ <p>The R/qtl2 bundle you uploaded has passed <emph>all</emph> quality control
+ checks successfully, and is now ready for uploading into the database.</p>
+ <p>Click "Continue" below to proceed.</p>
+</div>
+
+<!--
+ The "action" on this form takes us to the next step, where we can
+ select all the other data necessary to enter the data into the database.
+ -->
+<div class="row">
+ <form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('expression-data.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ {{flash_all_messages()}}
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/select-geno-dataset.html b/uploader/templates/populations/rqtl2/select-geno-dataset.html
new file mode 100644
index 0000000..3233abc
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/select-geno-dataset.html
@@ -0,0 +1,69 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Select Genotypes Dataset</h2>
+
+<div class="row">
+ <p>Your R/qtl2 files bundle could contain a "geno" specification. You will
+ therefore need to select from one of the existing Genotype datasets or
+ create a new one.</p>
+ <p>This is the dataset where your data will be organised under.</p>
+</div>
+
+<div class="row">
+ <form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('expression-data.rqtl2.select_geno_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ <legend class="heading">select from existing genotype datasets</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+
+ {{flash_messages("error-rqtl2-select-geno-dataset")}}
+
+ <div class="form-group">
+ <legend>Datasets</legend>
+ <label for="select:geno-datasets" class="form-label">Dataset</label>
+ <select id="select:geno-datasets"
+ name="geno-dataset-id"
+ required="required"
+ {%if datasets | length == 0%}
+ disabled="disabled"
+ {%endif%}
+ class="form-control"
+ aria-describedby="help-geno-dataset-select-dataset">
+ <option value="">Select dataset</option>
+ {%for dset in datasets%}
+ <option value="{{dset['Id']}}">{{dset["Name"]}} ({{dset["FullName"]}})</option>
+ {%endfor%}
+ </select>
+ <span id="help-geno-dataset-select-dataset" class="form-text text-muted">
+ Select from the existing genotype datasets for species
+ {{species.SpeciesName}} ({{species.FullName}}).
+ </span>
+ </div>
+
+ <button type="submit" class="btn btn-primary">select dataset</button>
+ </form>
+</div>
+
+<div class="row">
+ <p>If the genotype dataset you need does not currently exist for your dataset,
+ go the <a href="{{url_for(
+ 'species.populations.genotypes.create_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ title="Create a new genotypes dataset for {{species.FullName}}">
+ genotypes page to create the genotype dataset</a></p>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/select-population.html b/uploader/templates/populations/rqtl2/select-population.html
new file mode 100644
index 0000000..ded425f
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/select-population.html
@@ -0,0 +1,57 @@
+{%extends "expression-data/index.html"%}
+{%from "flash_messages.html" import flash_messages%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+
+{%block title%}Select Grouping/Population{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Select grouping/population</h1>
+
+<div class="row">
+ <p>The data is organised in a hierarchical form, beginning with
+ <em>species</em> at the very top. Under <em>species</em> the data is
+ organised by <em>population</em>, sometimes referred to as <em>grouping</em>.
+ (In some really old documents/systems, you might see this referred to as
+ <em>InbredSet</em>.)</p>
+ <p>In this section, you get to define what population your data is to be
+ organised by.</p>
+</div>
+
+<div class="row">
+ <form method="POST"
+ action="{{url_for('expression-data.rqtl2.select_population',
+ species_id=species.SpeciesId)}}">
+ <legend class="heading">select grouping/population</legend>
+ {{flash_messages("error-select-population")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+
+ <div class="form-group">
+ <label for="select:inbredset" class="form-label">population</label>
+ <select id="select:inbredset"
+ name="inbredset_id"
+ required="required"
+ class="form-control">
+ <option value="">Select a grouping/population</option>
+ {%for pop in populations%}
+ <option value="{{pop.InbredSetId}}">
+ {{pop.InbredSetName}} ({{pop.FullName}})</option>
+ {%endfor%}
+ </select>
+ <span class="form-text text-muted">Select the population for your data from
+ the list below.</span>
+ </div>
+
+ <button type="submit" class="btn btn-primary" />select population</button>
+</form>
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
+
+
+{%block javascript%}
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/select-probeset-dataset.html b/uploader/templates/populations/rqtl2/select-probeset-dataset.html
new file mode 100644
index 0000000..74f8f69
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/select-probeset-dataset.html
@@ -0,0 +1,191 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Phenotype(ProbeSet) Dataset</h2>
+
+<div class="row">
+ <p>The R/qtl2 bundle you uploaded contains (a) "<strong>pheno</strong>"
+ file(s). This data needs to be organised under a dataset.</p>
+ <p>This page gives you the ability to do that.</p>
+</div>
+
+{%if datasets | length > 0%}
+<div class="row">
+ <form method="POST"
+ action="{{url_for('expression-data.rqtl2.select_probeset_dataset',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:select-probeset-dataset">
+ <legend class="heading">Select from existing ProbeSet datasets</legend>
+ {{flash_messages("error-rqtl2")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+ <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" />
+
+ <div class="form-group">
+ <label class="form-label" for="select:probe-dataset">Dataset</label>
+ <select id="select:probe-dataset"
+ name="probe-dataset-id"
+ required="required"
+ {%if datasets | length == 0%}disabled="disabled"{%endif%}
+ class="form-control"
+ aria-describedby="help-probe-dataset">
+ <option value="">Select a dataset</option>
+ {%for dataset in datasets%}
+ <option value={{dataset.Id}}>
+ {{dataset.Name}}
+ {%if dataset.FullName%}
+ -- ({{dataset.FullName}})
+ {%endif%}
+ </option>
+ {%endfor%}
+ </select>
+
+ <span id="help-probe-dataset" class="form-text text-muted">
+ Select from existing ProbeSet datasets.</span>
+ </div>
+
+ <button type="submit" class="btn btn-primary" />select dataset</button>
+</form>
+</div>
+
+<div class="row">
+ <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p>
+</div>
+{%endif%}
+
+<div class="row">
+ <p>Create an entirely new ProbeSet dataset for your data.</p>
+</div>
+
+<div class="row">
+ <form method="POST"
+ action="{{url_for('expression-data.rqtl2.create_probeset_dataset',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:create-probeset-dataset">
+ <legend class="heading">Create a new ProbeSet dataset</legend>
+ {{flash_messages("error-rqtl2-create-probeset-dataset")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+ <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" />
+
+ <div class="form-group">
+ <label class="form-label" for="select:average">averaging method</label>
+ <select id="select:average"
+ name="averageid"
+ required="required"
+ class="form-control"
+ aria-describedby="help-average">
+ <option value="">Select averaging method</option>
+ {%for avgmethod in avgmethods%}
+ <option value="{{avgmethod.Id}}">
+ {{avgmethod.Name}}
+ {%if avgmethod.Normalization%}
+ ({{avgmethod.Normalization}})
+ {%endif%}
+ </option>
+ {%endfor%}
+ </select>
+
+ <span id="help-average" class="form-text text-muted">
+ Select the averaging method used for your data.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="txt:datasetname">Name</label>
+ <input type="text" id="txt:datasetname" name="datasetname"
+ required="required"
+ maxlength="40"
+ title="Name of the dataset, e.g 'BXDMicroArray_ProbeSet_June03'. (Required)"
+ class="form-control"
+ aria-describedby="help-dataset-name" />
+
+ <span id="help-dataset-name" class="form-text text-muted">
+ Provide a name for the dataset e.g. "BXDMicroArray_ProbeSet_June03". This
+ is mandatory <strong>MUST</strong> be provided.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="txt:datasetfullname">Full Name</label>
+ <input type="text" id="txt:datasetfullname" name="datasetfullname"
+ required="required"
+ maxlength="100"
+ title="A longer name for the dataset, e.g. 'UTHSC Brain mRNA U74Av2 (Jun03) MAS5'. (Required)"
+ class="form-control"
+ aria-describedby="help-dataset-fullname" />
+
+ <span id="help-dataset-fullname" class="form-text text-muted">
+ Provide a longer, more descriptive name for the dataset e.g.
+ "UTHSC Brain mRNA U74Av2 (Jun03) MAS5". This is mandatory and
+ <strong>MUST</strong> be provided.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="txt:datasetshortname">Short Name</label>
+ <input type="text" id="txt:datasetshortname" name="datasetshortname"
+ maxlength="100"
+ title="An abbreviated name for the dataset, e.g 'Br_U_0603_M'. (Optional)"
+ class="form-control"
+ aria-describedby="help-dataset-shortname" />
+
+ <span id="help-dataset-shortname" class="form-text text-muted">
+ Provide a longer, more descriptive name for the dataset e.g. "Br_U_0603_M".
+ This is optional.
+ </span>
+ </div>
+
+ <div class="form-check">
+ <input type="checkbox" id="chk:public" name="datasetpublic"
+ checked="checked"
+ title="Whether or not the dataset is accessible by the general public."
+ class="form-check-input"
+ aria-describedby="help-public" />
+ <label class="form-check-label" for="chk:datasetpublic">Public?</label>
+
+ <span id="help-public" class="form-text text-muted">
+ Check to specify that the dataset will be publicly available. Uncheck to
+ limit access to the dataset.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="select:datasetdatascale">Data Scale</label>
+ <select id="select:datasetdatascale"
+ name="datasetdatascale"
+ required="required"
+ class="form-control"
+ aria-describedby="help-dataset-datascale">
+ <option value="log2" selected="selected">log2</option>
+ <option value="z_score">z_score</option>
+ <option value="log2_ratio">log2_ratio</option>
+ <option value="linear">linear</option>
+ <option value="linear_positive">linear_positive</option>
+ </select>
+
+ <span id="help-dataset-datascale" class="form-text text-muted">
+ Select from a list of scaling methods.
+ </span>
+ </div>
+
+ <button type="submit" class="btn btn-primary">create dataset</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/select-probeset-study-id.html b/uploader/templates/populations/rqtl2/select-probeset-study-id.html
new file mode 100644
index 0000000..e3fd9cc
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/select-probeset-study-id.html
@@ -0,0 +1,143 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages %}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Phenotype(ProbeSet) Study</h2>
+
+<div class="row">
+ <p>The R/qtl2 bundle you uploaded contains (a) "<strong>pheno</strong>"
+ file(s). This data needs to be organised under a study.</p>
+ <p>In this page, you can either select from a existing dataset:</p>
+
+ <form method="POST"
+ action="{{url_for('expression-data.rqtl2.select_probeset_study',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:select-probeset-study">
+ <legend class="heading">Select from existing ProbeSet studies</legend>
+ {{flash_messages("error-rqtl2-select-probeset-study")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+
+ <div>
+ <label for="select:probe-study" class="form-label">Study</label>
+ <select id="select:probe-study"
+ name="probe-study-id"
+ required="required"
+ aria-describedby="help-select-probeset-study"
+ {%if studies | length == 0%}disabled="disabled"{%endif%}
+ class="form-control">
+ <option value="">Select a study</option>
+ {%for study in studies%}
+ <option value={{study.Id}}>
+ {{study.Name}}
+ {%if study.FullName%}
+ -- ({{study.FullName}})
+ {%endif%}
+ </option>
+ {%endfor%}
+ </select>
+ <small id="help-select-probeset-study" class="form-text text-muted">
+ Select from existing ProbeSet studies.
+ </small>
+ </div>
+
+ <button type="submit" class="btn btn-primary">select study</button>
+ </form>
+</div>
+
+<div class="row">
+ <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p>
+</div>
+
+<div class="row">
+
+ <p>Create a new ProbeSet dataset below:</p>
+
+ <form method="POST"
+ action="{{url_for('expression-data.rqtl2.create_probeset_study',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:create-probeset-study">
+ <legend class="heading">Create new ProbeSet study</legend>
+
+ {{flash_messages("error-rqtl2-create-probeset-study")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+
+ <div>
+ <label for="select:platform" class="form-label">Platform</label>
+ <select id="select:platform"
+ name="platformid"
+ required="required"
+ aria-describedby="help-select-platform"
+ {%if platforms | length == 0%}disabled="disabled"{%endif%}
+ class="form-control">
+ <option value="">Select a platform</option>
+ {%for platform in platforms%}
+ <option value="{{platform.GeneChipId}}">
+ {{platform.GeneChipName}} ({{platform.Name}})
+ </option>
+ {%endfor%}
+ </select>
+ <small id="help-select-platform" class="form-text text-muted">
+ Select from a list of known genomics platforms.
+ </small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:studyname" class="form-label">Study name</label>
+ <input type="text" id="txt:studyname" name="studyname"
+ placeholder="Name of the study. (Required)"
+ required="required"
+ maxlength="100"
+ class="form-control" />
+ <span class="form-text text-muted" id="help-study-name">
+ Provide a name for the study.</span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:studyfullname" class="form-label">Full Study Name</label>
+ <input type="text"
+ id="txt:studyfullname"
+ name="studyfullname"
+ placeholder="Longer name of the study. (Optional)"
+ maxlength="100"
+ class="form-control" />
+ <span class="form-text text-muted" id="help-study-full-name">
+ Provide a longer, more descriptive name for the study. This is optional
+ and you can leave it blank.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:studyshortname" class="form-label">Short Study Name</label>
+ <input type="text"
+ id="txt:studyshortname"
+ name="studyshortname"
+ placeholder="Shorter name of the study. (Optional)"
+ maxlength="100"
+ class="form-control" />
+ <span class="form-text text-muted" id="help-study-short-name">
+ Provide a shorter name for the study. This is optional and you can leave
+ it blank.
+ </span>
+ </div>
+
+ <button type="submit" class="btn btn-primary">create study</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/select-tissue.html b/uploader/templates/populations/rqtl2/select-tissue.html
new file mode 100644
index 0000000..fe3080a
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/select-tissue.html
@@ -0,0 +1,115 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Tissue</h2>
+
+<div class="row">
+ <p>The data you are uploading concerns a tissue, cell, organ, or other
+ biological material used in an experiment.</p>
+ <p>Select the appropriate biological material below</p>
+</div>
+
+{%if tissues | length > 0%}
+<div class="row">
+ <form method="POST"
+ action="{{url_for('expression-data.rqtl2.select_tissue',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:select-probeset-dataset">
+ <legend class="heading">Select from existing ProbeSet datasets</legend>
+ {{flash_messages("error-select-tissue")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+
+ <div class="form-group">
+ <label class="form-label" for="select-tissue">Tissue</label>
+ <select id="select-tissue"
+ name="tissueid"
+ required="required"
+ {%if tissues | length == 0%}disabled="disabled"{%endif%}
+ class="form-control"
+ aria-describedby="help-select-tissue">
+ <option value="">Select a tissue</option>
+ {%for tissue in tissues%}
+ <option value={{tissue.Id}}>
+ {{tissue.Name}}
+ {%if tissue.Short_Name%}
+ -- ({{tissue.Short_Name}})
+ {%endif%}
+ </option>
+ {%endfor%}
+ </select>
+
+ <span id="help-select-tissue" class="form-text text-muted">
+ Select from existing biological material.</span>
+ </div>
+
+ <button type="submit" class="btn btn-primary">use selected</button>
+ </form>
+</div>
+
+<div class="row">
+ <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p>
+</div>
+{%endif%}
+
+<div class="row">
+ <p>If you cannot find the biological material in the drop-down above, add it
+ to the system below.</p>
+
+ <form method="POST"
+ action="{{url_for('expression-data.rqtl2.create_tissue',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:create-probeset-dataset">
+ <legend class="heading">Add new tissue, organ or biological material</legend>
+ {{flash_messages("error-create-tissue")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+
+ <div class="form-group">
+ <label class="form-label" for="tissue-name">name</label>
+ <input type="text"
+ id="txt-tissuename"
+ name="tissuename"
+ required="required"
+ title = "A name to identify the tissue, organ or biological material."
+ class="form-control"
+ aria-describedby="help-tissue-name" />
+
+ <span class="form-text text-muted" id="help-tissue-name">
+ A name to identify the tissue, organ or biological material.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-shortname" class="form-label">short name</label>
+ <input type="text" id="txt-tissueshortname" name="tissueshortname"
+ required="required"
+ maxlength="7"
+ title="A short name (e.g. 'Mam') for the biological material."
+ class="form-control"
+ aria-describedby="help-tissue-short-name" />
+
+ <span class="form-text text-muted" id="help-tissue-short-name">
+ Provide a short name for the tissue, organ or biological material used in
+ the experiment.
+ </span>
+ </div>
+
+ <button type="submit" class="btn btn-primary" />add new material</button>
+</form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/summary-info.html b/uploader/templates/populations/rqtl2/summary-info.html
new file mode 100644
index 0000000..0adba2e
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/summary-info.html
@@ -0,0 +1,65 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Summary</h2>
+
+<div class="row">
+ <p>This is the information you have provided to accompany the R/qtl2 bundle
+ you have uploaded. Please verify the information is correct before
+ proceeding.</p>
+</div>
+
+<div class="row">
+ <dl>
+ <dt>Species</dt>
+ <dd>{{species.SpeciesName}} ({{species.FullName}})</dd>
+
+ <dt>Population</dt>
+ <dd>{{population.InbredSetName}}</dd>
+
+ {%if geno_dataset%}
+ <dt>Genotype Dataset</dt>
+ <dd>{{geno_dataset.Name}} ({{geno_dataset.FullName}})</dd>
+ {%endif%}
+
+ {%if tissue%}
+ <dt>Tissue</dt>
+ <dd>{{tissue.TissueName}} ({{tissue.Name}}, {{tissue.Short_Name}})</dd>
+ {%endif%}
+
+ {%if probe_study%}
+ <dt>ProbeSet Study</dt>
+ <dd>{{probe_study.Name}} ({{probe_study.FullName}})</dd>
+ {%endif%}
+
+ {%if probe_dataset%}
+ <dt>ProbeSet Dataset</dt>
+ <dd>{{probe_dataset.Name2}} ({{probe_dataset.FullName}})</dd>
+ {%endif%}
+ </dl>
+</div>
+
+<div class="row">
+ <form id="frm:confirm-rqtl2bundle-details"
+ action="{{url_for('expression-data.rqtl2.confirm_bundle_details',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ <legend class="heading">Create ProbeSet dataset</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" />
+ <input type="hidden" name="probe-dataset-id" value="{{probe_dataset.Id}}" />
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-01.html b/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-01.html
new file mode 100644
index 0000000..9d45c5f
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-01.html
@@ -0,0 +1,276 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "upload_progress_indicator.html" import upload_progress_indicator%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+{%macro rqtl2_file_help()%}
+<span class="form-text text-muted">
+ <p>
+ Provide a valid R/qtl2 zip file here. In particular, ensure your zip bundle
+ contains exactly one control file and the corresponding files mentioned in
+ the control file.
+ </p>
+ <p>
+ The control file can be either a YAML or JSON file. <em>ALL</em> other data
+ files in the zip bundle should be CSV files.
+ </p>
+ <p>See the
+ <a href="https://kbroman.org/qtl2/assets/vignettes/input_files.html"
+ target="_blank">
+ R/qtl2 file format specifications
+ </a>
+ for more details.
+ </p>
+</span>
+{%endmacro%}
+{{upload_progress_indicator()}}
+
+<div id="resumable-file-display-template"
+ class="panel panel-info"
+ style="display: none">
+ <div class="panel-heading"></div>
+ <div class="panel-body"></div>
+</div>
+
+
+<h2 class="heading">Upload R/qtl2 Bundle</h2>
+
+<div id="resumable-drop-area"
+ style="display:none;background:#eeeeee;min-height:12em;border-radius:0.5em;padding:1em;">
+ <p>
+ <a id="resumable-browse-button" href="#"
+ class="btn btn-info">Browse</a>
+ </p>
+ <p class="form-text text-muted">
+ You can drag and drop your file here, or click the browse button.
+ Click on the file to remove it.
+ </p>
+ {{rqtl2_file_help()}}
+ <div id="resumable-selected-files"
+ style="display:flex;flex-direction:row;flex-wrap: wrap;justify-content:space-around;gap:10px 20px;"></div>
+ <div id="resumable-class-buttons" style="text-align: right;">
+ <button id="resumable-upload-button"
+ class="btn btn-primary"
+ style="display: none">start upload</button>
+ <button id="resumable-cancel-upload-button"
+ class="btn btn-danger"
+ style="display: none">cancel upload</button>
+ </div>
+ <div id="resumable-progress-bar" class="progress" style="display: none">
+ <div class="progress-bar"
+ role="progress-bar"
+ aria-valuenow="60"
+ aria-valuemin="0"
+ aria-valuemax="100"
+ style="width: 0%;">
+ Uploading: 60%
+ </div>
+ </div>
+</div>
+
+<form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('expression-data.rqtl2.upload_rqtl2_bundle',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data"
+ data-resumable-target="{{url_for(
+ 'expression-data.rqtl2.upload_rqtl2_bundle_chunked_post',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}">
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+
+ {{flash_all_messages()}}
+
+ <div class="form-group">
+ <legend class="heading">file upload</legend>
+ <label for="file-rqtl2-bundle" class="form-label">R/qtl2 bundle</label>
+ <input type="file" id="file-rqtl2-bundle" name="rqtl2_bundle_file"
+ accept="application/zip, .zip"
+ required="required"
+ class="form-control" />
+ {{rqtl2_file_help()}}
+ </div>
+
+ <button type="submit"
+ class="btn btn-primary"
+ data-toggle="modal"
+ data-target="#upload-progress-indicator">upload R/qtl2 bundle</button>
+</form>
+
+{%endblock%}
+
+{%block javascript%}
+<script src="{{url_for('base.node_modules',
+ filename='resumablejs/resumable.js')}}"></script>
+<script type="text/javascript" src="/static/js/upload_progress.js"></script>
+<script type="text/javascript">
+ function readBinaryFile(file) {
+ return new Promise((resolve, reject) => {
+ var _reader = new FileReader();
+ _reader.onload = (event) => {resolve(_reader.result);};
+ _reader.readAsArrayBuffer(file);
+ });
+ }
+
+ function computeFileChecksum(file) {
+ return readBinaryFile(file)
+ .then((content) => {
+ return window.crypto.subtle.digest(
+ "SHA-256", new Uint8Array(content));
+ }).then((digest) => {
+ return Uint8ArrayToHex(new Uint8Array(digest))
+ });
+ }
+
+ function Uint8ArrayToHex(arr) {
+ var toHex = (val) => {
+ _hex = val.toString(16);
+ if(_hex.length < 2) {
+ return "0" + val;
+ }
+ return _hex;
+ };
+ _hexstr = ""
+ arr.forEach((val) => {_hexstr += toHex(val)});
+ return _hexstr
+ }
+
+ var r = Resumable({
+ target: $("#frm-upload-rqtl2-bundle").attr("data-resumable-target"),
+ fileType: ["zip"],
+ maxFiles: 1,
+ forceChunkSize: true,
+ generateUniqueIdentifier: (file, event) => {
+ return computeFileChecksum(file).then((checksum) => {
+ var _relativePath = (file.webkitRelativePath
+ || file.relativePath
+ || file.fileName
+ || file.name);
+ return checksum + "-" + _relativePath.replace(
+ /[^a-zA-Z0-9_-]/img, "");
+ });
+ }
+ });
+
+ if(r.support) {
+ //Hide form and display drag&drop UI
+ $("#frm-upload-rqtl2-bundle").css("display", "none");
+ $("#resumable-drop-area").css("display", "block");
+
+ // Define UI elements for browse and drag&drop
+ r.assignDrop(document.getElementById("resumable-drop-area"));
+ r.assignBrowse(document.getElementById("resumable-browse-button"));
+
+ // Event handlers
+
+ function display_files(files) {
+ displayArea = $("#resumable-selected-files")
+ displayArea.empty();
+ files.forEach((file) => {
+ var displayElement = $(
+ "#resumable-file-display-template").clone();
+ displayElement.removeAttr("id");
+ displayElement.css("display", "");
+ displayElement.find(".panel-heading").text(file.fileName);
+ list = $("<ul></ul>");
+ list.append($("<li><strong>Name</strong>: "
+ + (file.name
+ || file.fileName
+ || file.relativePath
+ || file.webkitRelativePath)
+ + "</li>"));
+ list.append($("<li><strong>Size</strong>: "
+ + (file.size / (1024*1024)).toFixed(2)
+ + " MB</li>"));
+ list.append($("<li><strong>Unique Identifier</strong>: "
+ + file.uniqueIdentifier + "</li>"));
+ list.append($("<li><strong>Mime</strong>: "
+ + file.file.type
+ + "</li>"));
+ displayElement.find(".panel-body").append(list);
+ displayElement.appendTo("#resumable-selected-files");
+ });
+ }
+
+ r.on("filesAdded", function(files) {
+ display_files(files);
+ $("#resumable-upload-button").css("display", "");
+ $("#resumable-upload-button").on("click", (event) => {
+ r.upload();
+ });
+ });
+
+ r.on("uploadStart", (event) => {
+ $("#resumable-upload-button").css("display", "none");
+ $("#resumable-cancel-upload-button").css("display", "");
+ $("#resumable-cancel-upload-button").on("click", (event) => {
+ r.files.forEach((file) => {
+ if(file.isUploading()) {
+ file.abort();
+ }
+ });
+ $("#resumable-cancel-upload-button").css("display", "none");
+ $("#resumable-upload-button").on("click", (event) => {
+ r.files.forEach((file) => {file.retry();});
+ });
+ $("#resumable-upload-button").css("display", "");
+ });
+ });
+
+ r.on("progress", () => {
+ var progress = (r.progress() * 100).toFixed(2);
+ var pbar = $("#resumable-progress-bar > .progress-bar");
+ $("#resumable-progress-bar").css("display", "");
+ pbar.css("width", progress+"%");
+ pbar.attr("aria-valuenow", progress);
+ pbar.text("Uploading: " + progress + "%");
+ })
+
+ r.on("fileSuccess", (file, message) => {
+ if(message != "OK") {
+ var uri = (window.location.protocol
+ + "//"
+ + window.location.host
+ + message);
+ window.location.replace(uri);
+ }
+ });
+
+ r.on("error", (message, file) => {
+ filename = (file.webkitRelativePath
+ || file.relativePath
+ || file.fileName
+ || file.name);
+ jsonmsg = JSON.parse(message);
+ alert("There was an error while uploading your file '"
+ + filename
+ + "'. The error message was:\n\n\t"
+ + jsonmsg.error
+ + " ("
+ + jsonmsg.statuscode
+ + "): " + jsonmsg.message);
+ })
+ } else {
+ setup_upload_handlers(
+ "frm-upload-rqtl2-bundle", make_data_uploader(
+ function (form) {
+ var formdata = new FormData();
+ formdata.append(
+ "species_id",
+ form.querySelector('input[name="species_id"]').value);
+ formdata.append(
+ "population_id",
+ form.querySelector('input[name="population_id"]').value);
+ formdata.append(
+ "rqtl2_bundle_file",
+ form.querySelector("#file-rqtl2-bundle").files[0]);
+ return formdata;
+ }));
+ }
+</script>
+{%endblock%}
diff --git a/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-02.html b/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-02.html
new file mode 100644
index 0000000..8210ed0
--- /dev/null
+++ b/uploader/templates/populations/rqtl2/upload-rqtl2-bundle-step-02.html
@@ -0,0 +1,33 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Upload R/qtl2 Bundle</h2>
+
+<div class="row">
+ <p>You have successfully uploaded the zipped bundle of R/qtl2 files.</p>
+ <p>The next step is to select the various extra information we need to figure
+ out what to do with the data. You will select/create the relevant studies
+ and/or datasets to organise the data in the steps that follow.</p>
+ <p>Click "Continue" below to proceed.</p>
+
+ <form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('expression-data.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ {{flash_all_messages()}}
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/populations/view-population.html b/uploader/templates/populations/view-population.html
new file mode 100644
index 0000000..b23caeb
--- /dev/null
+++ b/uploader/templates/populations/view-population.html
@@ -0,0 +1,102 @@
+{%extends "populations/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-select-species.html" import select_species_form%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+
+{%block title%}Populations{%endblock%}
+
+{%block pagetitle%}Populations{%endblock%}
+
+{%block lvl3_breadcrumbs%}
+<li {%if activelink=="view-population"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.view_population',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}">view</a>
+</li>
+{%endblock%}
+
+
+{%block contents%}
+<div class="row">
+ <h2>Population Details</h2>
+
+ {{flash_all_messages()}}
+
+ <dl>
+ <dt>Name</dt>
+ <dd>{{population.Name}}</dd>
+
+ <dt>FullName</dt>
+ <dd>{{population.FullName}}</dd>
+
+ <dt>Code</dt>
+ <dd>{{population.InbredSetCode}}</dd>
+
+ <dt>Genetic Type</dt>
+ <dd>{{population.GeneticType}}</dd>
+
+ <dt>Family</dt>
+ <dd>{{population.Family}}</dd>
+
+ <dt>Description</dt>
+ <dd><pre>{{population.Description or "-"}}</pre></dd>
+ </dl>
+</div>
+
+<div class="row">
+ … maybe provide a way to organise populations in the same family here …
+</div>
+
+<div class="row">
+ <h3>Actions</h3>
+
+ <p>
+ Click any of the following links to use this population in performing the
+ subsequent operations.
+ </p>
+
+ <nav class="nav">
+ <ul>
+ <li>
+ <a href="{{url_for('species.populations.samples.list_samples',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ title="Manage samples: Add new or delete existing.">
+ manage samples</a>
+ </li>
+ <li>
+ <a href="{{url_for('species.populations.genotypes.list_genotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ title="Manage genotypes for {{species.FullName}}">Manage Genotypes</a>
+ </li>
+ <li>
+ <a href="{{url_for('species.populations.phenotypes.list_datasets',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ title="Manage phenotype data.">manage phenotype data</a>
+ </li>
+ <li>
+ <a href="#" title="Manage expression data"
+ class="not-implemented">manage expression data</a>
+ </li>
+ <li>
+ <a href="#" title="Manage individual data"
+ class="not-implemented">manage individual data</a>
+ </li>
+ <li>
+ <a href="#" title="Manage RNA-Seq data"
+ class="not-implemented">manage RNA-Seq data</a>
+ </li>
+ </ul>
+ </nav>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
diff --git a/uploader/templates/samples/base.html b/uploader/templates/samples/base.html
new file mode 100644
index 0000000..291782b
--- /dev/null
+++ b/uploader/templates/samples/base.html
@@ -0,0 +1,12 @@
+{%extends "populations/base.html"%}
+
+{%block lvl3_breadcrumbs%}
+<li {%if activelink=="samples"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.samples.index')}}">Samples</a>
+</li>
+{%block lvl4_breadcrumbs%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/samples/index.html b/uploader/templates/samples/index.html
new file mode 100644
index 0000000..ee98734
--- /dev/null
+++ b/uploader/templates/samples/index.html
@@ -0,0 +1,23 @@
+{%extends "samples/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-select-species.html" import select_species_form%}
+
+{%block title%}Populations{%endblock%}
+
+{%block pagetitle%}Populations{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <p>GeneNetwork has a selection of different species of organisms to choose from. Within those species, there are the populations of interest for a variety of experiments, from which you, the researcher, picked your samples (or individuals or cases) from. Here you can provide some basic details about your samples.</p>
+ <p>To start off, we will need to know what species and population your samples belong to. Please provide that information in the next sections.</p>
+
+ {{select_species_form(url_for("species.populations.samples.index"), species)}}
+</div>
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/species.js"></script>
+{%endblock%}
diff --git a/uploader/templates/samples/list-samples.html b/uploader/templates/samples/list-samples.html
new file mode 100644
index 0000000..185e784
--- /dev/null
+++ b/uploader/templates/samples/list-samples.html
@@ -0,0 +1,132 @@
+{%extends "samples/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "populations/macro-select-population.html" import select_population_form%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Samples &mdash; List Samples{%endblock%}
+
+{%block pagetitle%}Samples &mdash; List Samples{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="list-samples"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.samples.list_samples',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">List</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <p>
+ You selected the population "{{population.FullName}}" from the
+ "{{species.FullName}}" species.
+ </p>
+</div>
+
+{%if samples | length > 0%}
+<div class="row">
+ <p>
+ This population already has <strong>{{total_samples}}</strong>
+ samples/individuals entered. You can explore the list of samples in this
+ population in the table below.
+ </p>
+</div>
+
+<div class="row">
+ <div class="col-md-2">
+ {%if offset > 0:%}
+ <a href="{{url_for('species.populations.samples.list_samples',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ from=offset-count,
+ count=count)}}">
+ <span class="glyphicon glyphicon-backward"></span>
+ Previous
+ </a>
+ {%endif%}
+ </div>
+
+ <div class="col-md-8" style="text-align: center;">
+ Samples {{offset}} &mdash; {{offset+(count if offset + count < total_samples else total_samples - offset)}} / {{total_samples}}
+ </div>
+
+ <div class="col-md-2">
+ {%if offset + count < total_samples:%}
+ <a href="{{url_for('species.populations.samples.list_samples',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ from=offset+count,
+ count=count)}}">
+ Next
+ <span class="glyphicon glyphicon-forward"></span>
+ </a>
+ {%endif%}
+ </div>
+</div>
+<div class="row">
+ <table class="table">
+ <thead>
+ <tr>
+ <th></th>
+ <th>Name</th>
+ <th>Auxilliary Name</th>
+ <th>Symbol</th>
+ <th>Alias</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for sample in samples%}
+ <tr>
+ <td>{{sample.sequence_number}}</td>
+ <td>{{sample.Name}}</td>
+ <td>{{sample.Name2}}</td>
+ <td>{{sample.Symbol or "-"}}</td>
+ <td>{{sample.Alias or "-"}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+
+ <p>
+ <a href="#"
+ title="Add samples for population '{{population.FullName}}' from species
+ '{{species.FullName}}'."
+ class="btn btn-danger">
+ delete all samples
+ </a>
+ </p>
+</div>
+
+{%else%}
+
+<div class="row">
+ <p>
+ There are no samples entered for this population. Do please go ahead and add
+ the samples for this population by clicking on the button below.
+ </p>
+
+ <p>
+ <a href="{{url_for('species.populations.samples.upload_samples',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ title="Add samples for population '{{population.FullName}}' from species
+ '{{species.FullName}}'."
+ class="btn btn-primary">
+ add samples
+ </a>
+ </p>
+</div>
+{%endif%}
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/samples/select-population.html b/uploader/templates/samples/select-population.html
new file mode 100644
index 0000000..1cc7573
--- /dev/null
+++ b/uploader/templates/samples/select-population.html
@@ -0,0 +1,26 @@
+{%extends "samples/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "populations/macro-select-population.html" import select_population_form%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+
+{%block title%}Samples &mdash; Select Population{%endblock%}
+
+{%block pagetitle%}Samples &mdash; Select Population{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ {{select_population_form(
+ url_for("species.populations.samples.select_population", species_id=species.SpeciesId), species, populations)}}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/populations.js"></script>
+{%endblock%}
diff --git a/uploader/templates/samples/upload-failure.html b/uploader/templates/samples/upload-failure.html
new file mode 100644
index 0000000..458ab55
--- /dev/null
+++ b/uploader/templates/samples/upload-failure.html
@@ -0,0 +1,37 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Samples Upload Failure{%endblock%}
+
+{%block contents%}
+<div class="row">
+<h2 class="heading">{{job.job_name[0:50]}}&hellip;</h2>
+
+<p>There was a failure attempting to upload the samples.</p>
+
+<p>Here is some information to help with debugging the issue. Provide this
+ information to the developer/maintainer.</p>
+
+<h3>Debugging Information</h3>
+<ul>
+ <li><strong>job id</strong>: {{job.job_id}}</li>
+ <li><strong>status</strong>: {{job.status}}</li>
+ <li><strong>job type</strong>: {{job["job-type"]}}</li>
+</ul>
+</div>
+
+<div class="row">
+<h4>stdout</h4>
+{{cli_output(job, "stdout")}}
+</div>
+
+<div class="row">
+<h4>stderr</h4>
+{{cli_output(job, "stderr")}}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/samples/upload-progress.html b/uploader/templates/samples/upload-progress.html
new file mode 100644
index 0000000..677d457
--- /dev/null
+++ b/uploader/templates/samples/upload-progress.html
@@ -0,0 +1,31 @@
+{%extends "samples/base.html"%}
+{%from "cli-output.html" import cli_output%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="5">
+{%endblock%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<div class="row" style="overflow-x: clip;">
+<h2 class="heading">{{job.job_name[0:50]}}&hellip;</h2>
+
+<p>
+<strong>status</strong>:
+<span>{{job["status"]}} ({{job.get("message", "-")}})</span><br />
+</p>
+
+<p>saving to database...</p>
+</div>
+
+<div class="row">
+ {{cli_output(job, "stdout")}}
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/samples/upload-samples.html b/uploader/templates/samples/upload-samples.html
new file mode 100644
index 0000000..25d3290
--- /dev/null
+++ b/uploader/templates/samples/upload-samples.html
@@ -0,0 +1,160 @@
+{%extends "samples/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "populations/macro-select-population.html" import select_population_form%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Samples &mdash; Upload Samples{%endblock%}
+
+{%block pagetitle%}Samples &mdash; Upload Samples{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="uploade-samples"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.samples.upload_samples',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">List</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <p>
+ You can now upload the samples for the "{{population.FullName}}" population
+ from the "{{species.FullName}}" species here.
+ </p>
+ <p>
+ Upload a <strong>character-separated value (CSV)</strong> file that contains
+ details about your samples. The CSV file should have the following fields:
+ <dl>
+ <dt>Name</dt>
+ <dd>The primary name/identifier for the sample/individual.</dd>
+
+ <dt>Name2</dt>
+ <dd>A secondary name for the sample. This can simply be the same as
+ <strong>Name</strong> above. This field <strong>MUST</strong> contain a
+ value.</dd>
+
+ <dt>Symbol</dt>
+ <dd>A symbol for the sample. This can be a strain name, e.g. 'BXD60' for
+ species that have strains. This field can be left empty for species like
+ Humans that do not have strains..</dd>
+
+ <dt>Alias</dt>
+ <dd>An alias for the sample. Can be an empty field, or take on the same
+ value as that of the Symbol.</dd>
+ </dl>
+ </p>
+</div>
+
+<div class="row">
+ <form id="form-samples"
+ method="POST"
+ action="{{url_for('species.populations.samples.upload_samples',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ enctype="multipart/form-data">
+ <legend class="heading">upload samples</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id" value="{{population.Id}}" />
+
+ <div class="form-group">
+ <label for="file-samples" class="form-label">select file</label>
+ <input type="file" name="samples_file" id="file:samples"
+ accept="text/csv, text/tab-separated-values"
+ class="form-control" />
+ </div>
+
+ <div class="form-group">
+ <label for="select:separator" class="form-label">field separator</label>
+ <select id="select:separator"
+ name="separator"
+ required="required"
+ class="form-control">
+ <option value="">Select separator for your file: (default is comma)</option>
+ <option value="&#x0009;">TAB</option>
+ <option value="&#x0020;">Space</option>
+ <option value=",">Comma</option>
+ <option value=";">Semicolon</option>
+ <option value="other">Other</option>
+ </select>
+ <input id="txt:separator"
+ type="text"
+ name="other_separator"
+ class="form-control" />
+ <small class="form-text text-muted">
+ If you select '<strong>Other</strong>' for the field separator value,
+ enter the character that separates the fields in your CSV file in the form
+ field below.
+ </small>
+ </div>
+
+ <div class="form-group form-check">
+ <input id="chk:heading"
+ type="checkbox"
+ name="first_line_heading"
+ class="form-check-input" />
+ <label for="chk:heading" class="form-check-label">
+ first line is a heading?</label>
+ <small class="form-text text-muted">
+ Select this if the first line in your file contains headings for the
+ columns.
+ </small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:delimiter" class="form-label">field delimiter</label>
+ <input id="txt:delimiter"
+ type="text"
+ name="field_delimiter"
+ maxlength="1"
+ class="form-control" />
+ <small class="form-text text-muted">
+ If there is a character delimiting the string texts within particular
+ fields in your CSV, provide the character here. This can be left blank if
+ no such delimiters exist in your file.
+ </small>
+ </div>
+
+ <button type="submit"
+ class="btn btn-primary">upload samples file</button>
+ </form>
+</div>
+
+<div class="row">
+ <h3>Preview File Content</h3>
+
+ <table id="tbl:samples-preview" class="table">
+ <caption class="heading">preview content</caption>
+
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Name2</th>
+ <th>Symbol</th>
+ <th>Alias</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr id="default-row">
+ <td colspan="4">
+ Please make some selections in the form above to preview the data.</td>
+ </tr>
+ </tbody>
+ </table>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
+
+{%block javascript%}
+<script src="/static/js/upload_samples.js" type="text/javascript"></script>
+{%endblock%}
diff --git a/uploader/templates/samples/upload-success.html b/uploader/templates/samples/upload-success.html
new file mode 100644
index 0000000..881d466
--- /dev/null
+++ b/uploader/templates/samples/upload-success.html
@@ -0,0 +1,36 @@
+{%extends "samples/base.html"%}
+{%from "cli-output.html" import cli_output%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+
+<div class="row" style="overflow-x: clip;">
+ <h2 class="heading">{{job.job_name[0:50]}}&hellip;</h2>
+
+ <p>
+ <strong>status</strong>:
+ <span>{{job["status"]}} ({{job.get("message", "-")}})</span><br />
+ </p>
+
+ <p>Successfully uploaded the samples.</p>
+ <p>
+ <a href="{{url_for('species.populations.samples.list_samples',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ title="View population samples">
+ View samples
+ </a>
+ </p>
+</div>
+
+<div class="row">
+ {{cli_output(job, "stdout")}}
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/select_dataset.html b/uploader/templates/select_dataset.html
new file mode 100644
index 0000000..2f07de8
--- /dev/null
+++ b/uploader/templates/select_dataset.html
@@ -0,0 +1,161 @@
+{%extends "base.html"%}
+{%from "dbupdate_hidden_fields.html" import hidden_fields%}
+
+{%block title%}Select Dataset{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
+{%endblock%}
+
+{%block contents%}
+<h2 class="heading">{{filename}}: select dataset</h2>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.final_confirmation')}}"
+ id="select-dataset-form" class="two-col-sep-col1">
+ <legend class="heading">choose existing dataset</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, totallines=totallines)}}
+
+ <div class="form-group">
+ <label for="datasetid" class="form-label">dataset:</label>
+ <select id="datasetid" name="datasetid" class="form-control"
+ {%if datasets | length == 0:%}
+ disabled="disabled"
+ {%endif%}>
+ {%for dataset in datasets:%}
+ <option value="{{dataset['Id']}}">
+ [{{dataset["Name"]}}] - {{dataset["FullName"]}}
+ </option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <button type="submit" class="btn btn-primary"
+ {%if datasets | length == 0:%}
+ disabled="disabled"
+ {%endif%} />update database</button>
+</form>
+</div>
+
+<div class="row">
+ <p class="two-col-sep-separator">OR</p>
+</div>
+
+<div class="row">
+ <form method="POST" id="create-dataset-form"
+ action="{{url_for('dbinsert.create_dataset')}}"
+ class="two-col-sep-col2">
+ <legend class="heading">create new dataset</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, totallines=totallines)}}
+
+ {%with messages = get_flashed_messages(with_categories=true)%}
+ {%if messages:%}
+ <ul>
+ {%for category, message in messages:%}
+ <li class="{{category}}">{{message}}</li>
+ {%endfor%}
+ </ul>
+ {%endif%}
+ {%endwith%}
+
+ <div class="form-group">
+ <label for="avgid" class="form-label">average:</label>
+ <select id="avgid" name="avgid" required="required" class="form-control">
+ <option value="">Select averaging method</option>
+ {%for method in avgmethods:%}
+ <option value="{{method['AvgMethodId']}}"
+ {%if avgid is defined and method['AvgMethodId'] | int == avgid | int%}
+ selected="selected"
+ {%endif%}>
+ {{method["Name"]}}
+ </option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <div class="form-group">
+ <label for="datasetname" class="form-label">name:</label>
+ <input id="datasetname" name="datasetname" type="text"
+ class="form-control"
+ {%if datasetname is defined %}
+ value="{{datasetname}}"
+ {%endif%} />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetname2" class="form-label">name 2:</label>
+ <input id="datasetname2" name="datasetname2" type="text"
+ required="required" class="form-control"
+ {%if datasetname2 is defined %}
+ value="{{datasetname2}}"
+ {%endif%} />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetfullname" class="form-label">full name:</label>
+ <input id="datasetfullname" name="datasetfullname" type="text"
+ required="required" class="form-control"
+ {%if datasetfullname is defined %}
+ value="{{datasetfullname}}"
+ {%endif%} />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetshortname" class="form-label">short name:</label>
+ <input id="datasetshortname" name="datasetshortname" type="text"
+ required="required" class="form-control"
+ {%if datasetshortname is defined %}
+ value="{{datasetshortname}}"
+ {%endif%} />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetpublic" class="form-label">public:</label>
+ <input id="datasetpublic" name="datasetpublic" type="number"
+ required="required" min="0" max="2"
+ {%if datasetpublic is defined %}
+ value="{{datasetpublic | int}}"
+ {%else%}
+ value="0"
+ {%endif%}
+ class="form-control" />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetconfidentiality">confidentiality:</label>
+ <input id="datasetconfidentiality" name="datasetconfidentiality"
+ type="number" required="required" min="0" max="2"
+ {%if datasetconfidentiality is defined %}
+ value="{{datasetconfidentiality | int}}"
+ {%else%}
+ value="0"
+ {%endif%}
+ class="form-control" />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetdatascale" class="form-label">data scale:</label>
+ <select id="datasetdatascale" name="datasetdatascale" class="form-control">
+ <option value="">None</option>
+ {%for dscale in datascales:%}
+ <option value="{{dscale}}"
+ {%if datasetdatascale is defined and dscale == datasetdatascale%}
+ selected="selected"
+ {%elif dscale == "log2":%}
+ selected="selected"
+ {%endif%}>
+ {{dscale}}
+ </option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <button type="submit" class="btn btn-primary">create new dataset</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/select_platform.html b/uploader/templates/select_platform.html
new file mode 100644
index 0000000..d9bc68f
--- /dev/null
+++ b/uploader/templates/select_platform.html
@@ -0,0 +1,82 @@
+{%extends "base.html"%}
+
+{%block title%}Select Dataset{%endblock%}
+
+{%block contents%}
+<h2 class="heading">{{filename}}: select platform</h2>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.select_study')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}">
+ <input type="hidden" name="filename" value="{{filename}}" />
+ <input type="hidden" name="filetype" value="{{filetype}}" />
+ <input type="hidden" name="totallines" value="{{totallines}}" />
+
+ <div class="form-group">
+ <label for="species" class="form-label">species</label>
+ <select id="species" name="species" class="form-control">
+ {%for row in species:%}
+ <option value="{{row['SpeciesId']}}"
+ {%if row["Name"] == default_species:%}
+ selected="selected"
+ {%endif%}>
+ {{row["MenuName"]}}
+ </option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <table id="genechips-table" class="table">
+ <caption>select platform</caption>
+ <thead>
+ <tr>
+ <th>Select</th>
+ <th>GeneChip ID</th>
+ <th>GeneChip Name</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for chip in genechips:%}
+ <tr>
+ <td>
+ <input type="radio" name="genechipid" value="{{chip['GeneChipId']}}"
+ required="required" />
+ </td>
+ <td>{{chip["GeneChipId"]}}</td>
+ <td>{{chip["GeneChipName"]}}</td>
+ </tr>
+ {%else%}
+ <tr>
+ <td colspan="5">No chips found for selected species</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+
+ <button type="submit" class="btn btn-primary">submit platform</button>
+ </form>
+</div>
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/utils.js"></script>
+<script type="text/javascript" src="/static/js/select_platform.js"></script>
+<script type="text/javascript">
+ document.getElementById(
+ "species").addEventListener("change", update_genechips);
+ document.getElementById(
+ "genechips-table").getElementsByTagName(
+ "tbody")[0].addEventListener(
+ "click",
+ function(event) {
+ if(event.target.tagName.toLowerCase() == "td") {
+ return select_row_radio(event.target.parentElement);
+ }
+ if(event.target.tagName.toLowerCase() == "td") {
+ return select_row_radio(event.target);
+ }
+ return false;
+ });
+</script>
+{%endblock%}
diff --git a/uploader/templates/select_study.html b/uploader/templates/select_study.html
new file mode 100644
index 0000000..648ad4c
--- /dev/null
+++ b/uploader/templates/select_study.html
@@ -0,0 +1,108 @@
+{%extends "base.html"%}
+{%from "dbupdate_hidden_fields.html" import hidden_fields%}
+
+{%block title%}Select Dataset{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
+{%endblock%}
+
+{%block contents%}
+<h2 class="heading">{{filename}}: select study</h2>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.select_dataset')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col1">
+ <legend class="heading">Select from existing study</legend>
+ {{hidden_fields(filename, filetype, species=species, genechipid=genechipid,
+ totallines=totallines)}}
+
+ <div class="form-group">
+ <label class="form-label" for="study">study:</label>
+ <select id="study" name="studyid" class="form-control">
+ {%for study in studies:%}
+ <option value="{{study['Id']}}">{{study["Name"]}}</option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <button type="submit"
+ class="btn btn-primary"
+ {%if studies | length == 0:%}
+ disabled="disabled"
+ {%endif%} />submit selected study</button>
+</form>
+</div>
+
+<div class="row">
+ <p class="two-col-sep-separator">OR</p>
+</div>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.create_study')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col2">
+ {%with messages = get_flashed_messages(with_categories=true)%}
+ {%if messages:%}
+ <ul>
+ {%for category, message in messages:%}
+ <li class="{{category}}">{{message}}</li>
+ {%endfor%}
+ </ul>
+ {%endif%}
+ {%endwith%}
+ <legend class="heading">Create new study</legend>
+ {{hidden_fields(filename, filetype, species=species, genechipid=genechipid,
+ totallines=totallines)}}
+
+ <div class="form-group">
+ <label class="form-label" for="studyname">name:</label>
+ <input type="text" id="studyname" name="studyname" class="form-control"
+ required="required"
+ {%if studyname:%}
+ value="{{studyname}}"
+ {%endif%} />
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="group">group:</label>
+ <select id="group" name="inbredsetid" class="form-control"
+ required="required">
+ <option value="">Select group</option>
+ {%for family in groups:%}
+ <optgroup label="{{family}}">
+ {%for group in groups[family]:%}
+ <option value="{{group['InbredSetId']}}"
+ {%if group["InbredSetId"] == selected_group:%}
+ selected="selected"
+ {%endif%}>
+ {{group["FullName"]}}
+ </option>
+ {%endfor%}
+ </optgroup>
+ {%endfor%}
+ </select>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="tissue">tissue:</label>
+ <select id="tissue" name="tissueid" class="form-control"
+ required="required">
+ <option value="">Select type</option>
+ {%for tissue in tissues:%}
+ <option value="{{tissue['TissueId']}}"
+ {%if tissue["TissueId"] == selected_tissue:%}
+ selected="selected"
+ {%endif%}>
+ {{tissue["Name"]}}
+ </option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <button type="submit" class="btn btn-primary">create study</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/species/base.html b/uploader/templates/species/base.html
new file mode 100644
index 0000000..f64f72b
--- /dev/null
+++ b/uploader/templates/species/base.html
@@ -0,0 +1,17 @@
+{%extends "base.html"%}
+
+{%block lvl1_breadcrumbs%}
+<li {%if activelink=="species"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ {%if species is mapping%}
+ <a href="{{url_for('species.view_species', species_id=species.SpeciesId)}}">
+ {{species.Name}}</a>
+ {%else%}
+ <a href="{{url_for('species.list_species')}}">Species</a>
+ {%endif%}
+</li>
+{%block lvl2_breadcrumbs%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/species/create-species.html b/uploader/templates/species/create-species.html
new file mode 100644
index 0000000..138dbaa
--- /dev/null
+++ b/uploader/templates/species/create-species.html
@@ -0,0 +1,148 @@
+{%extends "species/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Create Species{%endblock%}
+
+{%block pagetitle%}Create Species{%endblock%}
+
+{%block lvl2_breadcrumbs%}
+<li {%if activelink=="create-species"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.create_species')}}">Create</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+<div class="row">
+ <form id="frm-create-species"
+ method="POST"
+ action="{{url_for('species.create_species', return_to=return_to)}}"
+ class="form-horizontal">
+ <legend>Create Species</legend>
+
+ {{flash_all_messages()}}
+
+ <input type="hidden" name="return_to" value="{{return_to}}">
+
+ <div class="form-group">
+ <label for="txt-taxonomy-id" class="control-label col-sm-2">
+ Taxonomy ID</label>
+ <div class="col-sm-10">
+ <div class="input-group">
+ <input id="txt-taxonomy-id"
+ name="species_taxonomy_id"
+ type="text"
+ class="form-control" />
+ <span class="input-group-btn">
+ <button id="btn-search-taxonid" class="btn btn-info">Search</button>
+ </span>
+ </div>
+ <small class="form-text text-small text-muted">
+ Use
+ <a href="https://www.ncbi.nlm.nih.gov/Taxonomy/taxonomyhome.html/"
+ title="NCBI's Taxonomy Browser homepage"
+ target="_blank">
+ NCBI's Taxonomy Browser homepage</a> to search for the species you
+ want. If the species exists on NCBI, they will have a Taxonomy ID. Copy
+ that Taxonomy ID to this field, and click "Search" to auto-fill the
+ details.<br />
+ This field is optional.</small>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-species-name" class="control-label col-sm-2">Common Name</label>
+ <div class="col-sm-10">
+ <input id="txt-species-name"
+ name="common_name"
+ type="text"
+ class="form-control"
+ required="required" />
+ <small class="form-text text-muted">This is the day-to-day term used by
+ laymen, e.g. Mouse (instead of Mus musculus), round worm (instead of
+ Ascaris lumbricoides), etc.<br />
+ For species without this, just enter the scientific name.
+ </small>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-species-scientific" class="control-label col-sm-2">
+ Scientific Name</label>
+ <div class="col-sm-10">
+ <input id="txt-species-scientific"
+ name="scientific_name"
+ type="text"
+ class="form-control"
+ required="required" />
+ <small class="form-text text-muted">This is the scientific name for the
+ species e.g. Homo sapiens, Mus musculus, etc.</small>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="select-species-family" class="control-label col-sm-2">Family</label>
+ <div class="col-sm-10">
+ <select id="select-species-family"
+ name="species_family"
+ required="required"
+ class="form-control">
+ <option value="ungrouped">I do not know what to pick</option>
+ {%for family in families%}
+ <option value="{{family}}">{{family}}</option>
+ {%endfor%}
+ </select>
+ <small class="form-text text-muted">
+ This is a rough grouping of the species.</small>
+ </div>
+ </div>
+
+ <div class="col-sm-offset-2 col-sm-10">
+ <input type="submit"
+ value="create new species"
+ class="btn btn-primary" />
+ </div>
+
+ </form>
+</div>
+{%endblock%}
+
+{%block javascript%}
+<script>
+ var lastTaxonId = null;
+
+ var fetch_taxonomy = (taxonId) => {
+ var uri = (
+ "https://rest.uniprot.org/taxonomy/" + encodeURIComponent(taxonId));
+ $.get(
+ uri,
+ {},
+ (data, textStatus, jqXHR) => {
+ if(textStatus == "success") {
+ lastTaxonId = taxonId;
+ $("#txt-species-scientific").val(data.scientificName);
+ $("#txt-species-name").val(data.commonName);
+ return false;
+ }
+ msg = (
+ "Request to '${uri}' failed with message '${textStatus}'. "
+ + "Please try again later, or fill the details manually.");
+ alert(msg);
+ console.error(msg, data, textStatus);
+ return false;
+ },
+ "json");
+ };
+
+ $("#btn-search-taxonid").on("click", (event) => {
+ event.preventDefault();
+ taxonId = $("#txt-taxonomy-id").val();
+ if((taxonId !== "") && (taxonId !== lastTaxonId)) {
+ fetch_taxonomy(taxonId);
+ }
+ });
+</script>
+{%endblock%}
diff --git a/uploader/templates/species/edit-species.html b/uploader/templates/species/edit-species.html
new file mode 100644
index 0000000..5a26455
--- /dev/null
+++ b/uploader/templates/species/edit-species.html
@@ -0,0 +1,177 @@
+{%extends "species/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Edit Species{%endblock%}
+
+{%block pagetitle%}Edit Species{%endblock%}
+
+{%block css%}
+<style type="text/css">
+ .card {
+ margin-top: 0.3em;
+ border-width: 1px;
+ border-style: solid;
+ border-radius: 0.3em;
+ border-color: #AAAAAA;
+ padding: 0.5em;
+ }
+</style>
+{%endblock%}
+
+{%block lvl2_breadcrumbs%}
+<li {%if activelink=="edit-species"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.edit_species_extra',
+ species_id=species.SpeciesId)}}">Edit</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+<div class="row">
+ <form id="frm-edit-species"
+ method="POST"
+ action="{{url_for('species.edit_species_extra',
+ species_id=species.SpeciesId)}}">
+
+ <legend>Edit Extra Detail for Species '{{species.FullName}}'</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+
+ <div class="form-group">
+ <label for="lbl-species-taxonid" class="form-label">
+ Taxonomy Id
+ </label>
+ <label id="lbl-species-taxonid"
+ disabled="disabled"
+ class="form-control">{{species.TaxonomyId}}</label>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-species-name" class="form-label">
+ Common Name
+ </label>
+ <input type="text"
+ id="txt-species-name"
+ name="species_name"
+ required="required"
+ value="{{species.SpeciesName}}"
+ class="form-control" />
+ <small class="form-text text-muted">
+ This is the layman's name for the species, e.g. mouse</mall>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-species-fullname" class="form-label">
+ Scientific Name
+ </label>
+ <input type="text"
+ id="txt-species-fullname"
+ name="species_fullname"
+ required="required"
+ value="{{species.FullName}}"
+ class="form-control" />
+ <small class="form-text text-muted">
+ A scientific name for the species that mostly adheres to the biological
+ binomial nomenclature system.</small>
+ </div>
+
+ <div class="form-group">
+ <label for="select-species-family" class="form-label">
+ Family
+ </label>
+ <select id="select-species-family"
+ name="species_family"
+ class="form-control">
+ <option value="">Select the family</option>
+ {%for family in families%}
+ <option value="{{family}}"
+ {%if species.Family == family%}
+ selected="selected"
+ {%endif%}>{{family}}</option>
+ {%endfor%}
+ </select>
+ <small class="form-text text-muted">
+ A general classification for the species. This is mostly for use in
+ GeneNetwork's menus.</small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-species-familyorderid" class="form-label">
+ Family Order Id
+ </label>
+ <input type="number"
+ id="txt-species-familyorderid"
+ name="species_familyorderid"
+ value="{{species.FamilyOrderId}}"
+ required="required"
+ class="form-control" />
+ <small class="form-text text-muted">
+ This is a number that determines the order of the "Family" groupings
+ above in the GeneNetwork menus. This is an integer value that's manually
+ assigned.</small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-species-orderid" class="form-label">
+ Order Id
+ </label>
+ <input type="number"
+ id="txt-species-orderid"
+ name="species_orderid"
+ value="{{species.OrderId or (max_order_id + 5)}}"
+ class="form-control" />
+ <small class="form-text text-muted">
+ This integer value determines the order of the species in relation to
+ each other, but also within the respective "Family" groups.</small>
+ </div>
+
+ <div class="form-group">
+ <input type="submit" value="Submit Changes" class="btn btn-primary" />
+ </div>
+
+ </form>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+
+<div class="card">
+ <div class="card-body">
+ <h5 class="card-title">Family Order</h5>
+ <div class="card-text">
+ <p>The current family order is as follows</p>
+ <table class="table">
+ <thead>
+ <tr>
+ <th>Family Order Id</th>
+ <th>Family</th>
+ </tr>
+ </thead>
+ <tbody>
+ {%for item in family_order%}
+ <tr>
+ <td>{{item[0]}}</td>
+ <td>{{item[1]}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+ </div>
+ </div>
+</div>
+
+<div class="card">
+ <div class="card-body">
+ <h5 class="card-title">Order ID</h5>
+ <div class="card-text">
+ <p>The current largest OrderID is: {{max_order_id}}</p>
+ <p>We recommend giving a new species an order ID that is five more than
+ the current highest i.e. {{max_order_id + 5}}.</p>
+ </div>
+ </div>
+</div>
+{%endblock%}
diff --git a/uploader/templates/species/list-species.html b/uploader/templates/species/list-species.html
new file mode 100644
index 0000000..64084b0
--- /dev/null
+++ b/uploader/templates/species/list-species.html
@@ -0,0 +1,75 @@
+{%extends "species/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}List Species{%endblock%}
+
+{%block pagetitle%}List Species{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+<div class="row">
+ <p>
+ All data in GeneNetwork revolves around species. This is the core of the
+ system.</p>
+ <p>Here you can see a list of all the species available in GeneNetwork.
+ Click on the link besides each species to view greater detail on the species,
+ and access further operations that are possible for said species.</p>
+</div>
+
+<div class="row">
+ <p>If you cannot find the species you are looking for below, click the button
+ below to create it</p>
+ <p><a href="{{url_for('species.create_species')}}"
+ title="Add a new species to GeneNetwork"
+ class="btn btn-danger">Create Species</a></p>
+</div>
+
+<div class="row">
+ <table class="table">
+ <caption>Available Species</caption>
+ <thead>
+ <tr>
+ <th></td>
+ <th title="A common, layman's name for the species.">Common Name</th>
+ <th title="The scientific name for the species">Organism Name</th>
+ <th title="An identifier for the species in the NCBI taxonomy database">
+ Taxonomy ID
+ </th>
+ <th title="A generic grouping used internally by GeneNetwork for organising species.">
+ Family
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {%for species in allspecies%}
+ <tr>
+ <td>{{species["sequence_number"]}}</td>
+ <td>{{species["SpeciesName"]}}</td>
+ <td>
+ <a href="{{url_for('species.view_species',
+ species_id=species['SpeciesId'])}}"
+ title="View details in GeneNetwork on {{species['FullName']}}">
+ {{species["FullName"]}}
+ </a>
+ </td>
+ <td>
+ <a href="https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id={{species['TaxonomyId']}}"
+ title="View species details on NCBI"
+ target="_blank">{{species["TaxonomyId"]}}</a>
+ </td>
+ <td>{{species.Family}}</td>
+ </tr>
+ {%else%}
+ <tr>
+ <td colspan="3">
+ <p class="text-danger">
+ <span class="glyphicon glyphicon-exclamation-mark"></span>
+ There were no species found!
+ </p>
+ </td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+</div>
+{%endblock%}
diff --git a/uploader/templates/species/macro-display-species-card.html b/uploader/templates/species/macro-display-species-card.html
new file mode 100644
index 0000000..166c7b9
--- /dev/null
+++ b/uploader/templates/species/macro-display-species-card.html
@@ -0,0 +1,22 @@
+{%macro display_species_card(species)%}
+<div class="card">
+ <div class="card-body">
+ <h5 class="card-title">Species</h5>
+ <div class="card-text">
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Common Name</td>
+ <td>{{species.SpeciesName}}</td>
+ </tr>
+
+ <tr>
+ <td>Scientific Name</td>
+ <td>{{species.FullName}}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+</div>
+{%endmacro%}
diff --git a/uploader/templates/species/macro-select-species.html b/uploader/templates/species/macro-select-species.html
new file mode 100644
index 0000000..3714ae4
--- /dev/null
+++ b/uploader/templates/species/macro-select-species.html
@@ -0,0 +1,59 @@
+{%from "macro-step-indicator.html" import step_indicator%}
+
+{%macro select_species_form(form_action, species)%}
+<form method="GET" action="{{form_action}}" class="form-horizontal">
+
+ <h2>{{step_indicator("1")}} What species do you want to work with?</h2>
+
+ {%if species | length != 0%}
+
+ <p class="form-text">Search for, and select the species from the table below
+ and click "Continue"</p>
+
+ <div class="radio">
+ <label for="rdo-cant-find-species"
+ style="font-weight: 1;">
+ <input id="rdo-cant-find-species" type="radio" name="species_id"
+ value="CREATE-SPECIES" />
+ I could not find the species I want (create it).
+ </label>
+ </div>
+
+ <div class="col-sm-offset-10 col-sm-2">
+ <input type="submit"
+ class="btn btn-primary"
+ value="continue" />
+ </div>
+
+ <div style="margin-top:3em;">
+ <table id="tbl-select-species" class="table compact stripe"
+ data-species-list='{{species | tojson}}'>
+ <div class="">
+ <thead>
+ <tr>
+ <th></th>
+ <th>Species Name</th>
+ </tr>
+ </thead>
+
+ <tbody></tbody>
+ </table>
+ </div>
+
+ {%else%}
+
+ <label class="control-label" for="rdo-cant-find-species">
+ <input id="rdo-cant-find-species" type="radio" name="species_id"
+ value="CREATE-SPECIES" />
+ There are no species to select from. Create the first one.</label>
+
+ <div class="col-sm-offset-10 col-sm-2">
+ <input type="submit"
+ class="btn btn-primary col-sm-offset-1"
+ value="continue" />
+ </div>
+
+ {%endif%}
+
+</form>
+{%endmacro%}
diff --git a/uploader/templates/species/view-species.html b/uploader/templates/species/view-species.html
new file mode 100644
index 0000000..2d02f7e
--- /dev/null
+++ b/uploader/templates/species/view-species.html
@@ -0,0 +1,90 @@
+{%extends "species/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}View Species{%endblock%}
+
+{%block pagetitle%}View Species{%endblock%}
+
+{%block lvl2_breadcrumbs%}
+<li {%if activelink=="view-species"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.view_species', species_id=species.SpeciesId)}}">View</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+<div class="row">
+ <h2>Details on species {{species.FullName}}</h2>
+
+ <dl>
+ <dt>Common Name</dt>
+ <dd>{{species.SpeciesName}}</dd>
+
+ <dt>Scientific Name</dt>
+ <dd>{{species.FullName}}</dd>
+
+ <dt>Taxonomy ID</dt>
+ <dd>{{species.TaxonomyId}}</dd>
+ </dl>
+
+ <h3>Actions</h3>
+
+ <p>
+ You can proceed to perform any of the following actions for species
+ {{species.FullName}}
+ </p>
+
+ <ol>
+ <li>
+ <a href="{{url_for('species.populations.list_species_populations',
+ species_id=species.SpeciesId)}}"
+ title="Create/Edit populations for {{species.FullName}}">
+ Manage populations</a>
+ </li>
+ <li>
+ <a href="{{url_for('species.platforms.list_platforms',
+ species_id=species.SpeciesId)}}"
+ title="Create/Edit sequencing platforms for {{species.FullName}}">
+ Manage sequencing platforms</a>
+ </li>
+ </ol>
+
+
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+<div class="card">
+ <div class="card-body">
+ <h5 class="card-title">Species Extras</h5>
+ <div class="card-text">
+ <p>Some extra internal-use details (mostly for UI concerns on GeneNetwork)</p>
+ <p>
+ <small>
+ If you do not understand what the following are about, simply ignore them
+ &mdash;
+ They have no bearing whatsoever on your data, or its analysis.
+ </small>
+ </p>
+ <dl>
+ <dt>Family</dt>
+ <dd>{{species.Family}}</dd>
+
+ <dt>FamilyOrderId</dt>
+ <dd>{{species.FamilyOrderId}}</dd>
+
+ <dt>OrderId</dt>
+ <dd>{{species.OrderId}}</dd>
+ </dl>
+ </div>
+ <a href="{{url_for('species.edit_species_extra',
+ species_id=species.SpeciesId)}}"
+ class="card-link"
+ title="Edit the species' internal-use details.">Edit</a>
+ </div>
+</div>
+{%endblock%}
diff --git a/uploader/templates/stdout_output.html b/uploader/templates/stdout_output.html
new file mode 100644
index 0000000..85345a9
--- /dev/null
+++ b/uploader/templates/stdout_output.html
@@ -0,0 +1,8 @@
+{%macro stdout_output(job)%}
+
+<h4>STDOUT Output</h4>
+<div class="cli-output">
+ <pre>{{job.get("stdout", "")}}</pre>
+</div>
+
+{%endmacro%}
diff --git a/uploader/templates/unhandled_exception.html b/uploader/templates/unhandled_exception.html
new file mode 100644
index 0000000..cfb0c0b
--- /dev/null
+++ b/uploader/templates/unhandled_exception.html
@@ -0,0 +1,24 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}System Error{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
+{%endblock%}
+
+{%block contents%}
+<div class="row">
+ {{flash_all_messages()}}
+ <h1>Exception!</h1>
+
+ <p>An error has occured, and your request has been aborted. Please notify the
+ administrator to try and get this fixed.</p>
+ <p>The system has failed with the following error:</p>
+</div>
+<div class="row">
+ <pre>
+ {{trace}}
+ </pre>
+</div>
+{%endblock%}
diff --git a/uploader/templates/upload_progress_indicator.html b/uploader/templates/upload_progress_indicator.html
new file mode 100644
index 0000000..e274e83
--- /dev/null
+++ b/uploader/templates/upload_progress_indicator.html
@@ -0,0 +1,35 @@
+{%macro upload_progress_indicator()%}
+<div id="upload-progress-indicator" class="modal fade" tabindex="-1" role="dialog">
+ <div class="modal-dialog" role="document">
+ <div class="modal-content">
+ <div class="modal-header">
+ <h3 class="modal-title">Uploading file</h3>
+ </div>
+
+ <div class="modal-body">
+ <form id="frm-cancel-upload" style="border-style: none;">
+ <div class="form-group">
+ <span id="progress-filename" class="form-text">No file selected!</span>
+ <progress id="progress-bar" value="0" max="100" class="form-control">
+ 0</progress>
+ </div>
+
+ <div class="form-group">
+ <span class="form-text text-muted" id="progress-text">
+ Uploading 0%</span>
+ <span class="form-text text-muted" id="progress-extra-text">
+ Processing</span>
+ </div>
+ </form>
+ </div>
+
+ <div class="modal-footer">
+ <button id="btn-cancel-upload"
+ type="button"
+ class="btn btn-danger"
+ data-dismiss="modal">Cancel</button>
+ </div>
+ </div>
+ </div>
+</div>
+{%endmacro%}
diff --git a/uploader/templates/worker_failure.html b/uploader/templates/worker_failure.html
new file mode 100644
index 0000000..b65b140
--- /dev/null
+++ b/uploader/templates/worker_failure.html
@@ -0,0 +1,24 @@
+{%extends "base.html"%}
+
+{%block title%}Worker Failure{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Worker Failure</h1>
+
+<p>
+ There was a critical failure launching the job to parse your file.
+ This is our fault and (probably) has nothing to do with the file you uploaded.
+</p>
+
+<p>
+ Please notify the developers of this issue when you encounter it,
+ providing the link to this page, or the information below.
+</p>
+
+<h4>Debugging Information</h4>
+
+<ul>
+ <li><strong>job id</strong>: {{job_id}}</li>
+</ul>
+
+{%endblock%}
diff --git a/uploader/ui.py b/uploader/ui.py
new file mode 100644
index 0000000..1994056
--- /dev/null
+++ b/uploader/ui.py
@@ -0,0 +1,14 @@
+"""Utilities to handle the UI"""
+from flask import render_template as flask_render_template
+
+def make_template_renderer(default):
+ """Render template for species."""
+ def render_template(template, **kwargs):
+ return flask_render_template(
+ template,
+ **{
+ **kwargs,
+ "activemenu": default,
+ "activelink": kwargs.get("activelink", default)
+ })
+ return render_template