aboutsummaryrefslogtreecommitdiff
path: root/uploader
diff options
context:
space:
mode:
Diffstat (limited to 'uploader')
-rw-r--r--uploader/__init__.py48
-rw-r--r--uploader/base_routes.py29
-rw-r--r--uploader/check_connections.py28
-rw-r--r--uploader/db/__init__.py8
-rw-r--r--uploader/db/averaging.py23
-rw-r--r--uploader/db/datasets.py133
-rw-r--r--uploader/db/platforms.py25
-rw-r--r--uploader/db/populations.py54
-rw-r--r--uploader/db/species.py22
-rw-r--r--uploader/db/tissues.py50
-rw-r--r--uploader/db_utils.py46
-rw-r--r--uploader/dbinsert.py397
-rw-r--r--uploader/default_settings.py14
-rw-r--r--uploader/entry.py127
-rw-r--r--uploader/errors.py29
-rw-r--r--uploader/files.py26
-rw-r--r--uploader/input_validation.py27
-rw-r--r--uploader/jobs.py130
-rw-r--r--uploader/parse.py175
-rw-r--r--uploader/samples.py354
-rw-r--r--uploader/static/css/custom-bootstrap.css23
-rw-r--r--uploader/static/css/styles.css7
-rw-r--r--uploader/static/css/two-column-with-separator.css27
-rw-r--r--uploader/static/images/CITGLogo.pngbin0 -> 11962 bytes
-rw-r--r--uploader/static/js/select_platform.js70
-rw-r--r--uploader/static/js/upload_progress.js97
-rw-r--r--uploader/static/js/upload_samples.js132
-rw-r--r--uploader/static/js/utils.js10
-rw-r--r--uploader/templates/base.html51
-rw-r--r--uploader/templates/cli-output.html8
-rw-r--r--uploader/templates/continue_from_create_dataset.html52
-rw-r--r--uploader/templates/continue_from_create_study.html52
-rw-r--r--uploader/templates/data_review.html85
-rw-r--r--uploader/templates/dbupdate_error.html12
-rw-r--r--uploader/templates/dbupdate_hidden_fields.html29
-rw-r--r--uploader/templates/errors_display.html47
-rw-r--r--uploader/templates/final_confirmation.html47
-rw-r--r--uploader/templates/flash_messages.html25
-rw-r--r--uploader/templates/http-error.html18
-rw-r--r--uploader/templates/index.html81
-rw-r--r--uploader/templates/insert_error.html32
-rw-r--r--uploader/templates/insert_progress.html46
-rw-r--r--uploader/templates/insert_success.html19
-rw-r--r--uploader/templates/job_progress.html40
-rw-r--r--uploader/templates/no_such_job.html14
-rw-r--r--uploader/templates/parse_failure.html26
-rw-r--r--uploader/templates/parse_results.html30
-rw-r--r--uploader/templates/rqtl2/create-geno-dataset-success.html55
-rw-r--r--uploader/templates/rqtl2/create-probe-dataset-success.html59
-rw-r--r--uploader/templates/rqtl2/create-probe-study-success.html49
-rw-r--r--uploader/templates/rqtl2/create-tissue-success.html106
-rw-r--r--uploader/templates/rqtl2/index.html36
-rw-r--r--uploader/templates/rqtl2/no-such-job.html13
-rw-r--r--uploader/templates/rqtl2/rqtl2-job-error.html39
-rw-r--r--uploader/templates/rqtl2/rqtl2-job-results.html24
-rw-r--r--uploader/templates/rqtl2/rqtl2-job-status.html20
-rw-r--r--uploader/templates/rqtl2/rqtl2-qc-job-error.html120
-rw-r--r--uploader/templates/rqtl2/rqtl2-qc-job-results.html66
-rw-r--r--uploader/templates/rqtl2/rqtl2-qc-job-status.html41
-rw-r--r--uploader/templates/rqtl2/rqtl2-qc-job-success.html37
-rw-r--r--uploader/templates/rqtl2/select-geno-dataset.html144
-rw-r--r--uploader/templates/rqtl2/select-population.html136
-rw-r--r--uploader/templates/rqtl2/select-probeset-dataset.html191
-rw-r--r--uploader/templates/rqtl2/select-probeset-study-id.html143
-rw-r--r--uploader/templates/rqtl2/select-tissue.html115
-rw-r--r--uploader/templates/rqtl2/summary-info.html65
-rw-r--r--uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html276
-rw-r--r--uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html33
-rw-r--r--uploader/templates/samples/select-population.html99
-rw-r--r--uploader/templates/samples/select-species.html30
-rw-r--r--uploader/templates/samples/upload-failure.html27
-rw-r--r--uploader/templates/samples/upload-progress.html22
-rw-r--r--uploader/templates/samples/upload-samples.html139
-rw-r--r--uploader/templates/samples/upload-success.html18
-rw-r--r--uploader/templates/select_dataset.html161
-rw-r--r--uploader/templates/select_platform.html82
-rw-r--r--uploader/templates/select_species.html92
-rw-r--r--uploader/templates/select_study.html108
-rw-r--r--uploader/templates/stdout_output.html8
-rw-r--r--uploader/templates/unhandled_exception.html21
-rw-r--r--uploader/templates/upload_progress_indicator.html35
-rw-r--r--uploader/templates/worker_failure.html24
-rw-r--r--uploader/upload/__init__.py7
-rw-r--r--uploader/upload/rqtl2.py1157
84 files changed, 6623 insertions, 0 deletions
diff --git a/uploader/__init__.py b/uploader/__init__.py
new file mode 100644
index 0000000..3ee8aa0
--- /dev/null
+++ b/uploader/__init__.py
@@ -0,0 +1,48 @@
+"""The Quality-Control Web Application entry point"""
+import os
+import logging
+from pathlib import Path
+
+from flask import Flask, request
+
+from .entry import entrybp
+from .upload import upload
+from .parse import parsebp
+from .samples import samples
+from .base_routes import base
+from .dbinsert import dbinsertbp
+from .errors import register_error_handlers
+
+def override_settings_with_envvars(
+ app: Flask, ignore: tuple[str, ...]=tuple()) -> None:
+ """Override settings in `app` with those in ENVVARS"""
+ for setting in (key for key in app.config if key not in ignore):
+ app.config[setting] = os.environ.get(setting) or app.config[setting]
+
+
+def create_app():
+ """The application factory"""
+ app = Flask(__name__)
+ app.config.from_pyfile(
+ Path(__file__).parent.joinpath("default_settings.py"))
+ if "QCAPP_CONF" in os.environ:
+ app.config.from_envvar("QCAPP_CONF") # Override defaults with instance path
+
+ override_settings_with_envvars(app, ignore=tuple())
+
+ if "QCAPP_SECRETS" in os.environ:
+ app.config.from_envvar("QCAPP_SECRETS")
+
+ # setup jinja2 symbols
+ app.jinja_env.globals.update(request_url=lambda : request.url)
+
+ # setup blueprints
+ app.register_blueprint(base, url_prefix="/")
+ app.register_blueprint(entrybp, url_prefix="/")
+ app.register_blueprint(parsebp, url_prefix="/parse")
+ app.register_blueprint(upload, url_prefix="/upload")
+ app.register_blueprint(dbinsertbp, url_prefix="/dbinsert")
+ app.register_blueprint(samples, url_prefix="/samples")
+
+ register_error_handlers(app)
+ return app
diff --git a/uploader/base_routes.py b/uploader/base_routes.py
new file mode 100644
index 0000000..9daf439
--- /dev/null
+++ b/uploader/base_routes.py
@@ -0,0 +1,29 @@
+"""Basic routes required for all pages"""
+import os
+from flask import Blueprint, send_from_directory
+
+base = Blueprint("base", __name__)
+
+def appenv():
+ """Get app's guix environment path."""
+ return os.environ.get("GN_UPLOADER_ENVIRONMENT")
+
+@base.route("/bootstrap/<path:filename>")
+def bootstrap(filename):
+ """Fetch bootstrap files."""
+ return send_from_directory(
+ appenv(), f"share/genenetwork2/javascript/bootstrap/{filename}")
+
+
+@base.route("/jquery/<path:filename>")
+def jquery(filename):
+ """Fetch jquery files."""
+ return send_from_directory(
+ appenv(), f"share/genenetwork2/javascript/jquery/{filename}")
+
+
+@base.route("/node-modules/<path:filename>")
+def node_modules(filename):
+ """Fetch node-js modules."""
+ return send_from_directory(
+ appenv(), f"lib/node_modules/{filename}")
diff --git a/uploader/check_connections.py b/uploader/check_connections.py
new file mode 100644
index 0000000..2561e55
--- /dev/null
+++ b/uploader/check_connections.py
@@ -0,0 +1,28 @@
+"""Check the various connection used in the application"""
+import sys
+import traceback
+
+import redis
+import MySQLdb
+
+from uploader.db_utils import database_connection
+
+def check_redis(uri: str):
+ "Check the redis connection"
+ try:
+ with redis.Redis.from_url(uri) as rconn:
+ rconn.ping()
+ except redis.exceptions.ConnectionError as conn_err:
+ print(conn_err, file=sys.stderr)
+ print(traceback.format_exc(), file=sys.stderr)
+ sys.exit(1)
+
+def check_db(uri: str):
+ "Check the mysql connection"
+ try:
+ with database_connection(uri) as dbconn: # pylint: disable=[unused-variable]
+ pass
+ except MySQLdb.OperationalError as op_err:
+ print(op_err, file=sys.stderr)
+ print(traceback.format_exc(), file=sys.stderr)
+ sys.exit(1)
diff --git a/uploader/db/__init__.py b/uploader/db/__init__.py
new file mode 100644
index 0000000..36e93e8
--- /dev/null
+++ b/uploader/db/__init__.py
@@ -0,0 +1,8 @@
+"""Database functions"""
+from .species import species, species_by_id
+from .populations import (
+ save_population,
+ population_by_id,
+ populations_by_species,
+ population_by_species_and_id)
+from .datasets import geno_datasets_by_species_and_population
diff --git a/uploader/db/averaging.py b/uploader/db/averaging.py
new file mode 100644
index 0000000..62bbe67
--- /dev/null
+++ b/uploader/db/averaging.py
@@ -0,0 +1,23 @@
+"""Functions for db interactions for averaging methods"""
+from typing import Optional
+
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+def averaging_methods(conn: mdb.Connection) -> tuple[dict, ...]:
+ """Fetch all available averaging methods"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM AvgMethod")
+ return tuple(dict(row) for row in cursor.fetchall())
+
+def averaging_method_by_id(
+ conn: mdb.Connection, averageid: int) -> Optional[dict]:
+ """Fetch the averaging method by its ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM AvgMethod WHERE Id=%s",
+ (averageid,))
+ result = cursor.fetchone()
+ if bool(result):
+ return dict(result)
+
+ return None
diff --git a/uploader/db/datasets.py b/uploader/db/datasets.py
new file mode 100644
index 0000000..767ec41
--- /dev/null
+++ b/uploader/db/datasets.py
@@ -0,0 +1,133 @@
+"""Functions for accessing the database relating to datasets."""
+from datetime import date
+from typing import Optional
+
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+def geno_datasets_by_species_and_population(
+ conn: mdb.Connection,
+ speciesid: int,
+ populationid: int) -> tuple[dict, ...]:
+ """Retrieve all genotypes datasets by species and population"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT gf.* FROM InbredSet AS iset INNER JOIN GenoFreeze AS gf "
+ "ON iset.InbredSetId=gf.InbredSetId "
+ "WHERE iset.SpeciesId=%(sid)s AND iset.InbredSetId=%(pid)s",
+ {"sid": speciesid, "pid": populationid})
+ return tuple(dict(row) for row in cursor.fetchall())
+
+def geno_dataset_by_id(conn: mdb.Connection, dataset_id) -> Optional[dict]:
+ """Retrieve genotype dataset by ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s", (dataset_id,))
+ _dataset = cursor.fetchone()
+ return dict(_dataset) if bool(_dataset) else None
+
+def probeset_studies_by_species_and_population(
+ conn: mdb.Connection,
+ speciesid: int,
+ populationid: int) -> tuple[dict, ...]:
+ """Retrieve all probesets"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT pf.* FROM InbredSet AS iset INNER JOIN ProbeFreeze AS pf "
+ "ON iset.InbredSetId=pf.InbredSetId "
+ "WHERE iset.SpeciesId=%(sid)s AND iset.InbredSetId=%(pid)s",
+ {"sid": speciesid, "pid": populationid})
+ return tuple(dict(row) for row in cursor.fetchall())
+
+def probeset_datasets_by_study(conn: mdb.Connection,
+ studyid: int) -> tuple[dict, ...]:
+ """Retrieve all probeset databases by study."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s",
+ (studyid,))
+ return tuple(dict(row) for row in cursor.fetchall())
+
+def probeset_study_by_id(conn: mdb.Connection, studyid) -> Optional[dict]:
+ """Retrieve ProbeSet study by ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM ProbeFreeze WHERE Id=%s", (studyid,))
+ _study = cursor.fetchone()
+ return dict(_study) if bool(_study) else None
+
+def probeset_create_study(conn: mdb.Connection,#pylint: disable=[too-many-arguments]
+ populationid: int,
+ platformid: int,
+ tissueid: int,
+ studyname: str,
+ studyfullname: str = "",
+ studyshortname: str = ""):
+ """Create a new ProbeSet study."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ studydata = {
+ "platid": platformid,
+ "tissueid": tissueid,
+ "name": studyname,
+ "fname": studyfullname or studyname,
+ "sname": studyshortname,
+ "today": date.today().isoformat(),
+ "popid": populationid
+ }
+ cursor.execute(
+ """
+ INSERT INTO ProbeFreeze(
+ ChipId, TissueId, Name, FullName, ShortName, CreateTime,
+ InbredSetId
+ ) VALUES (
+ %(platid)s, %(tissueid)s, %(name)s, %(fname)s, %(sname)s,
+ %(today)s, %(popid)s
+ )
+ """,
+ studydata)
+ studyid = cursor.lastrowid
+ cursor.execute("UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s",
+ (studyid, studyid))
+ return {**studydata, "studyid": studyid}
+
+def probeset_create_dataset(conn: mdb.Connection,#pylint: disable=[too-many-arguments]
+ studyid: int,
+ averageid: int,
+ datasetname: str,
+ datasetfullname: str,
+ datasetshortname: str="",
+ public: bool = True,
+ datascale="log2") -> dict:
+ """Create a new ProbeSet dataset."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ dataset = {
+ "studyid": studyid,
+ "averageid": averageid,
+ "name2": datasetname,
+ "fname": datasetfullname,
+ "name": datasetshortname,
+ "sname": datasetshortname,
+ "today": date.today().isoformat(),
+ "public": 2 if public else 0,
+ "authorisedusers": "williamslab",
+ "datascale": datascale
+ }
+ cursor.execute(
+ """
+ INSERT INTO ProbeSetFreeze(
+ ProbeFreezeId, AvgId, Name, Name2, FullName, ShortName,
+ CreateTime, public, AuthorisedUsers, DataScale)
+ VALUES(
+ %(studyid)s, %(averageid)s, %(name)s, %(name2)s, %(fname)s,
+ %(sname)s, %(today)s, %(public)s, %(authorisedusers)s,
+ %(datascale)s)
+ """,
+ dataset)
+ return {**dataset, "datasetid": cursor.lastrowid}
+
+def probeset_dataset_by_id(conn: mdb.Connection, datasetid) -> Optional[dict]:
+ """Fetch a ProbeSet dataset by its ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Id=%s", (datasetid,))
+ result = cursor.fetchone()
+ if bool(result):
+ return dict(result)
+
+ return None
diff --git a/uploader/db/platforms.py b/uploader/db/platforms.py
new file mode 100644
index 0000000..cb527a7
--- /dev/null
+++ b/uploader/db/platforms.py
@@ -0,0 +1,25 @@
+"""Handle db interactions for platforms."""
+from typing import Optional
+
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+def platforms_by_species(
+ conn: mdb.Connection, speciesid: int) -> tuple[dict, ...]:
+ """Retrieve platforms by the species"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM GeneChip WHERE SpeciesId=%s "
+ "ORDER BY GeneChipName ASC",
+ (speciesid,))
+ return tuple(dict(row) for row in cursor.fetchall())
+
+def platform_by_id(conn: mdb.Connection, platformid: int) -> Optional[dict]:
+ """Retrieve a platform by its ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM GeneChip WHERE Id=%s",
+ (platformid,))
+ result = cursor.fetchone()
+ if bool(result):
+ return dict(result)
+
+ return None
diff --git a/uploader/db/populations.py b/uploader/db/populations.py
new file mode 100644
index 0000000..4485e52
--- /dev/null
+++ b/uploader/db/populations.py
@@ -0,0 +1,54 @@
+"""Functions for accessing the database relating to species populations."""
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+def population_by_id(conn: mdb.Connection, population_id) -> dict:
+ """Get the grouping/population by id."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM InbredSet WHERE InbredSetId=%s",
+ (population_id,))
+ return cursor.fetchone()
+
+def population_by_species_and_id(
+ conn: mdb.Connection, species_id, population_id) -> dict:
+ """Retrieve a population by its identifier and species."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM InbredSet WHERE SpeciesId=%s AND Id=%s",
+ (species_id, population_id))
+ return cursor.fetchone()
+
+def populations_by_species(conn: mdb.Connection, speciesid) -> tuple:
+ "Retrieve group (InbredSet) information from the database."
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ query = "SELECT * FROM InbredSet WHERE SpeciesId=%s"
+ cursor.execute(query, (speciesid,))
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+def save_population(conn: mdb.Connection, population_details: dict) -> dict:
+ """Save the population details to the db."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "INSERT INTO InbredSet("
+ "InbredSetId, InbredSetName, Name, SpeciesId, FullName, "
+ "MenuOrderId, Description"
+ ") "
+ "VALUES ("
+ "%(InbredSetId)s, %(InbredSetName)s, %(Name)s, %(SpeciesId)s, "
+ "%(FullName)s, %(MenuOrderId)s, %(Description)s"
+ ")",
+ {
+ "MenuOrderId": 0,
+ "InbredSetId": 0,
+ **population_details
+ })
+ new_id = cursor.lastrowid
+ cursor.execute("UPDATE InbredSet SET InbredSetId=%s WHERE Id=%s",
+ (new_id, new_id))
+ return {
+ **population_details,
+ "Id": new_id,
+ "InbredSetId": new_id,
+ "population_id": new_id
+ }
diff --git a/uploader/db/species.py b/uploader/db/species.py
new file mode 100644
index 0000000..653e59b
--- /dev/null
+++ b/uploader/db/species.py
@@ -0,0 +1,22 @@
+"""Database functions for species."""
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+def species(conn: mdb.Connection) -> tuple:
+ "Retrieve the species from the database."
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT SpeciesId, SpeciesName, LOWER(Name) AS Name, MenuName, "
+ "FullName FROM Species")
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+def species_by_id(conn: mdb.Connection, speciesid) -> dict:
+ "Retrieve the species from the database by id."
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT SpeciesId, SpeciesName, LOWER(Name) AS Name, MenuName, "
+ "FullName FROM Species WHERE SpeciesId=%s",
+ (speciesid,))
+ return cursor.fetchone()
diff --git a/uploader/db/tissues.py b/uploader/db/tissues.py
new file mode 100644
index 0000000..9fe7bab
--- /dev/null
+++ b/uploader/db/tissues.py
@@ -0,0 +1,50 @@
+"""Handle db interactions for tissue."""
+from typing import Union, Optional
+
+import MySQLdb as mdb
+from MySQLdb.cursors import DictCursor
+
+def all_tissues(conn: mdb.Connection) -> tuple[dict, ...]:
+ """All available tissue."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM Tissue ORDER BY TissueName")
+ return tuple(dict(row) for row in cursor.fetchall())
+
+
+def tissue_by_id(conn: mdb.Connection, tissueid) -> Optional[dict]:
+ """Retrieve a tissue by its ID"""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM Tissue WHERE Id=%s", (tissueid,))
+ result = cursor.fetchone()
+ if bool(result):
+ return dict(result)
+
+ return None
+
+
+def create_new_tissue(
+ conn: mdb.Connection,
+ name: str,
+ shortname: str,
+ birnlexid: Optional[str] = None,
+ birnlexname: Optional[str] = None
+) -> dict[str, Union[int, str, None]]:
+ """Add a new tissue, organ or biological material to the database."""
+ with conn.cursor() as cursor:
+ cursor.execute(
+ "INSERT INTO "
+ "Tissue(TissueName, Name, Short_Name, BIRN_lex_ID, BIRN_lex_Name) "
+ "VALUES (%s, %s, %s, %s, %s)",
+ (name, name, shortname, birnlexid, birnlexname))
+ tissueid = cursor.lastrowid
+ cursor.execute("UPDATE Tissue SET TissueId=%s WHERE Id=%s",
+ (tissueid, tissueid))
+ return {
+ "Id": tissueid,
+ "TissueId": tissueid,
+ "TissueName": name,
+ "Name": name,
+ "Short_Name": shortname,
+ "BIRN_lex_ID": birnlexid,
+ "BIRN_lex_Name": birnlexname
+ }
diff --git a/uploader/db_utils.py b/uploader/db_utils.py
new file mode 100644
index 0000000..ef26398
--- /dev/null
+++ b/uploader/db_utils.py
@@ -0,0 +1,46 @@
+"""module contains all db related stuff"""
+import logging
+import traceback
+import contextlib
+from urllib.parse import urlparse
+from typing import Any, Tuple, Optional, Iterator, Callable
+
+import MySQLdb as mdb
+from redis import Redis
+from flask import current_app as app
+
+def parse_db_url(db_url) -> Tuple:
+ """
+ Parse SQL_URI configuration variable.
+ """
+ parsed_db = urlparse(db_url)
+ return (parsed_db.hostname, parsed_db.username,
+ parsed_db.password, parsed_db.path[1:], parsed_db.port)
+
+
+@contextlib.contextmanager
+def database_connection(db_url: Optional[str] = None) -> Iterator[mdb.Connection]:
+ """function to create db connector"""
+ host, user, passwd, db_name, db_port = parse_db_url(
+ db_url or app.config["SQL_URI"])
+ connection = mdb.connect(
+ host, user, passwd, db_name, port=(db_port or 3306))
+ try:
+ yield connection
+ connection.commit()
+ except mdb.Error as _mdb_err:
+ logging.error(traceback.format_exc())
+ connection.rollback()
+ finally:
+ connection.close()
+
+def with_db_connection(func: Callable[[mdb.Connection], Any]) -> Any:
+ """Call `func` with a MySQDdb database connection."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ return func(conn)
+
+def with_redis_connection(func: Callable[[Redis], Any]) -> Any:
+ """Call `func` with a redis connection."""
+ redisuri = app.config["REDIS_URL"]
+ with Redis.from_url(redisuri, decode_responses=True) as rconn:
+ return func(rconn)
diff --git a/uploader/dbinsert.py b/uploader/dbinsert.py
new file mode 100644
index 0000000..88d16ef
--- /dev/null
+++ b/uploader/dbinsert.py
@@ -0,0 +1,397 @@
+"Handle inserting data into the database"
+import os
+import json
+from typing import Union
+from functools import reduce
+from datetime import datetime
+
+from redis import Redis
+from MySQLdb.cursors import DictCursor
+from flask import (
+ flash, request, url_for, Blueprint, redirect, render_template,
+ current_app as app)
+
+from uploader.db_utils import with_db_connection, database_connection
+from uploader.db import species, species_by_id, populations_by_species
+
+from . import jobs
+
+dbinsertbp = Blueprint("dbinsert", __name__)
+
+def render_error(error_msg):
+ "Render the generic error page"
+ return render_template("dbupdate_error.html", error_message=error_msg), 400
+
+def make_menu_items_grouper(grouping_fn=lambda item: item):
+ "Build function to be used to group menu items."
+ def __grouper__(acc, row):
+ grouping = grouping_fn(row[2])
+ row_values = (row[0].strip(), row[1].strip())
+ if acc.get(grouping) is None:
+ return {**acc, grouping: (row_values,)}
+ return {**acc, grouping: (acc[grouping] + (row_values,))}
+ return __grouper__
+
+def genechips():
+ "Retrieve the genechip information from the database"
+ def __organise_by_species__(acc, chip):
+ speciesid = chip["SpeciesId"]
+ if acc.get(speciesid) is None:
+ return {**acc, speciesid: (chip,)}
+ return {**acc, speciesid: acc[speciesid] + (chip,)}
+
+ with database_connection() as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM GeneChip ORDER BY GeneChipName ASC")
+ return reduce(__organise_by_species__, cursor.fetchall(), {})
+
+ return {}
+
+def platform_by_id(genechipid:int) -> Union[dict, None]:
+ "Retrieve the gene platform by id"
+ with database_connection() as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT * FROM GeneChip WHERE GeneChipId=%s",
+ (genechipid,))
+ return cursor.fetchone()
+
+def studies_by_species_and_platform(speciesid:int, genechipid:int) -> tuple:
+ "Retrieve the studies by the related species and gene platform"
+ with database_connection() as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ query = (
+ "SELECT Species.SpeciesId, ProbeFreeze.* "
+ "FROM Species INNER JOIN InbredSet "
+ "ON Species.SpeciesId=InbredSet.SpeciesId "
+ "INNER JOIN ProbeFreeze "
+ "ON InbredSet.InbredSetId=ProbeFreeze.InbredSetId "
+ "WHERE Species.SpeciesId = %s "
+ "AND ProbeFreeze.ChipId = %s")
+ cursor.execute(query, (speciesid, genechipid))
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+def organise_groups_by_family(acc:dict, group:dict) -> dict:
+ "Organise the group (InbredSet) information by the group field"
+ family = group["Family"]
+ if acc.get(family):
+ return {**acc, family: acc[family] + (group,)}
+ return {**acc, family: (group,)}
+
+def tissues() -> tuple:
+ "Retrieve type (Tissue) information from the database."
+ with database_connection() as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM Tissue ORDER BY Name")
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+@dbinsertbp.route("/platform", methods=["POST"])
+def select_platform():
+ "Select the platform (GeneChipId) used for the data."
+ job_id = request.form["job_id"]
+ with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn,
+ database_connection(app.config["SQL_URI"]) as conn):
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+ if job:
+ filename = job["filename"]
+ filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}"
+ if os.path.exists(filepath):
+ default_species = 1
+ gchips = genechips()
+ return render_template(
+ "select_platform.html", filename=filename,
+ filetype=job["filetype"], totallines=int(job["currentline"]),
+ default_species=default_species, species=species(conn),
+ genechips=gchips[default_species],
+ genechips_data=json.dumps(gchips))
+ return render_error(f"File '{filename}' no longer exists.")
+ return render_error(f"Job '{job_id}' no longer exists.")
+ return render_error("Unknown error")
+
+@dbinsertbp.route("/study", methods=["POST"])
+def select_study():
+ "View to select/create the study (ProbeFreeze) associated with the data."
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+
+ speciesid = form["species"]
+ genechipid = form["genechipid"]
+
+ the_studies = studies_by_species_and_platform(speciesid, genechipid)
+ the_groups = reduce(
+ organise_groups_by_family,
+ with_db_connection(
+ lambda conn: populations_by_species(conn, speciesid)),
+ {})
+ return render_template(
+ "select_study.html", filename=form["filename"],
+ filetype=form["filetype"], totallines=form["totallines"],
+ species=speciesid, genechipid=genechipid, studies=the_studies,
+ groups=the_groups, tissues = tissues(),
+ selected_group=int(form.get("inbredsetid", -13)),
+ selected_tissue=int(form.get("tissueid", -13)))
+ except AssertionError as aserr:
+ return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/create-study", methods=["POST"])
+def create_study():
+ "Create a new study (ProbeFreeze)."
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+ assert form.get("studyname"), "study name"
+ assert form.get("inbredsetid"), "group"
+ assert form.get("tissueid"), "type/tissue"
+
+ with database_connection() as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ values = (
+ form["genechipid"],
+ form["tissueid"],
+ form["studyname"],
+ form.get("studyfullname", ""),
+ form.get("studyshortname", ""),
+ datetime.now().date().strftime("%Y-%m-%d"),
+ form["inbredsetid"])
+ query = (
+ "INSERT INTO ProbeFreeze("
+ "ChipId, TissueId, Name, FullName, ShortName, CreateTime, "
+ "InbredSetId"
+ ") VALUES (%s, %s, %s, %s, %s, %s, %s)")
+ cursor.execute(query, values)
+ new_studyid = cursor.lastrowid
+ cursor.execute(
+ "UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s",
+ (new_studyid, new_studyid))
+ flash("Study created successfully", "alert-success")
+ return render_template(
+ "continue_from_create_study.html",
+ filename=form["filename"], filetype=form["filetype"],
+ totallines=form["totallines"], species=form["species"],
+ genechipid=form["genechipid"], studyid=new_studyid)
+ except AssertionError as aserr:
+ flash(f"Missing data: {aserr.args[0]}", "alert-error")
+ return redirect(url_for("dbinsert.select_study"), code=307)
+
+def datasets_by_study(studyid:int) -> tuple:
+ "Retrieve datasets associated with a study with the ID `studyid`."
+ with database_connection() as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ query = "SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s"
+ cursor.execute(query, (studyid,))
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+def averaging_methods() -> tuple:
+ "Retrieve averaging methods from database"
+ with database_connection() as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM AvgMethod")
+ return tuple(cursor.fetchall())
+
+ return tuple()
+
+def dataset_datascales() -> tuple:
+ "Retrieve datascales from database"
+ with database_connection() as conn:
+ with conn.cursor() as cursor:
+ cursor.execute(
+ 'SELECT DISTINCT DataScale FROM ProbeSetFreeze '
+ 'WHERE DataScale IS NOT NULL AND DataScale != ""')
+ return tuple(
+ item for item in
+ (res[0].strip() for res in cursor.fetchall())
+ if (item is not None and item != ""))
+
+ return tuple()
+
+@dbinsertbp.route("/dataset", methods=["POST"])
+def select_dataset():
+ "Select the dataset to add the file contents against"
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+ assert form.get("studyid"), "study"
+
+ studyid = form["studyid"]
+ datasets = datasets_by_study(studyid)
+ return render_template(
+ "select_dataset.html", **{**form, "studyid": studyid},
+ datasets=datasets, avgmethods=averaging_methods(),
+ datascales=dataset_datascales())
+ except AssertionError as aserr:
+ return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/create-dataset", methods=["POST"])
+def create_dataset():
+ "Select the dataset to add the file contents against"
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+ assert form.get("studyid"), "study"
+ assert form.get("avgid"), "averaging method"
+ assert form.get("datasetname2"), "Dataset Name 2"
+ assert form.get("datasetfullname"), "Dataset Full Name"
+ assert form.get("datasetshortname"), "Dataset Short Name"
+ assert form.get("datasetpublic"), "Dataset public specification"
+ assert form.get("datasetconfidentiality"), "Dataset confidentiality"
+ assert form.get("datasetdatascale"), "Dataset Datascale"
+
+ with database_connection() as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ datasetname = form["datasetname"]
+ cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Name=%s",
+ (datasetname,))
+ results = cursor.fetchall()
+ if bool(results):
+ flash("A dataset with that name already exists.",
+ "alert-error")
+ return redirect(url_for("dbinsert.select_dataset"), code=307)
+ values = (
+ form["studyid"], form["avgid"],
+ datasetname, form["datasetname2"],
+ form["datasetfullname"], form["datasetshortname"],
+ datetime.now().date().strftime("%Y-%m-%d"),
+ form["datasetpublic"], form["datasetconfidentiality"],
+ "williamslab", form["datasetdatascale"])
+ query = (
+ "INSERT INTO ProbeSetFreeze("
+ "ProbeFreezeId, AvgID, Name, Name2, FullName, "
+ "ShortName, CreateTime, OrderList, public, "
+ "confidentiality, AuthorisedUsers, DataScale) "
+ "VALUES"
+ "(%s, %s, %s, %s, %s, %s, %s, NULL, %s, %s, %s, %s)")
+ cursor.execute(query, values)
+ new_datasetid = cursor.lastrowid
+ return render_template(
+ "continue_from_create_dataset.html",
+ filename=form["filename"], filetype=form["filetype"],
+ species=form["species"], genechipid=form["genechipid"],
+ studyid=form["studyid"], datasetid=new_datasetid,
+ totallines=form["totallines"])
+ except AssertionError as aserr:
+ flash(f"Missing data {aserr.args[0]}", "alert-error")
+ return redirect(url_for("dbinsert.select_dataset"), code=307)
+
+def study_by_id(studyid:int) -> Union[dict, None]:
+ "Get a study by its Id"
+ with database_connection() as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT * FROM ProbeFreeze WHERE Id=%s",
+ (studyid,))
+ return cursor.fetchone()
+
+def dataset_by_id(datasetid:int) -> Union[dict, None]:
+ "Retrieve a dataset by its id"
+ with database_connection() as conn:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ ("SELECT AvgMethod.Name AS AvgMethodName, ProbeSetFreeze.* "
+ "FROM ProbeSetFreeze INNER JOIN AvgMethod "
+ "ON ProbeSetFreeze.AvgId=AvgMethod.AvgMethodId "
+ "WHERE ProbeSetFreeze.Id=%s"),
+ (datasetid,))
+ return cursor.fetchone()
+
+def selected_keys(original: dict, keys: tuple) -> dict:
+ "Return a new dict from the `original` dict with only `keys` present."
+ return {key: value for key,value in original.items() if key in keys}
+
+@dbinsertbp.route("/final-confirmation", methods=["POST"])
+def final_confirmation():
+ "Preview the data before triggering entry into the database"
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+ assert form.get("studyid"), "study"
+ assert form.get("datasetid"), "dataset"
+
+ speciesid = form["species"]
+ genechipid = form["genechipid"]
+ studyid = form["studyid"]
+ datasetid=form["datasetid"]
+ return render_template(
+ "final_confirmation.html", filename=form["filename"],
+ filetype=form["filetype"], totallines=form["totallines"],
+ species=speciesid, genechipid=genechipid, studyid=studyid,
+ datasetid=datasetid, the_species=selected_keys(
+ with_db_connection(lambda conn: species_by_id(conn, speciesid)),
+ ("SpeciesName", "Name", "MenuName")),
+ platform=selected_keys(
+ platform_by_id(genechipid),
+ ("GeneChipName", "Name", "GeoPlatform", "Title", "GO_tree_value")),
+ study=selected_keys(
+ study_by_id(studyid), ("Name", "FullName", "ShortName")),
+ dataset=selected_keys(
+ dataset_by_id(datasetid),
+ ("AvgMethodName", "Name", "Name2", "FullName", "ShortName",
+ "DataScale")))
+ except AssertionError as aserr:
+ return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/insert-data", methods=["POST"])
+def insert_data():
+ "Trigger data insertion"
+ form = request.form
+ try:
+ assert form.get("filename"), "filename"
+ assert form.get("filetype"), "filetype"
+ assert form.get("species"), "species"
+ assert form.get("genechipid"), "platform"
+ assert form.get("studyid"), "study"
+ assert form.get("datasetid"), "dataset"
+
+ filename = form["filename"]
+ filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}"
+ redisurl = app.config["REDIS_URL"]
+ if os.path.exists(filepath):
+ with Redis.from_url(redisurl, decode_responses=True) as rconn:
+ job = jobs.launch_job(
+ jobs.data_insertion_job(
+ rconn, filepath, form["filetype"], form["totallines"],
+ form["species"], form["genechipid"], form["datasetid"],
+ app.config["SQL_URI"], redisurl,
+ app.config["JOBS_TTL_SECONDS"]),
+ redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+ return redirect(url_for("dbinsert.insert_status", job_id=job["jobid"]))
+ return render_error(f"File '{filename}' no longer exists.")
+ except AssertionError as aserr:
+ return render_error(f"Missing data: {aserr.args[0]}")
+
+@dbinsertbp.route("/status/<job_id>", methods=["GET"])
+def insert_status(job_id: str):
+ "Retrieve status of data insertion."
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+ if job:
+ job_status = job["status"]
+ if job_status == "success":
+ return render_template("insert_success.html", job=job)
+ if job["status"] == "error":
+ return render_template("insert_error.html", job=job)
+ return render_template("insert_progress.html", job=job)
+ return render_template("no_such_job.html", job_id=job_id), 400
diff --git a/uploader/default_settings.py b/uploader/default_settings.py
new file mode 100644
index 0000000..7a9da0f
--- /dev/null
+++ b/uploader/default_settings.py
@@ -0,0 +1,14 @@
+"""
+The default configuration file. The values here should be overridden in the
+actual configuration file used for the production and staging systems.
+"""
+
+import os
+
+LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING")
+SECRET_KEY = b"<Please! Please! Please! Change This!>"
+UPLOAD_FOLDER = "/tmp/qc_app_files"
+REDIS_URL = "redis://"
+JOBS_TTL_SECONDS = 1209600 # 14 days
+GNQC_REDIS_PREFIX="GNQC"
+SQL_URI = ""
diff --git a/uploader/entry.py b/uploader/entry.py
new file mode 100644
index 0000000..4a02f1e
--- /dev/null
+++ b/uploader/entry.py
@@ -0,0 +1,127 @@
+"""Entry-point module"""
+import os
+import mimetypes
+from typing import Tuple
+from zipfile import ZipFile, is_zipfile
+
+from werkzeug.utils import secure_filename
+from flask import (
+ flash,
+ request,
+ url_for,
+ redirect,
+ Blueprint,
+ render_template,
+ current_app as app,
+ send_from_directory)
+
+from uploader.db import species
+from uploader.db_utils import with_db_connection
+
+entrybp = Blueprint("entry", __name__)
+
+@entrybp.route("/favicon.ico", methods=["GET"])
+def favicon():
+ """Return the favicon."""
+ return send_from_directory(os.path.join(app.root_path, "static"),
+ "images/CITGLogo.png",
+ mimetype="image/png")
+
+
+def errors(rqst) -> Tuple[str, ...]:
+ """Return a tuple of the errors found in the request `rqst`. If no error is
+ found, then an empty tuple is returned."""
+ def __filetype_error__():
+ return (
+ ("Invalid file type provided.",)
+ if rqst.form.get("filetype") not in ("average", "standard-error")
+ else tuple())
+
+ def __file_missing_error__():
+ return (
+ ("No file was uploaded.",)
+ if ("qc_text_file" not in rqst.files or
+ rqst.files["qc_text_file"].filename == "")
+ else tuple())
+
+ def __file_mimetype_error__():
+ text_file = rqst.files["qc_text_file"]
+ return (
+ (
+ ("Invalid file! Expected a tab-separated-values file, or a zip "
+ "file of the a tab-separated-values file."),)
+ if text_file.mimetype not in (
+ "text/plain", "text/tab-separated-values",
+ "application/zip")
+ else tuple())
+
+ return (
+ __filetype_error__() +
+ (__file_missing_error__() or __file_mimetype_error__()))
+
+def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]:
+ """Check the uploaded zip file for errors."""
+ zfile_errors: Tuple[str, ...] = tuple()
+ if is_zipfile(filepath):
+ with ZipFile(filepath, "r") as zfile:
+ infolist = zfile.infolist()
+ if len(infolist) != 1:
+ zfile_errors = zfile_errors + (
+ ("Expected exactly one (1) member file within the uploaded zip "
+ f"file. Got {len(infolist)} member files."),)
+ if len(infolist) == 1 and infolist[0].is_dir():
+ zfile_errors = zfile_errors + (
+ ("Expected a member text file in the uploaded zip file. Got a "
+ "directory/folder."),)
+
+ if len(infolist) == 1 and not infolist[0].is_dir():
+ zfile.extract(infolist[0], path=upload_dir)
+ mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}")
+ if mime[0] != "text/tab-separated-values":
+ zfile_errors = zfile_errors + (
+ ("Expected the member text file in the uploaded zip file to"
+ " be a tab-separated file."),)
+
+ return zfile_errors
+
+@entrybp.route("/", methods=["GET"])
+def index():
+ """Load the landing page"""
+ return render_template("index.html")
+
+@entrybp.route("/upload", methods=["GET", "POST"])
+def upload_file():
+ """Enables uploading the files"""
+ if request.method == "GET":
+ return render_template(
+ "select_species.html", species=with_db_connection(species))
+
+ upload_dir = app.config["UPLOAD_FOLDER"]
+ request_errors = errors(request)
+ if request_errors:
+ for error in request_errors:
+ flash(error, "alert-danger error-expr-data")
+ return redirect(url_for("entry.upload_file"))
+
+ filename = secure_filename(request.files["qc_text_file"].filename)
+ if not os.path.exists(upload_dir):
+ os.mkdir(upload_dir)
+
+ filepath = os.path.join(upload_dir, filename)
+ request.files["qc_text_file"].save(os.path.join(upload_dir, filename))
+
+ zip_errors = zip_file_errors(filepath, upload_dir)
+ if zip_errors:
+ for error in zip_errors:
+ flash(error, "alert-danger error-expr-data")
+ return redirect(url_for("entry.upload_file"))
+
+ return redirect(url_for("parse.parse",
+ speciesid=request.form["speciesid"],
+ filename=filename,
+ filetype=request.form["filetype"]))
+
+@entrybp.route("/data-review", methods=["GET"])
+def data_review():
+ """Provide some help on data expectations to the user."""
+ return render_template("data_review.html")
diff --git a/uploader/errors.py b/uploader/errors.py
new file mode 100644
index 0000000..3e7c893
--- /dev/null
+++ b/uploader/errors.py
@@ -0,0 +1,29 @@
+"""Application error handling."""
+import traceback
+from werkzeug.exceptions import HTTPException
+
+import MySQLdb as mdb
+from flask import Flask, request, render_template, current_app as app
+
+def handle_general_exception(exc: Exception):
+ """Handle generic exceptions."""
+ trace = traceback.format_exc()
+ app.logger.error(
+ "Error (%s.%s): Generic unhandled exception!! (URI: %s)\n%s",
+ exc.__class__.__module__, exc.__class__.__name__, request.url, trace)
+ return render_template("unhandled_exception.html", trace=trace), 500
+
+def handle_http_exception(exc: HTTPException):
+ """Handle HTTP exceptions."""
+ app.logger.error(
+ "HTTP Error %s: %s", exc.code, exc.description, exc_info=True)
+ return render_template("http-error.html",
+ request_url=request.url,
+ exc=exc,
+ trace=traceback.format_exception(exc)), exc.code
+
+def register_error_handlers(appl: Flask):
+ """Register top-level error/exception handlers."""
+ appl.register_error_handler(Exception, handle_general_exception)
+ appl.register_error_handler(HTTPException, handle_http_exception)
+ appl.register_error_handler(mdb.MySQLError, handle_general_exception)
diff --git a/uploader/files.py b/uploader/files.py
new file mode 100644
index 0000000..b163612
--- /dev/null
+++ b/uploader/files.py
@@ -0,0 +1,26 @@
+"""Utilities to deal with uploaded files."""
+import hashlib
+from pathlib import Path
+from datetime import datetime
+from flask import current_app
+
+from werkzeug.utils import secure_filename
+from werkzeug.datastructures import FileStorage
+
+def save_file(fileobj: FileStorage, upload_dir: Path) -> Path:
+ """Save the uploaded file and return the path."""
+ assert bool(fileobj), "Invalid file object!"
+ hashed_name = hashlib.sha512(
+ f"{fileobj.filename}::{datetime.now().isoformat()}".encode("utf8")
+ ).hexdigest()
+ filename = Path(secure_filename(hashed_name)) # type: ignore[arg-type]
+ if not upload_dir.exists():
+ upload_dir.mkdir()
+
+ filepath = Path(upload_dir, filename)
+ fileobj.save(filepath)
+ return filepath
+
+def fullpath(filename: str):
+ """Get a file's full path. This makes use of `flask.current_app`."""
+ return Path(current_app.config["UPLOAD_FOLDER"], filename).absolute()
diff --git a/uploader/input_validation.py b/uploader/input_validation.py
new file mode 100644
index 0000000..9abe742
--- /dev/null
+++ b/uploader/input_validation.py
@@ -0,0 +1,27 @@
+"""Input validation utilities"""
+from typing import Any
+
+def is_empty_string(value: str) -> bool:
+ """Check whether as string is empty"""
+ return (isinstance(value, str) and value.strip() == "")
+
+def is_empty_input(value: Any) -> bool:
+ """Check whether user provided an empty value."""
+ return (value is None or is_empty_string(value))
+
+def is_integer_input(value: Any) -> bool:
+ """
+ Check whether user provided a value that can be parsed into an integer.
+ """
+ def __is_int__(val, base):
+ try:
+ int(val, base=base)
+ except ValueError:
+ return False
+ return True
+ return isinstance(value, int) or (
+ (not is_empty_input(value)) and (
+ isinstance(value, str) and (
+ __is_int__(value, 10)
+ or __is_int__(value, 8)
+ or __is_int__(value, 16))))
diff --git a/uploader/jobs.py b/uploader/jobs.py
new file mode 100644
index 0000000..21889da
--- /dev/null
+++ b/uploader/jobs.py
@@ -0,0 +1,130 @@
+"""Handle jobs"""
+import os
+import sys
+import shlex
+import subprocess
+from uuid import UUID, uuid4
+from datetime import timedelta
+from typing import Union, Optional
+
+from redis import Redis
+from flask import current_app as app
+
+JOBS_PREFIX = "JOBS"
+
+class JobNotFound(Exception):
+ """Raised if we try to retrieve a non-existent job."""
+
+def jobsnamespace():
+ """
+ Return the jobs namespace prefix. It depends on app configuration.
+
+ Calling this function outside of an application context will cause an
+ exception to be raised. It is mostly a convenience utility to use within the
+ application.
+ """
+ return f"{app.config['GNQC_REDIS_PREFIX']}:{JOBS_PREFIX}"
+
+def job_key(namespaceprefix: str, jobid: Union[str, UUID]) -> str:
+ """Build the key by appending it to the namespace prefix."""
+ return f"{namespaceprefix}:{jobid}"
+
+def raise_jobnotfound(rprefix:str, jobid: Union[str,UUID]):
+ """Utility to raise a `NoSuchJobError`"""
+ raise JobNotFound(f"Could not retrieve job '{jobid}' from '{rprefix}.")
+
+def error_filename(jobid, error_dir):
+ "Compute the path of the file where errors will be dumped."
+ return f"{error_dir}/job_{jobid}.error"
+
+def initialise_job(# pylint: disable=[too-many-arguments]
+ rconn: Redis, rprefix: str, jobid: str, command: list, job_type: str,
+ ttl_seconds: int = 86400, extra_meta: Optional[dict] = None) -> dict:
+ "Initialise a job 'object' and put in on redis"
+ the_job = {
+ "jobid": jobid, "command": shlex.join(command), "status": "pending",
+ "percent": 0, "job-type": job_type, **(extra_meta or {})
+ }
+ rconn.hset(job_key(rprefix, jobid), mapping=the_job)
+ rconn.expire(
+ name=job_key(rprefix, jobid), time=timedelta(seconds=ttl_seconds))
+ return the_job
+
+def build_file_verification_job(#pylint: disable=[too-many-arguments]
+ redis_conn: Redis,
+ dburi: str,
+ redisuri: str,
+ speciesid: int,
+ filepath: str,
+ filetype: str,
+ ttl_seconds: int):
+ "Build a file verification job"
+ jobid = str(uuid4())
+ command = [
+ sys.executable, "-m", "scripts.validate_file",
+ dburi, redisuri, jobsnamespace(), jobid,
+ "--redisexpiry", str(ttl_seconds),
+ str(speciesid), filetype, filepath,
+ ]
+ return initialise_job(
+ redis_conn, jobsnamespace(), jobid, command, "file-verification",
+ ttl_seconds, {
+ "filetype": filetype,
+ "filename": os.path.basename(filepath), "percent": 0
+ })
+
+def data_insertion_job(# pylint: disable=[too-many-arguments]
+ redis_conn: Redis, filepath: str, filetype: str, totallines: int,
+ speciesid: int, platformid: int, datasetid: int, databaseuri: str,
+ redisuri: str, ttl_seconds: int) -> dict:
+ "Build a data insertion job"
+ jobid = str(uuid4())
+ command = [
+ sys.executable, "-m", "scripts.insert_data", filetype, filepath,
+ speciesid, platformid, datasetid, databaseuri, redisuri
+ ]
+ return initialise_job(
+ redis_conn, jobsnamespace(), jobid, command, "data-insertion",
+ ttl_seconds, {
+ "filename": os.path.basename(filepath), "filetype": filetype,
+ "totallines": totallines
+ })
+
+def launch_job(the_job: dict, redisurl: str, error_dir):
+ """Launch a job in the background"""
+ if not os.path.exists(error_dir):
+ os.mkdir(error_dir)
+
+ jobid = the_job["jobid"]
+ with open(error_filename(jobid, error_dir),
+ "w",
+ encoding="utf-8") as errorfile:
+ subprocess.Popen( # pylint: disable=[consider-using-with]
+ [sys.executable, "-m", "scripts.worker", redisurl, jobsnamespace(),
+ jobid],
+ stderr=errorfile,
+ env={"PYTHONPATH": ":".join(sys.path)})
+
+ return the_job
+
+def job(rconn: Redis, rprefix: str, jobid: Union[str,UUID]):
+ "Retrieve the job"
+ thejob = (rconn.hgetall(job_key(rprefix, jobid)) or
+ raise_jobnotfound(rprefix, jobid))
+ return thejob
+
+def update_status(
+ rconn: Redis, rprefix: str, jobid: Union[str, UUID], status: str):
+ """Update status of job in redis."""
+ rconn.hset(name=job_key(rprefix, jobid), key="status", value=status)
+
+def update_stdout_stderr(rconn: Redis,
+ rprefix: str,
+ jobid: Union[str, UUID],
+ bytes_read: bytes,
+ stream: str):
+ "Update the stdout/stderr keys according to the value of `stream`."
+ thejob = job(rconn, rprefix, jobid)
+ contents = thejob.get(stream, '')
+ new_contents = contents + bytes_read.decode("utf-8")
+ rconn.hset(name=job_key(rprefix, jobid), key=stream, value=new_contents)
diff --git a/uploader/parse.py b/uploader/parse.py
new file mode 100644
index 0000000..865dae2
--- /dev/null
+++ b/uploader/parse.py
@@ -0,0 +1,175 @@
+"""File parsing module"""
+import os
+
+import jsonpickle
+from redis import Redis
+from flask import flash, request, url_for, redirect, Blueprint, render_template
+from flask import current_app as app
+
+from quality_control.errors import InvalidValue, DuplicateHeading
+
+from uploader import jobs
+from uploader.dbinsert import species_by_id
+from uploader.db_utils import with_db_connection
+
+parsebp = Blueprint("parse", __name__)
+
+def isinvalidvalue(item):
+ """Check whether item is of type InvalidValue"""
+ return isinstance(item, InvalidValue)
+
+def isduplicateheading(item):
+ """Check whether item is of type DuplicateHeading"""
+ return isinstance(item, DuplicateHeading)
+
+@parsebp.route("/parse", methods=["GET"])
+def parse():
+ """Trigger file parsing"""
+ errors = False
+ speciesid = request.args.get("speciesid")
+ filename = request.args.get("filename")
+ filetype = request.args.get("filetype")
+ if speciesid is None:
+ flash("No species selected", "alert-error error-expr-data")
+ errors = True
+ else:
+ try:
+ speciesid = int(speciesid)
+ species = with_db_connection(
+ lambda con: species_by_id(con, speciesid))
+ if not bool(species):
+ flash("No such species.", "alert-error error-expr-data")
+ errors = True
+ except ValueError:
+ flash("Invalid speciesid provided. Expected an integer.",
+ "alert-error error-expr-data")
+ errors = True
+
+ if filename is None:
+ flash("No file provided", "alert-error error-expr-data")
+ errors = True
+
+ if filetype is None:
+ flash("No filetype provided", "alert-error error-expr-data")
+ errors = True
+
+ if filetype not in ("average", "standard-error"):
+ flash("Invalid filetype provided", "alert-error error-expr-data")
+ errors = True
+
+ if filename:
+ filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename)
+ if not os.path.exists(filepath):
+ flash("Selected file does not exist (any longer)",
+ "alert-error error-expr-data")
+ errors = True
+
+ if errors:
+ return redirect(url_for("entry.upload_file"))
+
+ redisurl = app.config["REDIS_URL"]
+ with Redis.from_url(redisurl, decode_responses=True) as rconn:
+ job = jobs.launch_job(
+ jobs.build_file_verification_job(
+ rconn, app.config["SQL_URI"], redisurl,
+ speciesid, filepath, filetype,
+ app.config["JOBS_TTL_SECONDS"]),
+ redisurl,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+ return redirect(url_for("parse.parse_status", job_id=job["jobid"]))
+
+@parsebp.route("/status/<job_id>", methods=["GET"])
+def parse_status(job_id: str):
+ "Retrieve the status of the job"
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ try:
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+ except jobs.JobNotFound as _exc:
+ return render_template("no_such_job.html", job_id=job_id), 400
+
+ error_filename = jobs.error_filename(
+ job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+ if os.path.exists(error_filename):
+ stat = os.stat(error_filename)
+ if stat.st_size > 0:
+ return redirect(url_for("parse.fail", job_id=job_id))
+
+ job_id = job["jobid"]
+ progress = float(job["percent"])
+ status = job["status"]
+ filename = job.get("filename", "uploaded file")
+ errors = jsonpickle.decode(
+ job.get("errors", jsonpickle.encode(tuple())))
+ if status in ("success", "aborted"):
+ return redirect(url_for("parse.results", job_id=job_id))
+
+ if status == "parse-error":
+ return redirect(url_for("parse.fail", job_id=job_id))
+
+ app.jinja_env.globals.update(
+ isinvalidvalue=isinvalidvalue,
+ isduplicateheading=isduplicateheading)
+ return render_template(
+ "job_progress.html",
+ job_id = job_id,
+ job_status = status,
+ progress = progress,
+ message = job.get("message", ""),
+ job_name = f"Parsing '{filename}'",
+ errors=errors)
+
+@parsebp.route("/results/<job_id>", methods=["GET"])
+def results(job_id: str):
+ """Show results of parsing..."""
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+ if job:
+ filename = job["filename"]
+ errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple())))
+ app.jinja_env.globals.update(
+ isinvalidvalue=isinvalidvalue,
+ isduplicateheading=isduplicateheading)
+ return render_template(
+ "parse_results.html",
+ errors=errors,
+ job_name = f"Parsing '{filename}'",
+ user_aborted = job.get("user_aborted"),
+ job_id=job["jobid"])
+
+ return render_template("no_such_job.html", job_id=job_id)
+
+@parsebp.route("/fail/<job_id>", methods=["GET"])
+def fail(job_id: str):
+ """Handle parsing failure"""
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+ if job:
+ error_filename = jobs.error_filename(
+ job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+ if os.path.exists(error_filename):
+ stat = os.stat(error_filename)
+ if stat.st_size > 0:
+ return render_template(
+ "worker_failure.html", job_id=job_id)
+
+ return render_template("parse_failure.html", job=job)
+
+ return render_template("no_such_job.html", job_id=job_id)
+
+@parsebp.route("/abort", methods=["POST"])
+def abort():
+ """Handle user request to abort file processing"""
+ job_id = request.form["job_id"]
+
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+ if job:
+ rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id),
+ key="user_aborted",
+ value=int(True))
+
+ return redirect(url_for("parse.parse_status", job_id=job_id))
diff --git a/uploader/samples.py b/uploader/samples.py
new file mode 100644
index 0000000..9c95770
--- /dev/null
+++ b/uploader/samples.py
@@ -0,0 +1,354 @@
+"""Code regarding samples"""
+import os
+import sys
+import csv
+import uuid
+from pathlib import Path
+from typing import Iterator
+
+import MySQLdb as mdb
+from redis import Redis
+from MySQLdb.cursors import DictCursor
+from flask import (
+ flash,
+ request,
+ url_for,
+ redirect,
+ Blueprint,
+ render_template,
+ current_app as app)
+
+from functional_tools import take
+
+from uploader import jobs
+from uploader.files import save_file
+from uploader.input_validation import is_integer_input
+from uploader.db_utils import (
+ with_db_connection,
+ database_connection,
+ with_redis_connection)
+from uploader.db import (
+ species_by_id,
+ save_population,
+ population_by_id,
+ populations_by_species,
+ species as fetch_species)
+
+samples = Blueprint("samples", __name__)
+
+@samples.route("/upload/species", methods=["GET", "POST"])
+def select_species():
+ """Select the species."""
+ if request.method == "GET":
+ return render_template("samples/select-species.html",
+ species=with_db_connection(fetch_species))
+
+ index_page = redirect(url_for("entry.upload_file"))
+ species_id = request.form.get("species_id")
+ if bool(species_id):
+ species_id = int(species_id)
+ species = with_db_connection(
+ lambda conn: species_by_id(conn, species_id))
+ if bool(species):
+ return redirect(url_for(
+ "samples.select_population", species_id=species_id))
+ flash("Invalid species selected!", "alert-error")
+ flash("You need to select a species", "alert-error")
+ return index_page
+
+@samples.route("/upload/species/<int:species_id>/create-population",
+ methods=["POST"])
+def create_population(species_id: int):
+ """Create new grouping/population."""
+ if not is_integer_input(species_id):
+ flash("You did not provide a valid species. Please select one to "
+ "continue.",
+ "alert-danger")
+ return redirect(url_for("samples.select_species"))
+ species = with_db_connection(lambda conn: species_by_id(conn, species_id))
+ if not bool(species):
+ flash("Species with given ID was not found.", "alert-danger")
+ return redirect(url_for("samples.select_species"))
+
+ species_page = redirect(url_for("samples.select_species"), code=307)
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ pop_name = request.form.get("inbredset_name", "").strip()
+ pop_fullname = request.form.get("inbredset_fullname", "").strip()
+
+ if not bool(species):
+ flash("Invalid species!", "alert-error error-create-population")
+ return species_page
+ if (not bool(pop_name)) or (not bool(pop_fullname)):
+ flash("You *MUST* provide a grouping/population name",
+ "alert-error error-create-population")
+ return species_page
+
+ pop = save_population(conn, {
+ "SpeciesId": species["SpeciesId"],
+ "Name": pop_name,
+ "InbredSetName": pop_fullname,
+ "FullName": pop_fullname,
+ "Family": request.form.get("inbredset_family") or None,
+ "Description": request.form.get("description") or None
+ })
+
+ flash("Grouping/Population created successfully.", "alert-success")
+ return redirect(url_for("samples.upload_samples",
+ species_id=species_id,
+ population_id=pop["population_id"]))
+
+@samples.route("/upload/species/<int:species_id>/population",
+ methods=["GET", "POST"])
+def select_population(species_id: int):
+ """Select from existing groupings/populations."""
+ if not is_integer_input(species_id):
+ flash("You did not provide a valid species. Please select one to "
+ "continue.",
+ "alert-danger")
+ return redirect(url_for("samples.select_species"))
+ species = with_db_connection(lambda conn: species_by_id(conn, species_id))
+ if not bool(species):
+ flash("Species with given ID was not found.", "alert-danger")
+ return redirect(url_for("samples.select_species"))
+
+ if request.method == "GET":
+ return render_template(
+ "samples/select-population.html",
+ species=species,
+ populations=with_db_connection(
+ lambda conn: populations_by_species(conn, species_id)))
+
+ population_page = redirect(url_for(
+ "samples.select_population", species_id=species_id), code=307)
+ _population_id = request.form.get("inbredset_id")
+ if not is_integer_input(_population_id):
+ flash("You did not provide a valid population. Please select one to "
+ "continue.",
+ "alert-danger")
+ return population_page
+ population = with_db_connection(
+ lambda conn: population_by_id(conn, _population_id))
+ if not bool(population):
+ flash("Invalid grouping/population!",
+ "alert-error error-select-population")
+ return population_page
+
+ return redirect(url_for("samples.upload_samples",
+ species_id=species_id,
+ population_id=_population_id),
+ code=307)
+
+def read_samples_file(filepath, separator: str, firstlineheading: bool, **kwargs) -> Iterator[dict]:
+ """Read the samples file."""
+ with open(filepath, "r", encoding="utf-8") as inputfile:
+ reader = csv.DictReader(
+ inputfile,
+ fieldnames=(
+ None if firstlineheading
+ else ("Name", "Name2", "Symbol", "Alias")),
+ delimiter=separator,
+ quotechar=kwargs.get("quotechar", '"'))
+ for row in reader:
+ yield row
+
+def save_samples_data(conn: mdb.Connection,
+ speciesid: int,
+ file_data: Iterator[dict]):
+ """Save the samples to DB."""
+ data = ({**row, "SpeciesId": speciesid} for row in file_data)
+ total = 0
+ with conn.cursor() as cursor:
+ while True:
+ batch = take(data, 5000)
+ if len(batch) == 0:
+ break
+ cursor.executemany(
+ "INSERT INTO Strain(Name, Name2, SpeciesId, Symbol, Alias) "
+ "VALUES("
+ " %(Name)s, %(Name2)s, %(SpeciesId)s, %(Symbol)s, %(Alias)s"
+ ") ON DUPLICATE KEY UPDATE Name=Name",
+ batch)
+ total += len(batch)
+ print(f"\tSaved {total} samples total so far.")
+
+def cross_reference_samples(conn: mdb.Connection,
+ species_id: int,
+ population_id: int,
+ strain_names: Iterator[str]):
+ """Link samples to their population."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT MAX(OrderId) AS loid FROM StrainXRef WHERE InbredSetId=%s",
+ (population_id,))
+ last_order_id = (cursor.fetchone()["loid"] or 10)
+ total = 0
+ while True:
+ batch = take(strain_names, 5000)
+ if len(batch) == 0:
+ break
+ params_str = ", ".join(["%s"] * len(batch))
+ ## This query is slow -- investigate.
+ cursor.execute(
+ "SELECT s.Id FROM Strain AS s LEFT JOIN StrainXRef AS sx "
+ "ON s.Id = sx.StrainId WHERE s.SpeciesId=%s AND s.Name IN "
+ f"({params_str}) AND sx.StrainId IS NULL",
+ (species_id,) + tuple(batch))
+ strain_ids = (sid["Id"] for sid in cursor.fetchall())
+ params = tuple({
+ "pop_id": population_id,
+ "strain_id": strain_id,
+ "order_id": last_order_id + (order_id * 10),
+ "mapping": "N",
+ "pedigree": None
+ } for order_id, strain_id in enumerate(strain_ids, start=1))
+ cursor.executemany(
+ "INSERT INTO StrainXRef( "
+ " InbredSetId, StrainId, OrderId, Used_for_mapping, PedigreeStatus"
+ ")"
+ "VALUES ("
+ " %(pop_id)s, %(strain_id)s, %(order_id)s, %(mapping)s, "
+ " %(pedigree)s"
+ ")",
+ params)
+ last_order_id += (len(params) * 10)
+ total += len(batch)
+ print(f"\t{total} total samples cross-referenced to the population "
+ "so far.")
+
+def build_sample_upload_job(# pylint: disable=[too-many-arguments]
+ speciesid: int,
+ populationid: int,
+ samplesfile: Path,
+ separator: str,
+ firstlineheading: bool,
+ quotechar: str):
+ """Define the async command to run the actual samples data upload."""
+ return [
+ sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"],
+ str(speciesid), str(populationid), str(samplesfile.absolute()),
+ separator, f"--redisuri={app.config['REDIS_URL']}",
+ f"--quotechar={quotechar}"
+ ] + (["--firstlineheading"] if firstlineheading else [])
+
+@samples.route("/upload/species/<int:species_id>/populations/<int:population_id>/samples",
+ methods=["GET", "POST"])
+def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements]
+ """Upload the samples."""
+ samples_uploads_page = redirect(url_for("samples.upload_samples",
+ species_id=species_id,
+ population_id=population_id))
+ if not is_integer_input(species_id):
+ flash("You did not provide a valid species. Please select one to "
+ "continue.",
+ "alert-danger")
+ return redirect(url_for("samples.select_species"))
+ species = with_db_connection(lambda conn: species_by_id(conn, species_id))
+ if not bool(species):
+ flash("Species with given ID was not found.", "alert-danger")
+ return redirect(url_for("samples.select_species"))
+
+ if not is_integer_input(population_id):
+ flash("You did not provide a valid population. Please select one "
+ "to continue.",
+ "alert-danger")
+ return redirect(url_for("samples.select_population",
+ species_id=species_id),
+ code=307)
+ population = with_db_connection(
+ lambda conn: population_by_id(conn, int(population_id)))
+ if not bool(population):
+ flash("Invalid grouping/population!", "alert-error")
+ return redirect(url_for("samples.select_population",
+ species_id=species_id),
+ code=307)
+
+ if request.method == "GET" or request.files.get("samples_file") is None:
+ return render_template("samples/upload-samples.html",
+ species=species,
+ population=population)
+
+ try:
+ samples_file = save_file(request.files["samples_file"],
+ Path(app.config["UPLOAD_FOLDER"]))
+ except AssertionError:
+ flash("You need to provide a file with the samples data.",
+ "alert-error")
+ return samples_uploads_page
+
+ firstlineheading = (request.form.get("first_line_heading") == "on")
+
+ separator = request.form.get("separator", ",")
+ if separator == "other":
+ separator = request.form.get("other_separator", ",")
+ if not bool(separator):
+ flash("You need to provide a separator character.", "alert-error")
+ return samples_uploads_page
+
+ quotechar = (request.form.get("field_delimiter", '"') or '"')
+
+ redisuri = app.config["REDIS_URL"]
+ with Redis.from_url(redisuri, decode_responses=True) as rconn:
+ the_job = jobs.launch_job(
+ jobs.initialise_job(
+ rconn,
+ jobs.jobsnamespace(),
+ str(uuid.uuid4()),
+ build_sample_upload_job(
+ species["SpeciesId"],
+ population["InbredSetId"],
+ samples_file,
+ separator,
+ firstlineheading,
+ quotechar),
+ "samples_upload",
+ app.config["JOBS_TTL_SECONDS"],
+ {"job_name": f"Samples Upload: {samples_file.name}"}),
+ redisuri,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors")
+ return redirect(url_for(
+ "samples.upload_status", job_id=the_job["jobid"]))
+
+@samples.route("/upload/status/<uuid:job_id>", methods=["GET"])
+def upload_status(job_id: uuid.UUID):
+ """Check on the status of a samples upload job."""
+ job = with_redis_connection(lambda rconn: jobs.job(
+ rconn, jobs.jobsnamespace(), job_id))
+ if job:
+ status = job["status"]
+ if status == "success":
+ return render_template("samples/upload-success.html", job=job)
+
+ if status == "error":
+ return redirect(url_for("samples.upload_failure", job_id=job_id))
+
+ error_filename = Path(jobs.error_filename(
+ job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors"))
+ if error_filename.exists():
+ stat = os.stat(error_filename)
+ if stat.st_size > 0:
+ return redirect(url_for(
+ "samples.upload_failure", job_id=job_id))
+
+ return render_template(
+ "samples/upload-progress.html",
+ job=job) # maybe also handle this?
+
+ return render_template("no_such_job.html", job_id=job_id), 400
+
+@samples.route("/upload/failure/<uuid:job_id>", methods=["GET"])
+def upload_failure(job_id: uuid.UUID):
+ """Display the errors of the samples upload failure."""
+ job = with_redis_connection(lambda rconn: jobs.job(
+ rconn, jobs.jobsnamespace(), job_id))
+ if not bool(job):
+ return render_template("no_such_job.html", job_id=job_id), 400
+
+ error_filename = Path(jobs.error_filename(
+ job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors"))
+ if error_filename.exists():
+ stat = os.stat(error_filename)
+ if stat.st_size > 0:
+ return render_template("worker_failure.html", job_id=job_id)
+
+ return render_template("samples/upload-failure.html", job=job)
diff --git a/uploader/static/css/custom-bootstrap.css b/uploader/static/css/custom-bootstrap.css
new file mode 100644
index 0000000..67f1199
--- /dev/null
+++ b/uploader/static/css/custom-bootstrap.css
@@ -0,0 +1,23 @@
+/** Customize some bootstrap selectors **/
+.btn {
+ text-transform: capitalize;
+}
+
+.navbar-inverse {
+ background-color: #336699;
+ border-color: #080808;
+ color: #FFFFFF;
+ background-image: none;
+}
+
+.navbar-inverse .navbar-nav>li>a {
+ color: #FFFFFF;
+}
+
+.navbar-nav > li > a {
+ padding: 5px;
+}
+
+.navbar {
+ min-height: 30px;
+}
diff --git a/uploader/static/css/styles.css b/uploader/static/css/styles.css
new file mode 100644
index 0000000..a88c229
--- /dev/null
+++ b/uploader/static/css/styles.css
@@ -0,0 +1,7 @@
+.heading {
+ text-transform: capitalize;
+}
+
+label {
+ text-transform: capitalize;
+}
diff --git a/uploader/static/css/two-column-with-separator.css b/uploader/static/css/two-column-with-separator.css
new file mode 100644
index 0000000..b6efd46
--- /dev/null
+++ b/uploader/static/css/two-column-with-separator.css
@@ -0,0 +1,27 @@
+.two-column-with-separator {
+ display: grid;
+ grid-template-columns: 9fr 1fr 9fr;
+}
+
+.two-col-sep-col1 {
+ grid-column: 1 / 2;
+}
+
+.two-col-sep-separator {
+ grid-column: 2 / 3;
+ text-align: center;
+ color: #FE3535;
+ font-weight: bolder;
+}
+
+.two-col-sep-col2 {
+ grid-column: 3 / 4;
+}
+
+.two-col-sep-col1, .two-col-sep-col2 {
+ border-style: solid;
+ border-color: #FE3535;
+ border-width: 1px;
+ border-radius: 2em;
+ padding: 2em 3em 2em 3em;
+}
diff --git a/uploader/static/images/CITGLogo.png b/uploader/static/images/CITGLogo.png
new file mode 100644
index 0000000..ae99fed
--- /dev/null
+++ b/uploader/static/images/CITGLogo.png
Binary files differ
diff --git a/uploader/static/js/select_platform.js b/uploader/static/js/select_platform.js
new file mode 100644
index 0000000..4fdd865
--- /dev/null
+++ b/uploader/static/js/select_platform.js
@@ -0,0 +1,70 @@
+function radio_column(chip) {
+ col = document.createElement("td");
+ radio = document.createElement("input");
+ radio.setAttribute("type", "radio");
+ radio.setAttribute("name", "genechipid");
+ radio.setAttribute("value", chip["GeneChipId"]);
+ radio.setAttribute("required", "required");
+ col.appendChild(radio);
+ return col;
+}
+
+function setup_genechips(genechip_data) {
+ columns = ["GeneChipId", "GeneChipName"]
+ submit_button = document.querySelector(
+ "#select-platform-form button[type='submit']");
+ elt = document.getElementById(
+ "genechips-table").getElementsByTagName("tbody")[0];
+ remove_children(elt);
+ if((genechip_data === undefined) || genechip_data.length === 0) {
+ row = document.createElement("tr");
+ col = document.createElement("td");
+ col.setAttribute("colspan", "3");
+ text = document.createTextNode("No chips found for selected species");
+ col.appendChild(text);
+ row.appendChild(col);
+ elt.appendChild(row);
+ submit_button.setAttribute("disabled", true);
+ return false;
+ }
+
+ submit_button.removeAttribute("disabled")
+ genechip_data.forEach(chip => {
+ row = document.createElement("tr");
+ row.appendChild(radio_column(chip));
+ columns.forEach(column => {
+ col = document.createElement("td");
+ content = document.createTextNode(chip[column]);
+ col.appendChild(content);
+ row.appendChild(col);
+ });
+ elt.appendChild(row);
+ });
+}
+
+function genechips() {
+ return JSON.parse(
+ document.getElementById("select-platform-form").getAttribute(
+ "data-genechips"));
+}
+
+function update_genechips(event) {
+ genec = genechips();
+
+ species_elt = document.getElementById("species");
+
+ if(event.target == species_elt) {
+ setup_genechips(genec[species_elt.value.toLowerCase()]);
+ }
+}
+
+function select_row_radio(row) {
+ radio = row.getElementsByTagName(
+ "td")[0].getElementsByTagName(
+ "input")[0];
+ if(radio === undefined) {
+ return false;
+ }
+ radio.setAttribute("checked", "checked");
+ return true;
+}
diff --git a/uploader/static/js/upload_progress.js b/uploader/static/js/upload_progress.js
new file mode 100644
index 0000000..9638b36
--- /dev/null
+++ b/uploader/static/js/upload_progress.js
@@ -0,0 +1,97 @@
+function make_processing_indicator(elt) {
+ var count = 0;
+ return function() {
+ var message = "Finalising upload and saving file: "
+ if(count > 5) {
+ count = 1;
+ }
+ for(i = 0; i < count; i++) {
+ message = message + ".";
+ }
+ elt.innerHTML = message
+ count = count + 1
+ };
+}
+
+function make_progress_updater(file, indicator_elt) {
+ var progress_bar = indicator_elt.querySelector("#progress-bar");
+ var progress_text = indicator_elt.querySelector("#progress-text");
+ var extra_text = indicator_elt.querySelector("#progress-extra-text");
+ return function(event) {
+ if(event.loaded <= file.size) {
+ var percent = Math.round((event.loaded / file.size) * 100);
+ progress_bar.value = percent
+ progress_text.innerHTML = "Uploading: " + percent + "%";
+ extra_text.setAttribute("class", "hidden")
+ }
+
+ if(event.loaded == event.total) {
+ progress_bar.value = 100;
+ progress_text.innerHTML = "Uploaded: 100%";
+ extra_text.setAttribute("class", null);
+ intv = setInterval(make_processing_indicator(extra_text), 400);
+ setTimeout(function() {clearTimeout(intv);}, 20000);
+ }
+ };
+}
+
+function setup_cancel_upload(request, indicator_elt) {
+ document.getElementById("btn-cancel-upload").addEventListener(
+ "click", function(event) {
+ event.preventDefault();
+ request.abort();
+ });
+}
+
+function setup_request(file, progress_indicator_elt) {
+ var request = new XMLHttpRequest();
+ var updater = make_progress_updater(file, progress_indicator_elt);
+ request.upload.addEventListener("progress", updater);
+ request.onload = function(event) {
+ document.location.assign(request.responseURL);
+ };
+ setup_cancel_upload(request, progress_indicator_elt)
+ return request;
+}
+
+function selected_filetype(radios) {
+ for(idx = 0; idx < radios.length; idx++) {
+ if(radios[idx].checked) {
+ return radios[idx].value;
+ }
+ }
+}
+
+function make_data_uploader(setup_formdata) {
+ return function(event) {
+ event.preventDefault();
+
+ var pindicator = document.getElementById("upload-progress-indicator");
+
+ var form = event.target;
+ var the_file = form.querySelector("input[type='file']").files[0];
+ if(the_file === undefined) {
+ form.querySelector("input[type='file']").parentElement.setAttribute(
+ "class", "invalid-input");
+ error_elt = form.querySelector("#no-file-error");
+ if(error_elt !== undefined) {
+ error_elt.setAttribute("style", "display: block;");
+ }
+ return false;
+ }
+ var formdata = setup_formdata(form);
+
+ document.getElementById("progress-filename").innerHTML = the_file.name;
+ var request = setup_request(the_file, pindicator);
+ request.open(form.getAttribute("method"), form.getAttribute("action"));
+ request.send(formdata);
+ return false;
+ }
+}
+
+
+function setup_upload_handlers(formid, datauploader) {
+ console.info("Setting up the upload handlers.")
+ upload_form = document.getElementById(formid);
+ upload_form.addEventListener("submit", datauploader);
+}
diff --git a/uploader/static/js/upload_samples.js b/uploader/static/js/upload_samples.js
new file mode 100644
index 0000000..aed536f
--- /dev/null
+++ b/uploader/static/js/upload_samples.js
@@ -0,0 +1,132 @@
+/*
+ * Read the file content and set the `data-preview-content` attribute on the
+ * file element
+ */
+function read_first_n_lines(event,
+ fileelement,
+ numlines,
+ firstlineheading = true) {
+ var thefile = fileelement.files[0];
+ var reader = new FileReader();
+ reader.addEventListener("load", (event) => {
+ var filecontent = event.target.result.split(
+ "\n").slice(
+ 0, (numlines + (firstlineheading ? 1 : 0))).map(
+ (line) => {return line.trim("\r");});
+ fileelement.setAttribute(
+ "data-preview-content", JSON.stringify(filecontent));
+ display_preview(event);
+ })
+ reader.readAsText(thefile);
+}
+
+function remove_rows(preview_table) {
+ var table_body = preview_table.getElementsByTagName("tbody")[0];
+ while(table_body.children.length > 0) {
+ table_body.removeChild(table_body.children.item(0));
+ }
+}
+
+/*
+ * Display error row
+ */
+function display_error_row(preview_table, error_message) {
+ remove_rows(preview_table);
+ row = document.createElement("tr");
+ cell = document.createElement("td");
+ cell.setAttribute("colspan", 4);
+ cell.innerHTML = error_message;
+ row.appendChild(cell);
+ preview_table.getElementsByTagName("tbody")[0].appendChild(row);
+}
+
+function strip(str, chars) {
+ var end = str.length;
+ var start = 0
+ for(var j = str.length; j > 0; j--) {
+ if(!chars.includes(str[j - 1])) {
+ break;
+ }
+ end = end - 1;
+ }
+ for(var i = 0; i < end; i++) {
+ if(!chars.includes(str[i])) {
+ break;
+ }
+ start = start + 1;
+ }
+ return str.slice(start, end);
+}
+
+function process_preview_data(preview_data, separator, delimiter) {
+ return preview_data.map((line) => {
+ return line.split(separator).map((field) => {
+ return strip(field, delimiter);
+ });
+ });
+}
+
+function render_preview(preview_table, preview_data) {
+ remove_rows(preview_table);
+ var table_body = preview_table.getElementsByTagName("tbody")[0];
+ preview_data.forEach((line) => {
+ var row = document.createElement("tr");
+ line.forEach((field) => {
+ var cell = document.createElement("td");
+ cell.innerHTML = field;
+ row.appendChild(cell);
+ });
+ table_body.appendChild(row);
+ });
+}
+
+/*
+ * Display a preview of the data, relying on the user's selection.
+ */
+function display_preview(event) {
+ var data_preview_table = document.getElementById("tbl:samples-preview");
+ remove_rows(data_preview_table);
+
+ var separator = document.getElementById("select:separator").value;
+ if(separator === "other") {
+ separator = document.getElementById("txt:separator").value;
+ }
+ if(separator == "") {
+ display_error_row(data_preview_table, "Please provide a separator.");
+ return false;
+ }
+
+ var delimiter = document.getElementById("txt:delimiter").value;
+
+ var firstlineheading = document.getElementById("chk:heading").checked;
+
+ var fileelement = document.getElementById("file:samples");
+ var preview_data = JSON.parse(
+ fileelement.getAttribute("data-preview-content") || "[]");
+ if(preview_data.length == 0) {
+ display_error_row(
+ data_preview_table,
+ "No file data to preview. Check that file is provided.");
+ }
+
+ render_preview(data_preview_table, process_preview_data(
+ preview_data.slice(0 + (firstlineheading ? 1 : 0)),
+ separator,
+ delimiter));
+}
+
+document.getElementById("chk:heading").addEventListener(
+ "change", display_preview);
+document.getElementById("select:separator").addEventListener(
+ "change", display_preview);
+document.getElementById("txt:separator").addEventListener(
+ "keyup", display_preview);
+document.getElementById("txt:delimiter").addEventListener(
+ "keyup", display_preview);
+document.getElementById("file:samples").addEventListener(
+ "change", (event) => {
+ read_first_n_lines(event,
+ document.getElementById("file:samples"),
+ 30,
+ document.getElementById("chk:heading").checked);
+ });
diff --git a/uploader/static/js/utils.js b/uploader/static/js/utils.js
new file mode 100644
index 0000000..045dd47
--- /dev/null
+++ b/uploader/static/js/utils.js
@@ -0,0 +1,10 @@
+function remove_children(element) {
+ Array.from(element.children).forEach(child => {
+ element.removeChild(child);
+ });
+}
+
+function trigger_change_event(element) {
+ evt = new Event("change");
+ element.dispatchEvent(evt);
+}
diff --git a/uploader/templates/base.html b/uploader/templates/base.html
new file mode 100644
index 0000000..eb5e6b7
--- /dev/null
+++ b/uploader/templates/base.html
@@ -0,0 +1,51 @@
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta application-name="GeneNetwork Quality-Control Application" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ {%block extrameta%}{%endblock%}
+
+ <title>GN Uploader: {%block title%}{%endblock%}</title>
+
+ <link rel="stylesheet" type="text/css"
+ href="{{url_for('base.bootstrap',
+ filename='css/bootstrap.min.css')}}" />
+ <link rel="stylesheet" type="text/css"
+ href="{{url_for('base.bootstrap',
+ filename='css/bootstrap-theme.min.css')}}" />
+
+
+ <link rel="shortcut icon" type="image/png" sizes="64x64"
+ href="{{url_for('static', filename='images/CITGLogo.png')}}" />
+
+ <link rel="stylesheet" type="text/css" href="/static/css/custom-bootstrap.css" />
+ <link rel="stylesheet" type="text/css" href="/static/css/styles.css" />
+
+ {%block css%}{%endblock%}
+ </head>
+
+ <body>
+ <div class="navbar navbar-inverse navbar-static-top pull-left"
+ role="navigation"
+ style="width: 100%;min-width: 850px;white-space: nowrap;">
+ <div class="container-fluid" style="width: 100%">
+ <ul class="nav navbar-nav">
+ <li><a href="/" style="font-weight: bold">GN Uploader</a></li>
+ <li>
+ <a href="{{gnuri or 'https://genenetwork.org'}}">GeneNetwork</a>
+ </li>
+ </ul>
+ </div>
+ </div>
+ <div class="container">
+ {%block contents%}{%endblock%}
+ </div>
+
+ <script src="{{url_for('base.jquery',
+ filename='jquery.min.js')}}"></script>
+ <script src="{{url_for('base.bootstrap',
+ filename='js/bootstrap.min.js')}}"></script>
+ {%block javascript%}{%endblock%}
+ </body>
+</html>
diff --git a/uploader/templates/cli-output.html b/uploader/templates/cli-output.html
new file mode 100644
index 0000000..33fb73b
--- /dev/null
+++ b/uploader/templates/cli-output.html
@@ -0,0 +1,8 @@
+{%macro cli_output(job, stream)%}
+
+<h4>{{stream | upper}} Output</h4>
+<div class="cli-output">
+ <pre>{{job.get(stream, "")}}</pre>
+</div>
+
+{%endmacro%}
diff --git a/uploader/templates/continue_from_create_dataset.html b/uploader/templates/continue_from_create_dataset.html
new file mode 100644
index 0000000..03bb49c
--- /dev/null
+++ b/uploader/templates/continue_from_create_dataset.html
@@ -0,0 +1,52 @@
+{%extends "base.html"%}
+{%from "dbupdate_hidden_fields.html" import hidden_fields%}
+
+{%block title%}Create Study{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
+{%endblock%}
+
+{%block contents%}
+<h2 class="heading">{{filename}}: create study</h2>
+
+{%with messages = get_flashed_messages(with_categories=true)%}
+{%if messages:%}
+<ul>
+ {%for category, message in messages:%}
+ <li class="{{category}}">{{message}}</li>
+ {%endfor%}
+</ul>
+{%endif%}
+{%endwith%}
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.final_confirmation')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col1">
+ <legend>continue with new dataset</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, datasetid=datasetid, totallines=totallines)}}
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+<div class="row">
+ <p class="two-col-sep-separator">OR</p>
+</div>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.select_dataset')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col2">
+ <legend>Select from existing dataset</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, datasetid=datasetid, totallines=totallines)}}
+
+ <button type="submit" class="btn btn-primary">go back</button>
+ </form>
+</div>
+{%endblock%}
diff --git a/uploader/templates/continue_from_create_study.html b/uploader/templates/continue_from_create_study.html
new file mode 100644
index 0000000..34e6e5e
--- /dev/null
+++ b/uploader/templates/continue_from_create_study.html
@@ -0,0 +1,52 @@
+{%extends "base.html"%}
+{%from "dbupdate_hidden_fields.html" import hidden_fields%}
+
+{%block title%}Create Study{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
+{%endblock%}
+
+{%block contents%}
+<h2 class="heading">{{filename}}: create study</h2>
+
+{%with messages = get_flashed_messages(with_categories=true)%}
+{%if messages:%}
+<ul>
+ {%for category, message in messages:%}
+ <li class="{{category}}">{{message}}</li>
+ {%endfor%}
+</ul>
+{%endif%}
+{%endwith%}
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.select_dataset')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col1">
+ <legend>continue with new study</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, totallines=totallines)}}
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+<div class="row">
+ <p class="two-col-sep-separator">OR</p>
+</div>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.select_study')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col2">
+ <legend>Select from existing study</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, totallines=totallines)}}
+
+ <button type="submit" class="btn btn-primary">go back</button>
+ </form>
+</div>
+{%endblock%}
diff --git a/uploader/templates/data_review.html b/uploader/templates/data_review.html
new file mode 100644
index 0000000..b7528fd
--- /dev/null
+++ b/uploader/templates/data_review.html
@@ -0,0 +1,85 @@
+{%extends "base.html"%}
+
+{%block title%}Data Review{%endblock%}
+
+{%block contents%}
+<h1 class="heading">data review</h1>
+
+<div class="row">
+ <h2 id="data-concerns">Data Concerns</h2>
+ <p>The following are some of the requirements that the data in your file
+ <strong>MUST</strong> fulfil before it is considered valid for this system:
+ </p>
+
+ <ol>
+ <li>File headings
+ <ul>
+ <li>The first row in the file should contains the headings. The number of
+ headings in this first row determines the number of columns expected for
+ all other lines in the file.</li>
+ <li>Each heading value in the first row MUST appear in the first row
+ <strong>ONE AND ONLY ONE</strong> time</li>
+ <li>The sample/cases (previously 'strains') headers in your first row will be
+ against those in the <a href="https://genenetwork.org"
+ title="Link to the GeneNetwork service">
+ GeneNetwork</a> database.<br />
+ <small class="text-muted">
+ If you encounter an error saying your sample(s)/case(s) do not exist
+ in the GeneNetwork database, then you will have to use the
+ <a href="{{url_for('samples.select_species')}}"
+ title="Upload samples/cases feature">Upload Samples/Cases</a>
+ option on this system to upload them.
+ </small>
+ </ul>
+ </li>
+
+ <li>Data
+ <ol>
+ <li><strong>NONE</strong> of the data cells/fields is allowed to be empty.
+ All fields/cells <strong>MUST</strong> contain a value.</li>
+ <li>The first column of the data rows will be considered a textual field,
+ holding the "identifier" for that row<li>
+ <li>Except for the first column/field for each data row,
+ <strong>NONE</strong> of the data columns/cells/fields should contain
+ spurious characters like `eeeee`, `5.555iloveguix`, etc...<br />
+ All of them should be decimal values</li>
+ <li>decimal numbers must conform to the following criteria:
+ <ul>
+ <li>when checking an average file decimal numbers must have exactly three
+ decimal places to the right of the decimal point.</li>
+ <li>when checking a standard error file decimal numbers must have six or
+ greater decimal places to the right of the decimal point.</li>
+ <li>there must be a number to the left side of the decimal place
+ (e.g. 0.55555 is allowed but .55555 is not).</li>
+ </ul>
+ </li>
+ </ol>
+ </li>
+ </ol>
+</div>
+
+
+<div class="row">
+ <h2 id="file-types">Supported File Types</h2>
+ We support the following file types:
+
+ <ul>
+ <li>Tab-Separated value files (.tsv)
+ <ul>
+ <li>The <strong>TAB</strong> character is used to separate the fields of each
+ column</li>
+ <li>The values of each field <strong>ARE NOT</strong> quoted.</li>
+ <li>Here is an
+ <a href="https://gitlab.com/fredmanglis/gnqc_py/-/blob/main/tests/test_data/no_data_errors.tsv">
+ example file</a> with a single data row.</li>
+ </ul>
+ </li>
+ <li>.txt files: Content has the same format as .tsv file above</li>
+ <li>.zip files: each zip file should contain
+ <strong>ONE AND ONLY ONE</strong> file of the .tsv or .txt type above.
+ <br />Any zip file with more than one file is invalid, and so is an empty
+ zip file.</li>
+ </ul>
+
+</div>
+{%endblock%}
diff --git a/uploader/templates/dbupdate_error.html b/uploader/templates/dbupdate_error.html
new file mode 100644
index 0000000..e1359d2
--- /dev/null
+++ b/uploader/templates/dbupdate_error.html
@@ -0,0 +1,12 @@
+{%extends "base.html"%}
+
+{%block title%}DB Update Error{%endblock%}
+
+{%block contents%}
+<h1 class="heading">database update error</h2>
+
+<p class="alert-danger">
+ <strong>Database Update Error</strong>: {{error_message}}
+</p>
+
+{%endblock%}
diff --git a/uploader/templates/dbupdate_hidden_fields.html b/uploader/templates/dbupdate_hidden_fields.html
new file mode 100644
index 0000000..ccbc299
--- /dev/null
+++ b/uploader/templates/dbupdate_hidden_fields.html
@@ -0,0 +1,29 @@
+{%macro hidden_fields(filename, filetype):%}
+
+<!-- {{kwargs}}: mostly for accessing the kwargs in macro -->
+
+<input type="hidden" name="filename" value="{{filename}}" />
+<input type="hidden" name="filetype" value="{{filetype}}" />
+{%if kwargs.get("totallines")%}
+<input type="hidden" name="totallines" value="{{kwargs['totallines']}}" />
+{%endif%}
+{%if kwargs.get("species"):%}
+<input type="hidden" name="species" value="{{kwargs['species']}}" />
+{%endif%}
+{%if kwargs.get("genechipid"):%}
+<input type="hidden" name="genechipid" value="{{kwargs['genechipid']}}" />
+{%endif%}
+{%if kwargs.get("inbredsetid"):%}
+<input type="hidden" name="inbredsetid" value="{{kwargs['inbredsetid']}}" />
+{%endif%}
+{%if kwargs.get("tissueid"):%}
+<input type="hidden" name="tissueid" value="{{kwargs['tissueid']}}" />
+{%endif%}
+{%if kwargs.get("studyid"):%}
+<input type="hidden" name="studyid" value="{{kwargs['studyid']}}" />
+{%endif%}
+{%if kwargs.get("datasetid"):%}
+<input type="hidden" name="datasetid" value="{{kwargs['datasetid']}}" />
+{%endif%}
+
+{%endmacro%}
diff --git a/uploader/templates/errors_display.html b/uploader/templates/errors_display.html
new file mode 100644
index 0000000..715cfcf
--- /dev/null
+++ b/uploader/templates/errors_display.html
@@ -0,0 +1,47 @@
+{%macro errors_display(errors, no_error_msg, error_message, complete)%}
+
+{%if errors | length == 0 %}
+<span {%if complete%}class="alert-success"{%endif%}>{{no_error_msg}}</span>
+{%else %}
+<p class="alert-danger">{{error_message}}</p>
+
+<table class="table reports-table">
+ <thead>
+ <tr>
+ <th>line number</th>
+ <th>column(s)</th>
+ <th>error</th>
+ <th>error message</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for error in errors%}
+ <tr>
+ <td>{{error["line"]}}</td>
+ <td>
+ {%if isinvalidvalue(error):%}
+ {{error.column}}
+ {%elif isduplicateheading(error): %}
+ {{error.columns}}
+ {%else: %}
+ -
+ {%endif %}
+ </td>
+ <td>
+ {%if isinvalidvalue(error):%}
+ Invalid Value
+ {%elif isduplicateheading(error): %}
+ Duplicate Header
+ {%else%}
+ Inconsistent Columns
+ {%endif %}
+ </td>
+ <td>{{error["message"]}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+</table>
+{%endif%}
+
+{%endmacro%}
diff --git a/uploader/templates/final_confirmation.html b/uploader/templates/final_confirmation.html
new file mode 100644
index 0000000..0727fc8
--- /dev/null
+++ b/uploader/templates/final_confirmation.html
@@ -0,0 +1,47 @@
+{%extends "base.html"%}
+{%from "dbupdate_hidden_fields.html" import hidden_fields%}
+
+{%block title%}Confirmation{%endblock%}
+
+{%macro display_item(item_name, item_data):%}
+<li>
+ <strong>{{item_name}}</strong>
+ {%if item_data%}
+ <ul>
+ {%for term,value in item_data.items():%}
+ <li><strong>{{term}}:</strong> {{value}}</li>
+ {%endfor%}
+ </ul>
+ {%endif%}
+</li>
+{%endmacro%}
+
+{%block contents%}
+<h2 class="heading">Final Confirmation</h2>
+
+<div class="two-col-sep-col1">
+ <p><strong>Selected Data</strong></p>
+ <ul>
+ <li><strong>File</strong>
+ <ul>
+ <li><strong>Filename</strong>: {{filename}}</li>
+ <li><strong>File Type</strong>: {{filetype}}</li>
+ </ul>
+ </li>
+ {{display_item("Species", the_species)}}
+ {{display_item("Platform", platform)}}
+ {{display_item("Study", study)}}
+ {{display_item("Dataset", dataset)}}
+ </ul>
+</div>
+
+<form method="POST" action="{{url_for('dbinsert.insert_data')}}">
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid,datasetid=datasetid, totallines=totallines)}}
+ <fieldset>
+ <input type="submit" class="btn btn-primary" value="confirm" />
+ </fieldset>
+</form>
+</div>
+{%endblock%}
diff --git a/uploader/templates/flash_messages.html b/uploader/templates/flash_messages.html
new file mode 100644
index 0000000..b7af178
--- /dev/null
+++ b/uploader/templates/flash_messages.html
@@ -0,0 +1,25 @@
+{%macro flash_all_messages()%}
+{%with messages = get_flashed_messages(with_categories=true)%}
+{%if messages:%}
+<ul>
+ {%for category, message in messages:%}
+ <li class="{{category}}">{{message}}</li>
+ {%endfor%}
+</ul>
+{%endif%}
+{%endwith%}
+{%endmacro%}
+
+{%macro flash_messages(filter_class)%}
+{%with messages = get_flashed_messages(with_categories=true)%}
+{%if messages:%}
+<ul>
+ {%for category, message in messages:%}
+ {%if filter_class in category%}
+ <li class="{{category}}">{{message}}</li>
+ {%endif%}
+ {%endfor%}
+</ul>
+{%endif%}
+{%endwith%}
+{%endmacro%}
diff --git a/uploader/templates/http-error.html b/uploader/templates/http-error.html
new file mode 100644
index 0000000..374fb86
--- /dev/null
+++ b/uploader/templates/http-error.html
@@ -0,0 +1,18 @@
+{%extends "base.html"%}
+
+{%block title%}HTTP Error: {{exc.code}}{%endblock%}
+
+{%block contents%}
+<h1>{{exc.code}}: {{exc.description}}</h1>
+
+<div class="row">
+ <p>
+ You attempted to access {{request_url}} which failed with the following
+ error:
+ </p>
+</div>
+
+<div class="row">
+ <pre>{{"\n".join(trace)}}</pre>
+</div>
+{%endblock%}
diff --git a/uploader/templates/index.html b/uploader/templates/index.html
new file mode 100644
index 0000000..89d2ae9
--- /dev/null
+++ b/uploader/templates/index.html
@@ -0,0 +1,81 @@
+{%extends "base.html"%}
+
+{%block title%}Data Upload{%endblock%}
+
+{%block contents%}
+<div class="row">
+ <h1 class="heading">data upload</h1>
+
+ <div class="explainer">
+ <p>Each of the sections below gives you a different option for data upload.
+ Please read the documentation for each section carefully to understand what
+ each section is about.</p>
+ </div>
+</div>
+
+<div class="row">
+ <h2 class="heading">R/qtl2 Bundles</h2>
+
+ <div class="explainer">
+ <p>This feature combines and extends the two upload methods below. Instead of
+ uploading one item at a time, the R/qtl2 bundle you upload can contain both
+ the genotypes data (samples/individuals/cases and their data) and the
+ expression data.</p>
+ <p>The R/qtl2 bundle, additionally, can contain extra metadata, that neither
+ of the methods below can handle.</p>
+
+ <a href="{{url_for('upload.rqtl2.select_species')}}"
+ title="Upload a zip bundle of R/qtl2 files">
+ <button class="btn btn-primary">upload R/qtl2 bundle</button></a>
+ </div>
+</div>
+
+
+<div class="row">
+ <h2 class="heading">Expression Data</h2>
+
+ <div class="explainer">
+ <p>This feature enables you to upload expression data. It expects the data to
+ be in <strong>tab-separated values (TSV)</strong> files. The data should be
+ a simple matrix of <em>phenotype × sample</em>, i.e. The first column is a
+ list of the <em>phenotypes</em> and the first row is a list of
+ <em>samples/cases</em>.</p>
+
+ <p>If you haven't done so please go to this page to learn the requirements for
+ file formats and helpful suggestions to enter your data in a fast and easy
+ way.</p>
+
+ <ol>
+ <li><strong>PLEASE REVIEW YOUR DATA.</strong>Make sure your data complies
+ with our system requirements. (
+ <a href="{{url_for('entry.data_review')}}#data-concerns"
+ title="Details for the data expectations.">Help</a>
+ )</li>
+ <li><strong>UPLOAD YOUR DATA FOR DATA VERIFICATION.</strong> We accept
+ <strong>.csv</strong>, <strong>.txt</strong> and <strong>.zip</strong>
+ files (<a href="{{url_for('entry.data_review')}}#file-types"
+ title="Details for the data expectations.">Help</a>)</li>
+ </ol>
+ </div>
+
+ <a href="{{url_for('entry.upload_file')}}"
+ title="Upload your expression data"
+ class="btn btn-primary">upload expression data</a>
+</div>
+
+<div class="row">
+ <h2 class="heading">samples/cases</h2>
+
+ <div class="explainer">
+ <p>For the expression data above, you need the samples/cases in your file to
+ already exist in the GeneNetwork database. If there are any samples that do
+ not already exist the upload of the expression data will fail.</p>
+ <p>This section gives you the opportunity to upload any missing samples</p>
+ </div>
+
+ <a href="{{url_for('samples.select_species')}}"
+ title="Upload samples/cases/individuals for your data"
+ class="btn btn-primary">upload Samples/Cases</a>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/insert_error.html b/uploader/templates/insert_error.html
new file mode 100644
index 0000000..5301288
--- /dev/null
+++ b/uploader/templates/insert_error.html
@@ -0,0 +1,32 @@
+{%extends "base.html"%}
+
+{%block title%}Data Insertion Failure{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Insertion Failure</h1>
+
+<div class="row">
+ <p>
+ There was an error inserting data into the database
+ </p>
+
+ <p>
+ Please notify the developers of this issue when you encounter it,
+ providing the information below.
+ </p>
+
+ <h4>Debugging Information</h4>
+
+ <ul>
+ <li><strong>job id</strong>: {{job["jobid"]}}</li>
+ </ul>
+</div>
+
+<div class="row">
+ <h4>STDERR Output</h4>
+ <pre class="cli-output">
+ {{job["stderr"]}}
+ </pre>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/insert_progress.html b/uploader/templates/insert_progress.html
new file mode 100644
index 0000000..52177d6
--- /dev/null
+++ b/uploader/templates/insert_progress.html
@@ -0,0 +1,46 @@
+{%extends "base.html"%}
+{%from "stdout_output.html" import stdout_output%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="5">
+{%endblock%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job_name}}</h1>
+
+<div class="row">
+ <form>
+ <div class="form-group">
+ <label for="job_status" class="form-label">status:</label>
+ <span class="form-text">{{job_status}}: {{message}}</span>
+ </div>
+
+{%if job.get("stdout", "").split("\n\n") | length < 3 %}
+{%set lines = 0%}
+{%else%}
+{%set lines = (job.get("stdout", "").split("\n\n") | length / 3) %}
+{%endif%}
+{%set totallines = job.get("totallines", lines+3) | int %}
+{%if totallines > 1000 %}
+{%set fraction = ((lines*1000)/totallines) %}
+{%else%}
+{%set fraction = (lines/totallines)%}
+{%endif%}
+
+ <div class="form-group">
+ <label for="job_{{job_id}}" class="form-label">inserting: </label>
+ <progress id="jobs_{{job_id}}"
+ value="{{(fraction)}}"
+ class="form-control">{{fraction*100}}</progress>
+ <span class="form-text text-muted">
+ {{"%.2f" | format(fraction * 100 | float)}}%</span>
+ </div>
+ </form>
+</div>
+
+
+{{stdout_output(job)}}
+
+{%endblock%}
diff --git a/uploader/templates/insert_success.html b/uploader/templates/insert_success.html
new file mode 100644
index 0000000..7e1fa8d
--- /dev/null
+++ b/uploader/templates/insert_success.html
@@ -0,0 +1,19 @@
+{%extends "base.html"%}
+{%from "stdout_output.html" import stdout_output%}
+
+{%block title%}Insertion Success{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Insertion Success</h1>
+
+<div class="row">
+<p>Data inserted successfully!</p>
+
+<p>The following queries were run:</p>
+</div>
+
+<div class="row">
+ {{stdout_output(job)}}
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/job_progress.html b/uploader/templates/job_progress.html
new file mode 100644
index 0000000..1af0763
--- /dev/null
+++ b/uploader/templates/job_progress.html
@@ -0,0 +1,40 @@
+{%extends "base.html"%}
+{%from "errors_display.html" import errors_display%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="5">
+{%endblock%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job_name}}</h2>
+
+<div class="row">
+ <form action="{{url_for('parse.abort')}}" method="POST">
+ <legend class="heading">Status</legend>
+ <div class="form-group">
+ <label for="job_status" class="form-label">status:</label>
+ <span class="form-text">{{job_status}}: {{message}}</span><br />
+ </div>
+
+ <div class="form-group">
+ <label for="job_{{job_id}}" class="form-label">parsing: </label>
+ <progress id="job_{{job_id}}"
+ value="{{progress/100}}"
+ class="form-control">
+ {{progress}}</progress>
+ <span class="form-text text-muted">{{"%.2f" | format(progress)}}%</span>
+ </div>
+
+ <input type="hidden" name="job_id" value="{{job_id}}" />
+
+ <button type="submit" class="btn btn-danger">Abort</button>
+ </form>
+</div>
+
+<div class="row">
+ {{errors_display(errors, "No errors found so far", "We have found the following errors so far", False)}}
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/no_such_job.html b/uploader/templates/no_such_job.html
new file mode 100644
index 0000000..42a2d48
--- /dev/null
+++ b/uploader/templates/no_such_job.html
@@ -0,0 +1,14 @@
+{%extends "base.html"%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="5;url={{url_for('entry.upload_file')}}">
+{%endblock%}
+
+{%block title%}No Such Job{%endblock%}
+
+{%block contents%}
+<h1 class="heading">No Such Job: {{job_id}}</h2>
+
+<p>No job, with the id '<em>{{job_id}}</em>' was found!</p>
+
+{%endblock%}
diff --git a/uploader/templates/parse_failure.html b/uploader/templates/parse_failure.html
new file mode 100644
index 0000000..31f6be8
--- /dev/null
+++ b/uploader/templates/parse_failure.html
@@ -0,0 +1,26 @@
+{%extends "base.html"%}
+
+{%block title%}Worker Failure{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Worker Failure</h1>
+
+<p>
+ There was an error while parsing your file.
+</p>
+
+<p>
+ Please notify the developers of this issue when you encounter it,
+ providing the information below.
+</p>
+
+<h4>Debugging Information</h4>
+
+<ul>
+ <li><strong>job id</strong>: {{job["job_id"]}}</li>
+ <li><strong>filename</strong>: {{job["filename"]}}</li>
+ <li><strong>line number</strong>: {{job["line_number"]}}</li>
+ <li><strong>Progress</strong>: {{job["percent"]}} %</li>
+</ul>
+
+{%endblock%}
diff --git a/uploader/templates/parse_results.html b/uploader/templates/parse_results.html
new file mode 100644
index 0000000..e2bf7f0
--- /dev/null
+++ b/uploader/templates/parse_results.html
@@ -0,0 +1,30 @@
+{%extends "base.html"%}
+{%from "errors_display.html" import errors_display%}
+
+{%block title%}Parse Results{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job_name}}: parse results</h2>
+
+{%if user_aborted%}
+<span class="alert-warning">Job aborted by the user</span>
+{%endif%}
+
+{{errors_display(errors, "No errors found in the file", "We found the following errors", True)}}
+
+{%if errors | length == 0 and not user_aborted %}
+<form method="post" action="{{url_for('dbinsert.select_platform')}}">
+ <input type="hidden" name="job_id" value="{{job_id}}" />
+ <input type="submit" value="update database" class="btn btn-primary" />
+</form>
+{%endif%}
+
+{%if errors | length > 0 or user_aborted %}
+<br />
+<a href="{{url_for('entry.upload_file')}}" title="Back to index page."
+ class="btn btn-primary">
+ Go back
+</a>
+{%endif%}
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/create-geno-dataset-success.html b/uploader/templates/rqtl2/create-geno-dataset-success.html
new file mode 100644
index 0000000..1b50221
--- /dev/null
+++ b/uploader/templates/rqtl2/create-geno-dataset-success.html
@@ -0,0 +1,55 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Select Genotypes Dataset</h2>
+
+<div class="explainer">
+ <p>You successfully created the genotype dataset with the following
+ information.
+ <dl>
+ <dt>ID</dt>
+ <dd>{{geno_dataset.id}}</dd>
+
+ <dt>Name</dt>
+ <dd>{{geno_dataset.name}}</dd>
+
+ <dt>Full Name</dt>
+ <dd>{{geno_dataset.fname}}</dd>
+
+ <dt>Short Name</dt>
+ <dd>{{geno_dataset.sname}}</dd>
+
+ <dt>Created On</dt>
+ <dd>{{geno_dataset.today}}</dd>
+
+ <dt>Public?</dt>
+ <dd>{%if geno_dataset.public == 0%}No{%else%}Yes{%endif%}</dd>
+ </dl>
+ </p>
+</div>
+
+<div class="row">
+ <form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('upload.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ <legend class="heading">select from existing genotype datasets</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id"
+ value="{{geno_dataset.id}}" />
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/create-probe-dataset-success.html b/uploader/templates/rqtl2/create-probe-dataset-success.html
new file mode 100644
index 0000000..790d174
--- /dev/null
+++ b/uploader/templates/rqtl2/create-probe-dataset-success.html
@@ -0,0 +1,59 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Create ProbeSet Dataset</h2>
+
+<div class="row">
+ <p>You successfully created the ProbeSet dataset with the following
+ information.
+ <dl>
+ <dt>Averaging Method</dt>
+ <dd>{{avgmethod.Name}}</dd>
+
+ <dt>ID</dt>
+ <dd>{{dataset.datasetid}}</dd>
+
+ <dt>Name</dt>
+ <dd>{{dataset.name2}}</dd>
+
+ <dt>Full Name</dt>
+ <dd>{{dataset.fname}}</dd>
+
+ <dt>Short Name</dt>
+ <dd>{{dataset.sname}}</dd>
+
+ <dt>Created On</dt>
+ <dd>{{dataset.today}}</dd>
+
+ <dt>DataScale</dt>
+ <dd>{{dataset.datascale}}</dd>
+ </dl>
+ </p>
+</div>
+
+<div class="row">
+ <form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('upload.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ <legend class="heading">Create ProbeSet dataset</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+ <input type="hidden" name="probe-study-id" value="{{study.Id}}" />
+ <input type="hidden" name="probe-dataset-id" value="{{dataset.datasetid}}" />
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/create-probe-study-success.html b/uploader/templates/rqtl2/create-probe-study-success.html
new file mode 100644
index 0000000..d0ee508
--- /dev/null
+++ b/uploader/templates/rqtl2/create-probe-study-success.html
@@ -0,0 +1,49 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Create ProbeSet Study</h2>
+
+<div class="row">
+ <p>You successfully created the ProbeSet study with the following
+ information.
+ <dl>
+ <dt>ID</dt>
+ <dd>{{study.id}}</dd>
+
+ <dt>Name</dt>
+ <dd>{{study.name}}</dd>
+
+ <dt>Full Name</dt>
+ <dd>{{study.fname}}</dd>
+
+ <dt>Short Name</dt>
+ <dd>{{study.sname}}</dd>
+
+ <dt>Created On</dt>
+ <dd>{{study.today}}</dd>
+ </dl>
+ </p>
+
+ <form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('upload.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ <legend class="heading">Create ProbeSet study</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="probe-study-id" value="{{study.studyid}}" />
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/create-tissue-success.html b/uploader/templates/rqtl2/create-tissue-success.html
new file mode 100644
index 0000000..5f2c5a0
--- /dev/null
+++ b/uploader/templates/rqtl2/create-tissue-success.html
@@ -0,0 +1,106 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Select Tissue</h2>
+
+<div class="row">
+ <p>You have successfully added a new tissue, organ or biological material with
+ the following details:</p>
+</div>
+
+<div class="row">
+ {{flash_all_messages()}}
+
+ <form id="frm-create-tissue-display"
+ method="POST"
+ action="#">
+ <legend class="heading">Create Tissue</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+
+ <div class="form-group">
+ <label>Name</label>
+ <label>{{tissue.TissueName}}</label>
+ </div>
+
+ <div class="form-group">
+ <label>Short Name</label>
+ <label>{{tissue.Short_Name}}</label>
+ </div>
+
+ {%if tissue.BIRN_lex_ID%}
+ <div class="form-group">
+ <label>BIRN Lex ID</label>
+ <label>{{tissue.BIRN_lex_ID}}</label>
+ </div>
+ {%endif%}
+
+ {%if tissue.BIRN_lex_Name%}
+ <div class="form-group">
+ <label>BIRN Lex Name</label>
+ <label>{{tissue.BIRN_lex_Name}}</label>
+ </div>
+ {%endif%}
+ </form>
+
+ <div id="action-buttons"
+ style="width:65ch;display:inline-grid;column-gap:5px;">
+
+ <form id="frm-create-tissue-success-continue"
+ method="POST"
+ action="{{url_for('upload.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ style="display: inline; width: 100%; grid-column: 1 / 2;
+ padding-top: 0.5em; text-align: center; border: none;
+ background-color: inherit;">
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+ </div>
+</div>
+
+<div class="row">
+ <p style="display:inline;width:100%;grid-column:2/3;text-align:center;
+ color:#336699;font-weight:bold;">
+ OR
+ </p>
+</div>
+
+<div class="row">
+ <form id="frm-create-tissue-success-select-existing"
+ method="POST"
+ action="{{url_for('upload.rqtl2.select_tissue',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ style="display: inline; width: 100%; grid-column: 3 / 4;
+ padding-top: 0.5em; text-align: center; border: none;
+ background-color: inherit;">
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+
+ <button type="submit" class="btn btn-primary">
+ select from existing tissues</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/index.html b/uploader/templates/rqtl2/index.html
new file mode 100644
index 0000000..f3329c2
--- /dev/null
+++ b/uploader/templates/rqtl2/index.html
@@ -0,0 +1,36 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Data Upload{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 data upload</h1>
+
+<h2>R/qtl2 Upload</h2>
+
+<form method="POST" action="{{url_for('upload.rqtl2.select_species')}}"
+ id="frm-rqtl2-upload">
+ <legend class="heading">upload R/qtl2 bundle</legend>
+ {{flash_messages("error-rqtl2")}}
+
+ <div class="form-group">
+ <label for="select:species" class="form-label">Species</label>
+ <select id="select:species"
+ name="species_id"
+ required="required"
+ class="form-control">
+ <option value="">Select species</option>
+ {%for spec in species%}
+ <option value="{{spec.SpeciesId}}">{{spec.MenuName}}</option>
+ {%endfor%}
+ </select>
+ <small class="form-text text-muted">
+ Data that you upload to the system should belong to a know species.
+ Here you can select the species that you wish to upload data for.
+ </small>
+ </div>
+
+ <button type="submit" class="btn btn-primary" />submit</button>
+</form>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/no-such-job.html b/uploader/templates/rqtl2/no-such-job.html
new file mode 100644
index 0000000..b17004f
--- /dev/null
+++ b/uploader/templates/rqtl2/no-such-job.html
@@ -0,0 +1,13 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 job status</h1>
+
+<h2>R/qtl2 Upload: No Such Job</h2>
+
+<p class="alert-danger">No job with ID {{jobid}} was found.</p>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/rqtl2-job-error.html b/uploader/templates/rqtl2/rqtl2-job-error.html
new file mode 100644
index 0000000..9817518
--- /dev/null
+++ b/uploader/templates/rqtl2/rqtl2-job-error.html
@@ -0,0 +1,39 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 job status</h1>
+
+<h2>R/qtl2 Upload: Job Status</h2>
+
+<div class="explainer">
+ <p>The processing of the R/qtl2 bundle you uploaded has failed. We have
+ provided some information below to help you figure out what the problem
+ could be.</p>
+ <p>If you find that you cannot figure out what the problem is on your own,
+ please contact the team running the system for assistance, providing the
+ following details:
+ <ul>
+ <li>R/qtl2 bundle you uploaded</li>
+ <li>This URL: <strong>{{request_url()}}</strong></li>
+ <li>(maybe) a screenshot of this page</li>
+ </ul>
+ </p>
+</div>
+
+<h4>stdout</h4>
+{{cli_output(job, "stdout")}}
+
+<h4>stderr</h4>
+{{cli_output(job, "stderr")}}
+
+<h4>Log</h4>
+<div class="cli-output">
+ {%for msg in messages%}
+ {{msg}}<br />
+ {%endfor%}
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/rqtl2-job-results.html b/uploader/templates/rqtl2/rqtl2-job-results.html
new file mode 100644
index 0000000..4ecd415
--- /dev/null
+++ b/uploader/templates/rqtl2/rqtl2-job-results.html
@@ -0,0 +1,24 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 job status</h1>
+
+<h2>R/qtl2 Upload: Job Status</h2>
+
+<div class="explainer">
+ <p>The processing of the R/qtl2 bundle you uploaded has completed
+ successfully.</p>
+ <p>You should now be able to use GeneNetwork to run analyses on your data.</p>
+</div>
+
+<h4>Log</h4>
+<div class="cli-output">
+ {%for msg in messages%}
+ {{msg}}<br />
+ {%endfor%}
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/rqtl2-job-status.html b/uploader/templates/rqtl2/rqtl2-job-status.html
new file mode 100644
index 0000000..e896f88
--- /dev/null
+++ b/uploader/templates/rqtl2/rqtl2-job-status.html
@@ -0,0 +1,20 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="3">
+{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 job status</h1>
+
+<h2>R/qtl2 Upload: Job Status</h2>
+
+<h4>Log</h4>
+<div class="cli-output">
+ <pre>{{"\n".join(messages)}}</pre>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/rqtl2-qc-job-error.html b/uploader/templates/rqtl2/rqtl2-qc-job-error.html
new file mode 100644
index 0000000..90e8887
--- /dev/null
+++ b/uploader/templates/rqtl2/rqtl2-qc-job-error.html
@@ -0,0 +1,120 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}R/qtl2 bundle: QC Job Error{%endblock%}
+
+{%macro errors_table(tableid, errors)%}
+<table id="{{tableid}}" class="table error-table">
+ <caption>{{caption}}</caption>
+ <thead>
+ <tr>
+ <th>Line</th>
+ <th>Column</th>
+ <th>Value</th>
+ <th>Message</th>
+ </tr>
+ </thead>
+ <tbody>
+ {%for error in errors%}
+ <tr>
+ <td>{{error.line}}</td>
+ <td>{{error.column}}</td>
+ <td>{{error.value}}</td>
+ <td>{{error.message}}</td>
+ </tr>
+ {%else%}
+ <tr>
+ <td colspan="4">No errors to display here.</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+</table>
+{%endmacro%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 bundle: QC job Error</h1>
+
+<div class="explainer">
+ <p>The R/qtl2 bundle has failed some <emph>Quality Control</emph> checks.</p>
+ <p>We list below some of the errors that need to be fixed before the data can
+ be uploaded onto GeneNetwork.</p>
+</div>
+
+{%if errorsgeneric | length > 0%}
+<h2 class="heading">Generic Errors ({{errorsgeneric | length}})</h3>
+<div class="explainer">
+ We found the following generic errors in your R/qtl2 bundle:
+</div>
+
+<h3>Missing Files</h3>
+<div class="explainer">
+ <p>These files are listed in the bundle's control file, but do not actually
+ exist in the bundle</p>
+</div>
+<table id="tbl-errors-missing-files" class="table error-table">
+ <thead>
+ <tr>
+ <th>Control File Key</th>
+ <th>Bundle File Name</th>
+ <th>Message</th>
+ </tr>
+ </thead>
+ <tbody>
+ {%for error in (errorsgeneric | selectattr("type", "equalto", "MissingFile"))%}
+ <tr>
+ <td>{{error.controlfilekey}}</td>
+ <td>{{error.filename}}</td>
+ <td>{{error.message}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+</table>
+
+<h3>Other Generic Errors</h3>
+{{errors_table("tbl-errors-generic", errorsgeneric| selectattr("type", "ne", "MissingFile"))}}
+{%endif%}
+
+{%if errorsgeno | length > 0%}
+<h2 class="heading">Geno Errors ({{errorsgeno | length}})</h3>
+<div class="explainer">
+ We found the following errors in the 'geno' file in your R/qtl2 bundle:
+</div>
+{{errors_table("tbl-errors-geno", errorsgeno[0:50])}}
+{%endif%}
+
+{%if errorspheno | length > 0%}
+<h2 class="heading">Pheno Errors ({{errorspheno | length}})</h3>
+<div class="explainer">
+ We found the following errors in the 'pheno' file in your R/qtl2 bundle:
+</div>
+{{errors_table("tbl-errors-pheno", errorspheno[0:50])}}
+{%endif%}
+
+{%if errorsphenose | length > 0%}
+<h2 class="heading">Phenose Errors ({{errorsphenose | length}})</h3>
+<div class="explainer">
+ We found the following errors in the 'phenose' file in your R/qtl2 bundle:
+</div>
+{{errors_table("tbl-errors-phenose", errorsphenose[0:50])}}
+{%endif%}
+
+{%if errorsphenocovar | length > 0%}
+<h2 class="heading">Phenocovar Errors ({{errorsphenocovar | length}})</h3>
+<div class="explainer">
+ We found the following errors in the 'phenocovar' file in your R/qtl2 bundle:
+</div>
+{{errorsphenocovar}}
+{%endif%}
+
+<h4>stdout</h4>
+{{cli_output(job, "stdout")}}
+
+<h4>stderr</h4>
+{{cli_output(job, "stderr")}}
+
+<h4>Log</h4>
+<div class="cli-output">
+ <pre>{{"\n".join(messages)}}</pre>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/rqtl2-qc-job-results.html b/uploader/templates/rqtl2/rqtl2-qc-job-results.html
new file mode 100644
index 0000000..59bc8cd
--- /dev/null
+++ b/uploader/templates/rqtl2/rqtl2-qc-job-results.html
@@ -0,0 +1,66 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}R/qtl2 bundle: QC job results{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 bundle: QC job results</h1>
+
+<div class="row">
+ <p>The R/qtl2 bundle you uploaded has passed all automated quality-control
+ checks successfully.</p>
+ <p>You may now continue to load the data into GeneNetwork for the bundle, with
+ the following details:</p>
+</div>
+
+<div class="row">
+ <form id="form-qc-job-results"
+ action="{{url_for('upload.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ method="POST">
+ <div class="form-group">
+ <legend>Species</legend>
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+
+ <span class="form-label">Name</span>
+ <span class="form-text">{{species.Name | capitalize}}</span>
+
+ <span class="form-label">Scientific</span>
+ <span class="form-text">{{species.FullName | capitalize}}</span>
+ </div>
+
+ <div class="form-group">
+ <legend>population</legend>
+ <input type="hidden" name="population_id" value="{{population.Id}}" />
+
+ <span class="form-label">Name</span>
+ <span class="form-text">{{population.InbredSetName}}</span>
+
+ <span class="form-label">Full Name</span>
+ <span class="form-text">{{population.FullName}}</span>
+
+ <span class="form-label">Genetic Type</span>
+ <span class="form-text">{{population.GeneticType}}</span>
+
+ <span class="form-label">Description</span>
+ <span class="form-text">{{population.Description or "-"}}</span>
+ </div>
+
+ <div class="form-group">
+ <legend>R/qtl2 Bundle File</legend>
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2bundle}}" />
+ <input type="hidden" name="original-filename" value="{{rqtl2bundleorig}}" />
+
+ <span class="form-label">Original Name</span>
+ <span class="form-text">{{rqtl2bundleorig}}</span>
+
+ <span class="form-label">Internal Name</span>
+ <span class="form-text">{{rqtl2bundle[0:25]}}&hellip;</span>
+ </div>
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/rqtl2-qc-job-status.html b/uploader/templates/rqtl2/rqtl2-qc-job-status.html
new file mode 100644
index 0000000..f4a6266
--- /dev/null
+++ b/uploader/templates/rqtl2/rqtl2-qc-job-status.html
@@ -0,0 +1,41 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="3">
+{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 bundle: QC job status</h1>
+
+{%if geno_percent%}
+<p>
+ <h2>Checking 'geno' file:</h2>
+ <progress id="prg-geno-checking" value="{{geno_percent}}" max="100">
+ {{geno_percent}}%</progress>
+ {{geno_percent}}%</p>
+{%endif%}
+
+{%if pheno_percent%}
+<p>
+ <h2>Checking 'pheno' file:</h2>
+ <progress id="prg-pheno-checking" value="{{pheno_percent}}" max="100">
+ {{pheno_percent}}%</progress>
+ {{pheno_percent}}%</p>
+{%endif%}
+
+{%if phenose_percent%}
+<p>
+ <h2>Checking 'phenose' file:</h2>
+ <progress id="prg-phenose-checking" value="{{phenose_percent}}" max="100">
+ {{phenose_percent}}%</progress>
+ {{phenose_percent}}%</p>
+{%endif%}
+
+<h4>Log</h4>
+<div class="cli-output">
+ <pre>{{"\n".join(messages)}}</pre>
+</div>
+{%endblock%}
diff --git a/uploader/templates/rqtl2/rqtl2-qc-job-success.html b/uploader/templates/rqtl2/rqtl2-qc-job-success.html
new file mode 100644
index 0000000..2861a04
--- /dev/null
+++ b/uploader/templates/rqtl2/rqtl2-qc-job-success.html
@@ -0,0 +1,37 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}R/qtl2 Bundle: Quality Control Successful{%endblock%}
+
+{%block contents%}
+<h2 class="heading">R/qtl2 Bundle: Quality Control Successful</h2>
+
+<div class="row">
+ <p>The R/qtl2 bundle you uploaded has passed <emph>all</emph> quality control
+ checks successfully, and is now ready for uploading into the database.</p>
+ <p>Click "Continue" below to proceed.</p>
+</div>
+
+<!--
+ The "action" on this form takes us to the next step, where we can
+ select all the other data necessary to enter the data into the database.
+ -->
+<div class="row">
+ <form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('upload.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ {{flash_all_messages()}}
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/select-geno-dataset.html b/uploader/templates/rqtl2/select-geno-dataset.html
new file mode 100644
index 0000000..873f9c3
--- /dev/null
+++ b/uploader/templates/rqtl2/select-geno-dataset.html
@@ -0,0 +1,144 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Select Genotypes Dataset</h2>
+
+<div class="row">
+ <p>Your R/qtl2 files bundle contains a "geno" specification. You will
+ therefore need to select from one of the existing Genotype datasets or
+ create a new one.</p>
+ <p>This is the dataset where your data will be organised under.</p>
+</div>
+
+<div class="row">
+ <form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('upload.rqtl2.select_geno_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ <legend class="heading">select from existing genotype datasets</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+
+ {{flash_messages("error-rqtl2-select-geno-dataset")}}
+
+ <div class="form-group">
+ <legend>Datasets</legend>
+ <label for="select:geno-datasets" class="form-label">Dataset</label>
+ <select id="select:geno-datasets"
+ name="geno-dataset-id"
+ required="required"
+ {%if datasets | length == 0%}
+ disabled="disabled"
+ {%endif%}
+ class="form-control"
+ aria-describedby="help-geno-dataset-select-dataset">
+ <option value="">Select dataset</option>
+ {%for dset in datasets%}
+ <option value="{{dset['Id']}}">{{dset["Name"]}} ({{dset["FullName"]}})</option>
+ {%endfor%}
+ </select>
+ <span id="help-geno-dataset-select-dataset" class="form-text text-muted">
+ Select from the existing genotype datasets for species
+ {{species.SpeciesName}} ({{species.FullName}}).
+ </span>
+ </div>
+
+ <button type="submit" class="btn btn-primary">select dataset</button>
+ </form>
+</div>
+
+<div class="row">
+ <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p>
+</div>
+
+<div class="row">
+ <form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('upload.rqtl2.create_geno_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ <legend class="heading">create a new genotype dataset</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+
+ {{flash_messages("error-rqtl2-create-geno-dataset")}}
+
+ <div class="form-group">
+ <label for="txt:dataset-name" class="form-label">Name</label>
+ <input type="text"
+ id="txt:dataset-name"
+ name="dataset-name"
+ maxlength="100"
+ required="required"
+ class="form-control"
+ aria-describedby="help-geno-dataset-name" />
+ <span id="help-geno-dataset-name" class="form-text text-muted">
+ Provide the new name for the genotype dataset, e.g. "BXDGeno"
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:dataset-fullname" class="form-label">Full Name</label>
+ <input type="text"
+ id="txt:dataset-fullname"
+ name="dataset-fullname"
+ required="required"
+ maxlength="100"
+ class="form-control"
+ aria-describedby="help-geno-dataset-fullname" />
+
+ <span id="help-geno-dataset-fullname" class="form-text text-muted">
+ Provide a longer name that better describes the genotype dataset, e.g.
+ "BXD Genotypes"
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:dataset-shortname" class="form-label">Short Name</label>
+ <input type="text"
+ id="txt:dataset-shortname"
+ name="dataset-shortname"
+ maxlength="100"
+ class="form-control"
+ aria-describedby="help-geno-dataset-shortname" />
+
+ <span id="help-geno-dataset-shortname" class="form-text text-muted">
+ Provide a short name for the genotype dataset. This is optional. If not
+ provided, we'll default to the same value as the "Name" above.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <input type="checkbox"
+ id="chk:dataset-public"
+ name="dataset-public"
+ checked="checked"
+ class="form-check"
+ aria-describedby="help-geno-datasent-public" />
+ <label for="chk:dataset-public" class="form-check-label">Public?</label>
+
+ <span id="help-geno-dataset-public" class="form-text text-muted">
+ Specify whether the dataset will be available publicly. Check to make the
+ dataset publicly available and uncheck to limit who can access the dataset.
+ </span>
+ </div>
+
+ <button type="submit" class="btn btn-primary">create new dataset</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/select-population.html b/uploader/templates/rqtl2/select-population.html
new file mode 100644
index 0000000..37731f0
--- /dev/null
+++ b/uploader/templates/rqtl2/select-population.html
@@ -0,0 +1,136 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Select Grouping/Population{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Select grouping/population</h1>
+
+<div class="explainer">
+ <p>The data is organised in a hierarchical form, beginning with
+ <em>species</em> at the very top. Under <em>species</em> the data is
+ organised by <em>population</em>, sometimes referred to as <em>grouping</em>.
+ (In some really old documents/systems, you might see this referred to as
+ <em>InbredSet</em>.)</p>
+ <p>In this section, you get to define what population your data is to be
+ organised by.</p>
+</div>
+
+<form method="POST"
+ action="{{url_for('upload.rqtl2.select_population', species_id=species.SpeciesId)}}">
+ <legend class="heading">select grouping/population</legend>
+ {{flash_messages("error-select-population")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+
+ <div class="form-group">
+ <label for="select:inbredset" class="form-label">population</label>
+ <select id="select:inbredset"
+ name="inbredset_id"
+ required="required"
+ class="form-control">
+ <option value="">Select a grouping/population</option>
+ {%for pop in populations%}
+ <option value="{{pop.InbredSetId}}">
+ {{pop.InbredSetName}} ({{pop.FullName}})</option>
+ {%endfor%}
+ </select>
+ <span class="form-text text-muted">If you are adding data to an already existing
+ population, simply pick the population from this drop-down selector. If
+ you cannot find your population from this list, try the form below to
+ create a new one..</span>
+ </div>
+
+ <button type="submit" class="btn btn-primary" />select population</button>
+</form>
+
+<p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p>
+
+<form method="POST"
+ action="{{url_for('upload.rqtl2.create_population', species_id=species.SpeciesId)}}">
+ <legend class="heading">create new grouping/population</legend>
+ {{flash_messages("error-create-population")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+
+ <div class="form-group">
+ <legend class="heading">mandatory</legend>
+
+ <div class="form-group">
+ <label for="txt:inbredset-name" class="form-label">name</label>
+ <input id="txt:inbredset-name"
+ name="inbredset_name"
+ type="text"
+ required="required"
+ maxlength="30"
+ placeholder="Enter grouping/population name"
+ class="form-control" />
+ <span class="form-text text-muted">This is a short name that identifies the
+ population. Useful for menus, and quick scanning.</span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:" class="form-label">full name</label>
+ <input id="txt:inbredset-fullname"
+ name="inbredset_fullname"
+ type="text"
+ required="required"
+ maxlength="100"
+ placeholder="Enter the grouping/population's full name"
+ class="form-control" />
+ <span class="form-text text-muted">This can be the same as the name above, or can
+ be longer. Useful for documentation, and human communication.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <legend class="heading">optional</legend>
+
+ <div class="form-group">
+ <label for="num:public" class="form-label">public?</label>
+ <select id="num:public"
+ name="public"
+ class="form-control">
+ <option value="0">0 - Only accessible to authorised users</option>
+ <option value="1">1 - Publicly accessible to all users</option>
+ <option value="2" selected>
+ 2 - Publicly accessible to all users</option>
+ </select>
+ <span class="form-text text-muted">This determines whether the
+ population/grouping will appear on the menus for users.</span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:inbredset-family" class="form-label">family</label>
+ <input id="txt:inbredset-family"
+ name="inbredset_family"
+ type="text"
+ placeholder="I am not sure what this is about."
+ class="form-control" />
+ <span class="form-text text-muted">I do not currently know what this is about.
+ This is a failure on my part to figure out what this is and provide a
+ useful description. Please feel free to remind me.</span>
+ </div>
+
+ <div class="form-group">
+ <label for="txtarea:" class="form-label">Description</label>
+ <textarea id="txtarea:description"
+ name="description"
+ rows="5"
+ placeholder="Enter a description of this grouping/population"
+ class="form-control"></textarea>
+ <span class="form-text text-muted">
+ A long-form description of what the population consists of. Useful for
+ humans.</span>
+ </div>
+ </div>
+
+ <button type="submit" class="btn btn-primary" />
+ create grouping/population</button>
+</form>
+
+{%endblock%}
+
+
+{%block javascript%}
+{%endblock%}
diff --git a/uploader/templates/rqtl2/select-probeset-dataset.html b/uploader/templates/rqtl2/select-probeset-dataset.html
new file mode 100644
index 0000000..26f52ed
--- /dev/null
+++ b/uploader/templates/rqtl2/select-probeset-dataset.html
@@ -0,0 +1,191 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Phenotype(ProbeSet) Dataset</h2>
+
+<div class="row">
+ <p>The R/qtl2 bundle you uploaded contains (a) "<strong>pheno</strong>"
+ file(s). This data needs to be organised under a dataset.</p>
+ <p>This page gives you the ability to do that.</p>
+</div>
+
+{%if datasets | length > 0%}
+<div class="row">
+ <form method="POST"
+ action="{{url_for('upload.rqtl2.select_probeset_dataset',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:select-probeset-dataset">
+ <legend class="heading">Select from existing ProbeSet datasets</legend>
+ {{flash_messages("error-rqtl2")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+ <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" />
+
+ <div class="form-group">
+ <label class="form-label" for="select:probe-dataset">Dataset</label>
+ <select id="select:probe-dataset"
+ name="probe-dataset-id"
+ required="required"
+ {%if datasets | length == 0%}disabled="disabled"{%endif%}
+ class="form-control"
+ aria-describedby="help-probe-dataset">
+ <option value="">Select a dataset</option>
+ {%for dataset in datasets%}
+ <option value={{dataset.Id}}>
+ {{dataset.Name}}
+ {%if dataset.FullName%}
+ -- ({{dataset.FullName}})
+ {%endif%}
+ </option>
+ {%endfor%}
+ </select>
+
+ <span id="help-probe-dataset" class="form-text text-muted">
+ Select from existing ProbeSet datasets.</span>
+ </div>
+
+ <button type="submit" class="btn btn-primary" />select dataset</button>
+</form>
+</div>
+
+<div class="row">
+ <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p>
+</div>
+{%endif%}
+
+<div class="row">
+ <p>Create an entirely new ProbeSet dataset for your data.</p>
+</div>
+
+<div class="row">
+ <form method="POST"
+ action="{{url_for('upload.rqtl2.create_probeset_dataset',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:create-probeset-dataset">
+ <legend class="heading">Create a new ProbeSet dataset</legend>
+ {{flash_messages("error-rqtl2-create-probeset-dataset")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+ <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" />
+
+ <div class="form-group">
+ <label class="form-label" for="select:average">averaging method</label>
+ <select id="select:average"
+ name="averageid"
+ required="required"
+ class="form-control"
+ aria-describedby="help-average">
+ <option value="">Select averaging method</option>
+ {%for avgmethod in avgmethods%}
+ <option value="{{avgmethod.Id}}">
+ {{avgmethod.Name}}
+ {%if avgmethod.Normalization%}
+ ({{avgmethod.Normalization}})
+ {%endif%}
+ </option>
+ {%endfor%}
+ </select>
+
+ <span id="help-average" class="form-text text-muted">
+ Select the averaging method used for your data.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="txt:datasetname">Name</label>
+ <input type="text" id="txt:datasetname" name="datasetname"
+ required="required"
+ maxlength="40"
+ title="Name of the dataset, e.g 'BXDMicroArray_ProbeSet_June03'. (Required)"
+ class="form-control"
+ aria-describedby="help-dataset-name" />
+
+ <span id="help-dataset-name" class="form-text text-muted">
+ Provide a name for the dataset e.g. "BXDMicroArray_ProbeSet_June03". This
+ is mandatory <strong>MUST</strong> be provided.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="txt:datasetfullname">Full Name</label>
+ <input type="text" id="txt:datasetfullname" name="datasetfullname"
+ required="required"
+ maxlength="100"
+ title="A longer name for the dataset, e.g. 'UTHSC Brain mRNA U74Av2 (Jun03) MAS5'. (Required)"
+ class="form-control"
+ aria-describedby="help-dataset-fullname" />
+
+ <span id="help-dataset-fullname" class="form-text text-muted">
+ Provide a longer, more descriptive name for the dataset e.g.
+ "UTHSC Brain mRNA U74Av2 (Jun03) MAS5". This is mandatory and
+ <strong>MUST</strong> be provided.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="txt:datasetshortname">Short Name</label>
+ <input type="text" id="txt:datasetshortname" name="datasetshortname"
+ maxlength="100"
+ title="An abbreviated name for the dataset, e.g 'Br_U_0603_M'. (Optional)"
+ class="form-control"
+ aria-describedby="help-dataset-shortname" />
+
+ <span id="help-dataset-shortname" class="form-text text-muted">
+ Provide a longer, more descriptive name for the dataset e.g. "Br_U_0603_M".
+ This is optional.
+ </span>
+ </div>
+
+ <div class="form-check">
+ <input type="checkbox" id="chk:public" name="datasetpublic"
+ checked="checked"
+ title="Whether or not the dataset is accessible by the general public."
+ class="form-check-input"
+ aria-describedby="help-public" />
+ <label class="form-check-label" for="chk:datasetpublic">Public?</label>
+
+ <span id="help-public" class="form-text text-muted">
+ Check to specify that the dataset will be publicly available. Uncheck to
+ limit access to the dataset.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="select:datasetdatascale">Data Scale</label>
+ <select id="select:datasetdatascale"
+ name="datasetdatascale"
+ required="required"
+ class="form-control"
+ aria-describedby="help-dataset-datascale">
+ <option value="log2" selected="selected">log2</option>
+ <option value="z_score">z_score</option>
+ <option value="log2_ratio">log2_ratio</option>
+ <option value="linear">linear</option>
+ <option value="linear_positive">linear_positive</option>
+ </select>
+
+ <span id="help-dataset-datascale" class="form-text text-muted">
+ Select from a list of scaling methods.
+ </span>
+ </div>
+
+ <button type="submit" class="btn btn-primary">create dataset</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/select-probeset-study-id.html b/uploader/templates/rqtl2/select-probeset-study-id.html
new file mode 100644
index 0000000..b9bf52e
--- /dev/null
+++ b/uploader/templates/rqtl2/select-probeset-study-id.html
@@ -0,0 +1,143 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages %}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Phenotype(ProbeSet) Study</h2>
+
+<div class="row">
+ <p>The R/qtl2 bundle you uploaded contains (a) "<strong>pheno</strong>"
+ file(s). This data needs to be organised under a study.</p>
+ <p>In this page, you can either select from a existing dataset:</p>
+
+ <form method="POST"
+ action="{{url_for('upload.rqtl2.select_probeset_study',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:select-probeset-study">
+ <legend class="heading">Select from existing ProbeSet studies</legend>
+ {{flash_messages("error-rqtl2-select-probeset-study")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+
+ <div>
+ <label for="select:probe-study" class="form-label">Study</label>
+ <select id="select:probe-study"
+ name="probe-study-id"
+ required="required"
+ aria-describedby="help-select-probeset-study"
+ {%if studies | length == 0%}disabled="disabled"{%endif%}
+ class="form-control">
+ <option value="">Select a study</option>
+ {%for study in studies%}
+ <option value={{study.Id}}>
+ {{study.Name}}
+ {%if study.FullName%}
+ -- ({{study.FullName}})
+ {%endif%}
+ </option>
+ {%endfor%}
+ </select>
+ <small id="help-select-probeset-study" class="form-text text-muted">
+ Select from existing ProbeSet studies.
+ </small>
+ </div>
+
+ <button type="submit" class="btn btn-primary">select study</button>
+ </form>
+</div>
+
+<div class="row">
+ <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p>
+</div>
+
+<div class="row">
+
+ <p>Create a new ProbeSet dataset below:</p>
+
+ <form method="POST"
+ action="{{url_for('upload.rqtl2.create_probeset_study',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:create-probeset-study">
+ <legend class="heading">Create new ProbeSet study</legend>
+
+ {{flash_messages("error-rqtl2-create-probeset-study")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="tissueid" value="{{tissue.Id}}" />
+
+ <div>
+ <label for="select:platform" class="form-label">Platform</label>
+ <select id="select:platform"
+ name="platformid"
+ required="required"
+ aria-describedby="help-select-platform"
+ {%if platforms | length == 0%}disabled="disabled"{%endif%}
+ class="form-control">
+ <option value="">Select a platform</option>
+ {%for platform in platforms%}
+ <option value="{{platform.GeneChipId}}">
+ {{platform.GeneChipName}} ({{platform.Name}})
+ </option>
+ {%endfor%}
+ </select>
+ <small id="help-select-platform" class="form-text text-muted">
+ Select from a list of known genomics platforms.
+ </small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:studyname" class="form-label">Study name</label>
+ <input type="text" id="txt:studyname" name="studyname"
+ placeholder="Name of the study. (Required)"
+ required="required"
+ maxlength="100"
+ class="form-control" />
+ <span class="form-text text-muted" id="help-study-name">
+ Provide a name for the study.</span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:studyfullname" class="form-label">Full Study Name</label>
+ <input type="text"
+ id="txt:studyfullname"
+ name="studyfullname"
+ placeholder="Longer name of the study. (Optional)"
+ maxlength="100"
+ class="form-control" />
+ <span class="form-text text-muted" id="help-study-full-name">
+ Provide a longer, more descriptive name for the study. This is optional
+ and you can leave it blank.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:studyshortname" class="form-label">Short Study Name</label>
+ <input type="text"
+ id="txt:studyshortname"
+ name="studyshortname"
+ placeholder="Shorter name of the study. (Optional)"
+ maxlength="100"
+ class="form-control" />
+ <span class="form-text text-muted" id="help-study-short-name">
+ Provide a shorter name for the study. This is optional and you can leave
+ it blank.
+ </span>
+ </div>
+
+ <button type="submit" class="btn btn-primary">create study</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/select-tissue.html b/uploader/templates/rqtl2/select-tissue.html
new file mode 100644
index 0000000..34e1758
--- /dev/null
+++ b/uploader/templates/rqtl2/select-tissue.html
@@ -0,0 +1,115 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Tissue</h2>
+
+<div class="row">
+ <p>The data you are uploading concerns a tissue, cell, organ, or other
+ biological material used in an experiment.</p>
+ <p>Select the appropriate biological material below</p>
+</div>
+
+{%if tissues | length > 0%}
+<div class="row">
+ <form method="POST"
+ action="{{url_for('upload.rqtl2.select_tissue',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:select-probeset-dataset">
+ <legend class="heading">Select from existing ProbeSet datasets</legend>
+ {{flash_messages("error-select-tissue")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+
+ <div class="form-group">
+ <label class="form-label" for="select-tissue">Tissue</label>
+ <select id="select-tissue"
+ name="tissueid"
+ required="required"
+ {%if tissues | length == 0%}disabled="disabled"{%endif%}
+ class="form-control"
+ aria-describedby="help-select-tissue">
+ <option value="">Select a tissue</option>
+ {%for tissue in tissues%}
+ <option value={{tissue.Id}}>
+ {{tissue.Name}}
+ {%if tissue.Short_Name%}
+ -- ({{tissue.Short_Name}})
+ {%endif%}
+ </option>
+ {%endfor%}
+ </select>
+
+ <span id="help-select-tissue" class="form-text text-muted">
+ Select from existing biological material.</span>
+ </div>
+
+ <button type="submit" class="btn btn-primary">use selected</button>
+ </form>
+</div>
+
+<div class="row">
+ <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p>
+</div>
+{%endif%}
+
+<div class="row">
+ <p>If you cannot find the biological material in the drop-down above, add it
+ to the system below.</p>
+
+ <form method="POST"
+ action="{{url_for('upload.rqtl2.create_tissue',
+ species_id=species.SpeciesId, population_id=population.Id)}}"
+ id="frm:create-probeset-dataset">
+ <legend class="heading">Add new tissue, organ or biological material</legend>
+ {{flash_messages("error-create-tissue")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+
+ <div class="form-group">
+ <label class="form-label" for="tissue-name">name</label>
+ <input type="text"
+ id="txt-tissuename"
+ name="tissuename"
+ required="required"
+ title = "A name to identify the tissue, organ or biological material."
+ class="form-control"
+ aria-describedby="help-tissue-name" />
+
+ <span class="form-text text-muted" id="help-tissue-name">
+ A name to identify the tissue, organ or biological material.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-shortname" class="form-label">short name</label>
+ <input type="text" id="txt-tissueshortname" name="tissueshortname"
+ required="required"
+ maxlength="7"
+ title="A short name (e.g. 'Mam') for the biological material."
+ class="form-control"
+ aria-describedby="help-tissue-short-name" />
+
+ <span class="form-text text-muted" id="help-tissue-short-name">
+ Provide a short name for the tissue, organ or biological material used in
+ the experiment.
+ </span>
+ </div>
+
+ <button type="submit" class="btn btn-primary" />add new material</button>
+</form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/rqtl2/summary-info.html b/uploader/templates/rqtl2/summary-info.html
new file mode 100644
index 0000000..1be87fa
--- /dev/null
+++ b/uploader/templates/rqtl2/summary-info.html
@@ -0,0 +1,65 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Summary</h2>
+
+<div class="row">
+ <p>This is the information you have provided to accompany the R/qtl2 bundle
+ you have uploaded. Please verify the information is correct before
+ proceeding.</p>
+</div>
+
+<div class="row">
+ <dl>
+ <dt>Species</dt>
+ <dd>{{species.SpeciesName}} ({{species.FullName}})</dd>
+
+ <dt>Population</dt>
+ <dd>{{population.InbredSetName}}</dd>
+
+ {%if geno_dataset%}
+ <dt>Genotype Dataset</dt>
+ <dd>{{geno_dataset.Name}} ({{geno_dataset.FullName}})</dd>
+ {%endif%}
+
+ {%if tissue%}
+ <dt>Tissue</dt>
+ <dd>{{tissue.TissueName}} ({{tissue.Name}}, {{tissue.Short_Name}})</dd>
+ {%endif%}
+
+ {%if probe_study%}
+ <dt>ProbeSet Study</dt>
+ <dd>{{probe_study.Name}} ({{probe_study.FullName}})</dd>
+ {%endif%}
+
+ {%if probe_dataset%}
+ <dt>ProbeSet Dataset</dt>
+ <dd>{{probe_dataset.Name2}} ({{probe_dataset.FullName}})</dd>
+ {%endif%}
+ </dl>
+</div>
+
+<div class="row">
+ <form id="frm:confirm-rqtl2bundle-details"
+ action="{{url_for('upload.rqtl2.confirm_bundle_details',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ <legend class="heading">Create ProbeSet dataset</legend>
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" />
+ <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" />
+ <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" />
+ <input type="hidden" name="probe-dataset-id" value="{{probe_dataset.Id}}" />
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+{%endblock%}
diff --git a/uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html
new file mode 100644
index 0000000..07c240f
--- /dev/null
+++ b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html
@@ -0,0 +1,276 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "upload_progress_indicator.html" import upload_progress_indicator%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+{%macro rqtl2_file_help()%}
+<span class="form-text text-muted">
+ <p>
+ Provide a valid R/qtl2 zip file here. In particular, ensure your zip bundle
+ contains exactly one control file and the corresponding files mentioned in
+ the control file.
+ </p>
+ <p>
+ The control file can be either a YAML or JSON file. <em>ALL</em> other data
+ files in the zip bundle should be CSV files.
+ </p>
+ <p>See the
+ <a href="https://kbroman.org/qtl2/assets/vignettes/input_files.html"
+ target="_blank">
+ R/qtl2 file format specifications
+ </a>
+ for more details.
+ </p>
+</span>
+{%endmacro%}
+{{upload_progress_indicator()}}
+
+<div id="resumable-file-display-template"
+ class="panel panel-info"
+ style="display: none">
+ <div class="panel-heading"></div>
+ <div class="panel-body"></div>
+</div>
+
+
+<h2 class="heading">Upload R/qtl2 Bundle</h2>
+
+<div id="resumable-drop-area"
+ style="display:none;background:#eeeeee;min-height:12em;border-radius:0.5em;padding:1em;">
+ <p>
+ <a id="resumable-browse-button" href="#"
+ class="btn btn-info">Browse</a>
+ </p>
+ <p class="form-text text-muted">
+ You can drag and drop your file here, or click the browse button.
+ Click on the file to remove it.
+ </p>
+ {{rqtl2_file_help()}}
+ <div id="resumable-selected-files"
+ style="display:flex;flex-direction:row;flex-wrap: wrap;justify-content:space-around;gap:10px 20px;"></div>
+ <div id="resumable-class-buttons" style="text-align: right;">
+ <button id="resumable-upload-button"
+ class="btn btn-primary"
+ style="display: none">start upload</button>
+ <button id="resumable-cancel-upload-button"
+ class="btn btn-danger"
+ style="display: none">cancel upload</button>
+ </div>
+ <div id="resumable-progress-bar" class="progress" style="display: none">
+ <div class="progress-bar"
+ role="progress-bar"
+ aria-valuenow="60"
+ aria-valuemin="0"
+ aria-valuemax="100"
+ style="width: 0%;">
+ Uploading: 60%
+ </div>
+ </div>
+</div>
+
+<form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('upload.rqtl2.upload_rqtl2_bundle',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data"
+ data-resumable-target="{{url_for(
+ 'upload.rqtl2.upload_rqtl2_bundle_chunked_post',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}">
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+
+ {{flash_all_messages()}}
+
+ <div class="form-group">
+ <legend class="heading">file upload</legend>
+ <label for="file-rqtl2-bundle" class="form-label">R/qtl2 bundle</label>
+ <input type="file" id="file-rqtl2-bundle" name="rqtl2_bundle_file"
+ accept="application/zip, .zip"
+ required="required"
+ class="form-control" />
+ {{rqtl2_file_help()}}
+ </div>
+
+ <button type="submit"
+ class="btn btn-primary"
+ data-toggle="modal"
+ data-target="#upload-progress-indicator">upload R/qtl2 bundle</button>
+</form>
+
+{%endblock%}
+
+{%block javascript%}
+<script src="{{url_for('base.node_modules',
+ filename='resumablejs/resumable.js')}}"></script>
+<script type="text/javascript" src="/static/js/upload_progress.js"></script>
+<script type="text/javascript">
+ function readBinaryFile(file) {
+ return new Promise((resolve, reject) => {
+ var _reader = new FileReader();
+ _reader.onload = (event) => {resolve(_reader.result);};
+ _reader.readAsArrayBuffer(file);
+ });
+ }
+
+ function computeFileChecksum(file) {
+ return readBinaryFile(file)
+ .then((content) => {
+ return window.crypto.subtle.digest(
+ "SHA-256", new Uint8Array(content));
+ }).then((digest) => {
+ return Uint8ArrayToHex(new Uint8Array(digest))
+ });
+ }
+
+ function Uint8ArrayToHex(arr) {
+ var toHex = (val) => {
+ _hex = val.toString(16);
+ if(_hex.length < 2) {
+ return "0" + val;
+ }
+ return _hex;
+ };
+ _hexstr = ""
+ arr.forEach((val) => {_hexstr += toHex(val)});
+ return _hexstr
+ }
+
+ var r = Resumable({
+ target: $("#frm-upload-rqtl2-bundle").attr("data-resumable-target"),
+ fileType: ["zip"],
+ maxFiles: 1,
+ forceChunkSize: true,
+ generateUniqueIdentifier: (file, event) => {
+ return computeFileChecksum(file).then((checksum) => {
+ var _relativePath = (file.webkitRelativePath
+ || file.relativePath
+ || file.fileName
+ || file.name);
+ return checksum + "-" + _relativePath.replace(
+ /[^a-zA-Z0-9_-]/img, "");
+ });
+ }
+ });
+
+ if(r.support) {
+ //Hide form and display drag&drop UI
+ $("#frm-upload-rqtl2-bundle").css("display", "none");
+ $("#resumable-drop-area").css("display", "block");
+
+ // Define UI elements for browse and drag&drop
+ r.assignDrop(document.getElementById("resumable-drop-area"));
+ r.assignBrowse(document.getElementById("resumable-browse-button"));
+
+ // Event handlers
+
+ function display_files(files) {
+ displayArea = $("#resumable-selected-files")
+ displayArea.empty();
+ files.forEach((file) => {
+ var displayElement = $(
+ "#resumable-file-display-template").clone();
+ displayElement.removeAttr("id");
+ displayElement.css("display", "");
+ displayElement.find(".panel-heading").text(file.fileName);
+ list = $("<ul></ul>");
+ list.append($("<li><strong>Name</strong>: "
+ + (file.name
+ || file.fileName
+ || file.relativePath
+ || file.webkitRelativePath)
+ + "</li>"));
+ list.append($("<li><strong>Size</strong>: "
+ + (file.size / (1024*1024)).toFixed(2)
+ + " MB</li>"));
+ list.append($("<li><strong>Unique Identifier</strong>: "
+ + file.uniqueIdentifier + "</li>"));
+ list.append($("<li><strong>Mime</strong>: "
+ + file.file.type
+ + "</li>"));
+ displayElement.find(".panel-body").append(list);
+ displayElement.appendTo("#resumable-selected-files");
+ });
+ }
+
+ r.on("filesAdded", function(files) {
+ display_files(files);
+ $("#resumable-upload-button").css("display", "");
+ $("#resumable-upload-button").on("click", (event) => {
+ r.upload();
+ });
+ });
+
+ r.on("uploadStart", (event) => {
+ $("#resumable-upload-button").css("display", "none");
+ $("#resumable-cancel-upload-button").css("display", "");
+ $("#resumable-cancel-upload-button").on("click", (event) => {
+ r.files.forEach((file) => {
+ if(file.isUploading()) {
+ file.abort();
+ }
+ });
+ $("#resumable-cancel-upload-button").css("display", "none");
+ $("#resumable-upload-button").on("click", (event) => {
+ r.files.forEach((file) => {file.retry();});
+ });
+ $("#resumable-upload-button").css("display", "");
+ });
+ });
+
+ r.on("progress", () => {
+ var progress = (r.progress() * 100).toFixed(2);
+ var pbar = $("#resumable-progress-bar > .progress-bar");
+ $("#resumable-progress-bar").css("display", "");
+ pbar.css("width", progress+"%");
+ pbar.attr("aria-valuenow", progress);
+ pbar.text("Uploading: " + progress + "%");
+ })
+
+ r.on("fileSuccess", (file, message) => {
+ if(message != "OK") {
+ var uri = (window.location.protocol
+ + "//"
+ + window.location.host
+ + message);
+ window.location.replace(uri);
+ }
+ });
+
+ r.on("error", (message, file) => {
+ filename = (file.webkitRelativePath
+ || file.relativePath
+ || file.fileName
+ || file.name);
+ jsonmsg = JSON.parse(message);
+ alert("There was an error while uploading your file '"
+ + filename
+ + "'. The error message was:\n\n\t"
+ + jsonmsg.error
+ + " ("
+ + jsonmsg.statuscode
+ + "): " + jsonmsg.message);
+ })
+ } else {
+ setup_upload_handlers(
+ "frm-upload-rqtl2-bundle", make_data_uploader(
+ function (form) {
+ var formdata = new FormData();
+ formdata.append(
+ "species_id",
+ form.querySelector('input[name="species_id"]').value);
+ formdata.append(
+ "population_id",
+ form.querySelector('input[name="population_id"]').value);
+ formdata.append(
+ "rqtl2_bundle_file",
+ form.querySelector("#file-rqtl2-bundle").files[0]);
+ return formdata;
+ }));
+ }
+</script>
+{%endblock%}
diff --git a/uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html
new file mode 100644
index 0000000..93b1dc9
--- /dev/null
+++ b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html
@@ -0,0 +1,33 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Upload R/qtl2 Bundle{%endblock%}
+
+{%block contents%}
+<h2 class="heading">Upload R/qtl2 Bundle</h2>
+
+<div class="row">
+ <p>You have successfully uploaded the zipped bundle of R/qtl2 files.</p>
+ <p>The next step is to select the various extra information we need to figure
+ out what to do with the data. You will select/create the relevant studies
+ and/or datasets to organise the data in the steps that follow.</p>
+ <p>Click "Continue" below to proceed.</p>
+
+ <form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('upload.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ {{flash_all_messages()}}
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+
+ <button type="submit" class="btn btn-primary">continue</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/samples/select-population.html b/uploader/templates/samples/select-population.html
new file mode 100644
index 0000000..da19ddc
--- /dev/null
+++ b/uploader/templates/samples/select-population.html
@@ -0,0 +1,99 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Select Grouping/Population{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Select grouping/population</h1>
+
+<div>
+ <p>We organise the samples/cases/strains in a hierarchichal form, starting
+ with <strong>species</strong> at the very top. Under species, we have a
+ grouping in terms of the relevant population
+ (e.g. Inbred populations, cell tissue, etc.)</p>
+</div>
+
+<form method="POST" action="{{url_for('samples.select_population',
+ species_id=species.SpeciesId)}}">
+ <legend class="heading">select grouping/population</legend>
+ {{flash_messages("error-select-population")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+
+ <div class="form-group">
+ <label for="select:inbredset" class="form-label">grouping/population</label>
+ <select id="select:inbredset"
+ name="inbredset_id"
+ required="required"
+ class="form-control">
+ <option value="">Select a grouping/population</option>
+ {%for pop in populations%}
+ <option value="{{pop.InbredSetId}}">
+ {{pop.InbredSetName}} ({{pop.FullName}})</option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <button type="submit" class="btn btn-primary">select population</button>
+</form>
+
+<p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p>
+
+<form method="POST" action="{{url_for('samples.create_population',
+ species_id=species.SpeciesId)}}">
+ <legend class="heading">create new grouping/population</legend>
+ {{flash_messages("error-create-population")}}
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <div class="form-group">
+ <legend>mandatory</legend>
+
+ <label for="txt:inbredset-name" class="form-label">name</label>
+ <input id="txt:inbredset-name"
+ name="inbredset_name"
+ type="text"
+ required="required"
+ placeholder="Enter grouping/population name"
+ class="form-control" />
+
+ <label for="txt:" class="form-label">full name</label>
+ <input id="txt:inbredset-fullname"
+ name="inbredset_fullname"
+ type="text"
+ required = "required"
+ placeholder="Enter the grouping/population's full name"
+ class="form-control" />
+ </div>
+ <div class="form-group">
+ <legend>Optional</legend>
+
+ <label for="num:public" class="form-label">public?</label>
+ <input id="num:public"
+ name="public"
+ type="number"
+ min="0" max="2" value="2"
+ class="form-control" />
+
+ <label for="txt:inbredset-family" class="form-label">family</label>
+ <input id="txt:inbredset-family"
+ name="inbredset_family"
+ type="text"
+ placeholder="I am not sure what this is about."
+ class="form-control" />
+
+ <label for="txtarea:" class="form-label">Description</label>
+ <textarea id="txtarea:description"
+ name="description"
+ rows="5"
+ placeholder="Enter a description of this grouping/population"
+ class="form-control"></textarea>
+ </div>
+
+ <button type="submit" class="btn btn-primary">create grouping/population</button>
+</form>
+
+{%endblock%}
+
+
+{%block javascript%}
+{%endblock%}
diff --git a/uploader/templates/samples/select-species.html b/uploader/templates/samples/select-species.html
new file mode 100644
index 0000000..edadc61
--- /dev/null
+++ b/uploader/templates/samples/select-species.html
@@ -0,0 +1,30 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Select Grouping/Population{%endblock%}
+
+{%block contents%}
+<h2 class="heading">upload samples/cases</h2>
+
+<p>We need to know what species your data belongs to.</p>
+
+{{flash_all_messages()}}
+
+<form method="POST" action="{{url_for('samples.select_species')}}">
+ <legend class="heading">upload samples</legend>
+ <div class="form-group">
+ <label for="select_species02" class="form-label">Species</label>
+ <select id="select_species02"
+ name="species_id"
+ required="required"
+ class="form-control">
+ <option value="">Select species</option>
+ {%for spec in species%}
+ <option value="{{spec.SpeciesId}}">{{spec.MenuName}}</option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <button type="submit" class="btn btn-primary">submit</button>
+</form>
+{%endblock%}
diff --git a/uploader/templates/samples/upload-failure.html b/uploader/templates/samples/upload-failure.html
new file mode 100644
index 0000000..09e2ecf
--- /dev/null
+++ b/uploader/templates/samples/upload-failure.html
@@ -0,0 +1,27 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}Samples Upload Failure{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job.job_name}}</h2>
+
+<p>There was a failure attempting to upload the samples.</p>
+
+<p>Here is some information to help with debugging the issue. Provide this
+ information to the developer/maintainer.</p>
+
+<h3>Debugging Information</h3>
+<ul>
+ <li><strong>job id</strong>: {{job.job_id}}</li>
+ <li><strong>status</strong>: {{job.status}}</li>
+ <li><strong>job type</strong>: {{job["job-type"]}}</li>
+</ul>
+
+<h4>stdout</h4>
+{{cli_output(job, "stdout")}}
+
+<h4>stderr</h4>
+{{cli_output(job, "stderr")}}
+
+{%endblock%}
diff --git a/uploader/templates/samples/upload-progress.html b/uploader/templates/samples/upload-progress.html
new file mode 100644
index 0000000..7bb02be
--- /dev/null
+++ b/uploader/templates/samples/upload-progress.html
@@ -0,0 +1,22 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="5">
+{%endblock%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job.job_name}}</h2>
+
+<p>
+<strong>status</strong>:
+<span>{{job["status"]}} ({{job.get("message", "-")}})</span><br />
+</p>
+
+<p>saving to database...</p>
+
+{{cli_output(job, "stdout")}}
+
+{%endblock%}
diff --git a/uploader/templates/samples/upload-samples.html b/uploader/templates/samples/upload-samples.html
new file mode 100644
index 0000000..e62de57
--- /dev/null
+++ b/uploader/templates/samples/upload-samples.html
@@ -0,0 +1,139 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Upload Samples{%endblock%}
+
+{%block css%}{%endblock%}
+
+{%block contents%}
+<h1 class="heading">upload samples</h1>
+
+{{flash_messages("alert-success")}}
+
+<p>You can now upload a character-separated value (CSV) file that contains
+ details about your samples. The CSV file should have the following fields:
+ <dl>
+ <dt>Name</dt>
+ <dd>The primary name for the sample</dd>
+
+ <dt>Name2</dt>
+ <dd>A secondary name for the sample. This can simply be the same as
+ <strong>Name</strong> above. This field <strong>MUST</strong> contain a
+ value.</dd>
+
+ <dt>Symbol</dt>
+ <dd>A symbol for the sample. Can be an empty field.</dd>
+
+ <dt>Alias</dt>
+ <dd>An alias for the sample. Can be an empty field.</dd>
+ </dl>
+</p>
+
+<form id="form-samples"
+ method="POST"
+ action="{{url_for('samples.upload_samples',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ enctype="multipart/form-data">
+ <legend class="heading">upload samples</legend>
+
+ <div class="form-group">
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <label class="form-label">species:</label>
+ <span class="form-text">{{species.SpeciesName}} [{{species.MenuName}}]</span>
+ </div>
+
+ <div class="form-group">
+ <input type="hidden" name="inbredset_id" value="{{population.InbredSetId}}" />
+ <label class="form-label">grouping/population:</label>
+ <span class="form-text">{{population.Name}} [{{population.FullName}}]</span>
+ </div>
+
+ <div class="form-group">
+ <label for="file-samples" class="form-label">select file</label>
+ <input type="file" name="samples_file" id="file:samples"
+ accept="text/csv, text/tab-separated-values"
+ class="form-control" />
+ </div>
+
+ <div class="form-group">
+ <label for="select:separator" class="form-label">field separator</label>
+ <select id="select:separator"
+ name="separator"
+ required="required"
+ class="form-control">
+ <option value="">Select separator for your file: (default is comma)</option>
+ <option value="&#x0009;">TAB</option>
+ <option value="&#x0020;">Space</option>
+ <option value=",">Comma</option>
+ <option value=";">Semicolon</option>
+ <option value="other">Other</option>
+ </select>
+ <input id="txt:separator"
+ type="text"
+ name="other_separator"
+ class="form-control" />
+ <small class="form-text text-muted">
+ If you select '<strong>Other</strong>' for the field separator value,
+ enter the character that separates the fields in your CSV file in the form
+ field below.
+ </small>
+ </div>
+
+ <div class="form-group form-check">
+ <input id="chk:heading"
+ type="checkbox"
+ name="first_line_heading"
+ class="form-check-input" />
+ <label for="chk:heading" class="form-check-label">
+ first line is a heading?</label>
+ <small class="form-text text-muted">
+ Select this if the first line in your file contains headings for the
+ columns.
+ </small>
+ </div>
+
+ <div class="form-group">
+ <label for="txt:delimiter" class="form-label">field delimiter</label>
+ <input id="txt:delimiter"
+ type="text"
+ name="field_delimiter"
+ maxlength="1"
+ class="form-control" />
+ <small class="form-text text-muted">
+ If there is a character delimiting the string texts within particular
+ fields in your CSV, provide the character here. This can be left blank if
+ no such delimiters exist in your file.
+ </small>
+ </div>
+
+ <button type="submit"
+ class="btn btn-primary">upload samples file</button>
+</form>
+
+<table id="tbl:samples-preview" class="table">
+ <caption class="heading">preview content</caption>
+
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Name2</th>
+ <th>Symbol</th>
+ <th>Alias</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr id="default-row">
+ <td colspan="4">
+ Please make some selections to preview the data.</td>
+ </tr>
+ </tbody>
+</table>
+
+{%endblock%}
+
+
+{%block javascript%}
+<script src="/static/js/upload_samples.js" type="text/javascript"></script>
+{%endblock%}
diff --git a/uploader/templates/samples/upload-success.html b/uploader/templates/samples/upload-success.html
new file mode 100644
index 0000000..cb745c3
--- /dev/null
+++ b/uploader/templates/samples/upload-success.html
@@ -0,0 +1,18 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job.job_name}}</h2>
+
+<p>
+<strong>status</strong>:
+<span>{{job["status"]}} ({{job.get("message", "-")}})</span><br />
+</p>
+
+<p>Successfully uploaded the samples.</p>
+
+{{cli_output(job, "stdout")}}
+
+{%endblock%}
diff --git a/uploader/templates/select_dataset.html b/uploader/templates/select_dataset.html
new file mode 100644
index 0000000..2f07de8
--- /dev/null
+++ b/uploader/templates/select_dataset.html
@@ -0,0 +1,161 @@
+{%extends "base.html"%}
+{%from "dbupdate_hidden_fields.html" import hidden_fields%}
+
+{%block title%}Select Dataset{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
+{%endblock%}
+
+{%block contents%}
+<h2 class="heading">{{filename}}: select dataset</h2>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.final_confirmation')}}"
+ id="select-dataset-form" class="two-col-sep-col1">
+ <legend class="heading">choose existing dataset</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, totallines=totallines)}}
+
+ <div class="form-group">
+ <label for="datasetid" class="form-label">dataset:</label>
+ <select id="datasetid" name="datasetid" class="form-control"
+ {%if datasets | length == 0:%}
+ disabled="disabled"
+ {%endif%}>
+ {%for dataset in datasets:%}
+ <option value="{{dataset['Id']}}">
+ [{{dataset["Name"]}}] - {{dataset["FullName"]}}
+ </option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <button type="submit" class="btn btn-primary"
+ {%if datasets | length == 0:%}
+ disabled="disabled"
+ {%endif%} />update database</button>
+</form>
+</div>
+
+<div class="row">
+ <p class="two-col-sep-separator">OR</p>
+</div>
+
+<div class="row">
+ <form method="POST" id="create-dataset-form"
+ action="{{url_for('dbinsert.create_dataset')}}"
+ class="two-col-sep-col2">
+ <legend class="heading">create new dataset</legend>
+ {{hidden_fields(
+ filename, filetype, species=species, genechipid=genechipid,
+ studyid=studyid, totallines=totallines)}}
+
+ {%with messages = get_flashed_messages(with_categories=true)%}
+ {%if messages:%}
+ <ul>
+ {%for category, message in messages:%}
+ <li class="{{category}}">{{message}}</li>
+ {%endfor%}
+ </ul>
+ {%endif%}
+ {%endwith%}
+
+ <div class="form-group">
+ <label for="avgid" class="form-label">average:</label>
+ <select id="avgid" name="avgid" required="required" class="form-control">
+ <option value="">Select averaging method</option>
+ {%for method in avgmethods:%}
+ <option value="{{method['AvgMethodId']}}"
+ {%if avgid is defined and method['AvgMethodId'] | int == avgid | int%}
+ selected="selected"
+ {%endif%}>
+ {{method["Name"]}}
+ </option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <div class="form-group">
+ <label for="datasetname" class="form-label">name:</label>
+ <input id="datasetname" name="datasetname" type="text"
+ class="form-control"
+ {%if datasetname is defined %}
+ value="{{datasetname}}"
+ {%endif%} />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetname2" class="form-label">name 2:</label>
+ <input id="datasetname2" name="datasetname2" type="text"
+ required="required" class="form-control"
+ {%if datasetname2 is defined %}
+ value="{{datasetname2}}"
+ {%endif%} />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetfullname" class="form-label">full name:</label>
+ <input id="datasetfullname" name="datasetfullname" type="text"
+ required="required" class="form-control"
+ {%if datasetfullname is defined %}
+ value="{{datasetfullname}}"
+ {%endif%} />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetshortname" class="form-label">short name:</label>
+ <input id="datasetshortname" name="datasetshortname" type="text"
+ required="required" class="form-control"
+ {%if datasetshortname is defined %}
+ value="{{datasetshortname}}"
+ {%endif%} />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetpublic" class="form-label">public:</label>
+ <input id="datasetpublic" name="datasetpublic" type="number"
+ required="required" min="0" max="2"
+ {%if datasetpublic is defined %}
+ value="{{datasetpublic | int}}"
+ {%else%}
+ value="0"
+ {%endif%}
+ class="form-control" />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetconfidentiality">confidentiality:</label>
+ <input id="datasetconfidentiality" name="datasetconfidentiality"
+ type="number" required="required" min="0" max="2"
+ {%if datasetconfidentiality is defined %}
+ value="{{datasetconfidentiality | int}}"
+ {%else%}
+ value="0"
+ {%endif%}
+ class="form-control" />
+ </div>
+
+ <div class="form-group">
+ <label for="datasetdatascale" class="form-label">data scale:</label>
+ <select id="datasetdatascale" name="datasetdatascale" class="form-control">
+ <option value="">None</option>
+ {%for dscale in datascales:%}
+ <option value="{{dscale}}"
+ {%if datasetdatascale is defined and dscale == datasetdatascale%}
+ selected="selected"
+ {%elif dscale == "log2":%}
+ selected="selected"
+ {%endif%}>
+ {{dscale}}
+ </option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <button type="submit" class="btn btn-primary">create new dataset</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/select_platform.html b/uploader/templates/select_platform.html
new file mode 100644
index 0000000..d9bc68f
--- /dev/null
+++ b/uploader/templates/select_platform.html
@@ -0,0 +1,82 @@
+{%extends "base.html"%}
+
+{%block title%}Select Dataset{%endblock%}
+
+{%block contents%}
+<h2 class="heading">{{filename}}: select platform</h2>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.select_study')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}">
+ <input type="hidden" name="filename" value="{{filename}}" />
+ <input type="hidden" name="filetype" value="{{filetype}}" />
+ <input type="hidden" name="totallines" value="{{totallines}}" />
+
+ <div class="form-group">
+ <label for="species" class="form-label">species</label>
+ <select id="species" name="species" class="form-control">
+ {%for row in species:%}
+ <option value="{{row['SpeciesId']}}"
+ {%if row["Name"] == default_species:%}
+ selected="selected"
+ {%endif%}>
+ {{row["MenuName"]}}
+ </option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <table id="genechips-table" class="table">
+ <caption>select platform</caption>
+ <thead>
+ <tr>
+ <th>Select</th>
+ <th>GeneChip ID</th>
+ <th>GeneChip Name</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for chip in genechips:%}
+ <tr>
+ <td>
+ <input type="radio" name="genechipid" value="{{chip['GeneChipId']}}"
+ required="required" />
+ </td>
+ <td>{{chip["GeneChipId"]}}</td>
+ <td>{{chip["GeneChipName"]}}</td>
+ </tr>
+ {%else%}
+ <tr>
+ <td colspan="5">No chips found for selected species</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+
+ <button type="submit" class="btn btn-primary">submit platform</button>
+ </form>
+</div>
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/utils.js"></script>
+<script type="text/javascript" src="/static/js/select_platform.js"></script>
+<script type="text/javascript">
+ document.getElementById(
+ "species").addEventListener("change", update_genechips);
+ document.getElementById(
+ "genechips-table").getElementsByTagName(
+ "tbody")[0].addEventListener(
+ "click",
+ function(event) {
+ if(event.target.tagName.toLowerCase() == "td") {
+ return select_row_radio(event.target.parentElement);
+ }
+ if(event.target.tagName.toLowerCase() == "td") {
+ return select_row_radio(event.target);
+ }
+ return false;
+ });
+</script>
+{%endblock%}
diff --git a/uploader/templates/select_species.html b/uploader/templates/select_species.html
new file mode 100644
index 0000000..3b1a8a9
--- /dev/null
+++ b/uploader/templates/select_species.html
@@ -0,0 +1,92 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+{%from "upload_progress_indicator.html" import upload_progress_indicator%}
+
+{%block title%}expression data: select species{%endblock%}
+
+{%block contents%}
+{{upload_progress_indicator()}}
+
+<h2 class="heading">expression data: select species</h2>
+
+<div class="row">
+ <form action="{{url_for('entry.upload_file')}}"
+ method="POST"
+ enctype="multipart/form-data"
+ id="frm-upload-expression-data">
+ <legend class="heading">upload expression data</legend>
+ {{flash_messages("error-expr-data")}}
+
+ <div class="form-group">
+ <label for="select_species01" class="form-label">Species</label>
+ <select id="select_species01"
+ name="speciesid"
+ required="required"
+ class="form-control">
+ <option value="">Select species</option>
+ {%for aspecies in species%}
+ <option value="{{aspecies.SpeciesId}}">{{aspecies.MenuName}}</option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <div class="form-group">
+ <legend class="heading">file type</legend>
+
+ <div class="form-check">
+ <input type="radio" name="filetype" value="average" id="filetype_average"
+ required="required" class="form-check-input" />
+ <label for="filetype_average" class="form-check-label">average</label>
+ </div>
+
+ <div class="form-check">
+ <input type="radio" name="filetype" value="standard-error"
+ id="filetype_standard_error" required="required"
+ class="form-check-input" />
+ <label for="filetype_standard_error" class="form-check-label">
+ standard error
+ </label>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <span id="no-file-error" class="alert-danger" style="display: none;">
+ No file selected
+ </span>
+ <label for="file_upload" class="form-label">select file</label>
+ <input type="file" name="qc_text_file" id="file_upload"
+ accept="text/plain, text/tab-separated-values, application/zip"
+ class="form-control"/>
+ </div>
+
+ <button type="submit"
+ class="btn btn-primary"
+ data-toggle="modal"
+ data-target="#upload-progress-indicator">upload file</button>
+ </form>
+</div>
+{%endblock%}
+
+
+{%block javascript%}
+<script type="text/javascript" src="static/js/upload_progress.js"></script>
+<script type="text/javascript">
+ function setup_formdata(form) {
+ var formdata = new FormData();
+ formdata.append(
+ "speciesid",
+ form.querySelector("#select_species01").value)
+ formdata.append(
+ "qc_text_file",
+ form.querySelector("input[type='file']").files[0]);
+ formdata.append(
+ "filetype",
+ selected_filetype(
+ Array.from(form.querySelectorAll("input[type='radio']"))));
+ return formdata;
+ }
+
+ setup_upload_handlers(
+ "frm-upload-expression-data", make_data_uploader(setup_formdata));
+</script>
+{%endblock%}
diff --git a/uploader/templates/select_study.html b/uploader/templates/select_study.html
new file mode 100644
index 0000000..648ad4c
--- /dev/null
+++ b/uploader/templates/select_study.html
@@ -0,0 +1,108 @@
+{%extends "base.html"%}
+{%from "dbupdate_hidden_fields.html" import hidden_fields%}
+
+{%block title%}Select Dataset{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
+{%endblock%}
+
+{%block contents%}
+<h2 class="heading">{{filename}}: select study</h2>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.select_dataset')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col1">
+ <legend class="heading">Select from existing study</legend>
+ {{hidden_fields(filename, filetype, species=species, genechipid=genechipid,
+ totallines=totallines)}}
+
+ <div class="form-group">
+ <label class="form-label" for="study">study:</label>
+ <select id="study" name="studyid" class="form-control">
+ {%for study in studies:%}
+ <option value="{{study['Id']}}">{{study["Name"]}}</option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <button type="submit"
+ class="btn btn-primary"
+ {%if studies | length == 0:%}
+ disabled="disabled"
+ {%endif%} />submit selected study</button>
+</form>
+</div>
+
+<div class="row">
+ <p class="two-col-sep-separator">OR</p>
+</div>
+
+<div class="row">
+ <form method="POST" action="{{url_for('dbinsert.create_study')}}"
+ id="select-platform-form" data-genechips="{{genechips_data}}"
+ class="two-col-sep-col2">
+ {%with messages = get_flashed_messages(with_categories=true)%}
+ {%if messages:%}
+ <ul>
+ {%for category, message in messages:%}
+ <li class="{{category}}">{{message}}</li>
+ {%endfor%}
+ </ul>
+ {%endif%}
+ {%endwith%}
+ <legend class="heading">Create new study</legend>
+ {{hidden_fields(filename, filetype, species=species, genechipid=genechipid,
+ totallines=totallines)}}
+
+ <div class="form-group">
+ <label class="form-label" for="studyname">name:</label>
+ <input type="text" id="studyname" name="studyname" class="form-control"
+ required="required"
+ {%if studyname:%}
+ value="{{studyname}}"
+ {%endif%} />
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="group">group:</label>
+ <select id="group" name="inbredsetid" class="form-control"
+ required="required">
+ <option value="">Select group</option>
+ {%for family in groups:%}
+ <optgroup label="{{family}}">
+ {%for group in groups[family]:%}
+ <option value="{{group['InbredSetId']}}"
+ {%if group["InbredSetId"] == selected_group:%}
+ selected="selected"
+ {%endif%}>
+ {{group["FullName"]}}
+ </option>
+ {%endfor%}
+ </optgroup>
+ {%endfor%}
+ </select>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="tissue">tissue:</label>
+ <select id="tissue" name="tissueid" class="form-control"
+ required="required">
+ <option value="">Select type</option>
+ {%for tissue in tissues:%}
+ <option value="{{tissue['TissueId']}}"
+ {%if tissue["TissueId"] == selected_tissue:%}
+ selected="selected"
+ {%endif%}>
+ {{tissue["Name"]}}
+ </option>
+ {%endfor%}
+ </select>
+ </div>
+
+ <button type="submit" class="btn btn-primary">create study</button>
+ </form>
+</div>
+
+{%endblock%}
diff --git a/uploader/templates/stdout_output.html b/uploader/templates/stdout_output.html
new file mode 100644
index 0000000..85345a9
--- /dev/null
+++ b/uploader/templates/stdout_output.html
@@ -0,0 +1,8 @@
+{%macro stdout_output(job)%}
+
+<h4>STDOUT Output</h4>
+<div class="cli-output">
+ <pre>{{job.get("stdout", "")}}</pre>
+</div>
+
+{%endmacro%}
diff --git a/uploader/templates/unhandled_exception.html b/uploader/templates/unhandled_exception.html
new file mode 100644
index 0000000..6e6a051
--- /dev/null
+++ b/uploader/templates/unhandled_exception.html
@@ -0,0 +1,21 @@
+{%extends "base.html"%}
+
+{%block title%}System Error{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
+{%endblock%}
+
+{%block contents%}
+<p>
+ An error has occured, and your request has been aborted. Please notify the
+ administrator to try and get this sorted.
+</p>
+<p>
+ Provide the following information to help the administrator figure out and fix
+ the issue:<br />
+ <hr /><br />
+ {{trace}}
+ <hr /><br />
+</p>
+{%endblock%}
diff --git a/uploader/templates/upload_progress_indicator.html b/uploader/templates/upload_progress_indicator.html
new file mode 100644
index 0000000..e274e83
--- /dev/null
+++ b/uploader/templates/upload_progress_indicator.html
@@ -0,0 +1,35 @@
+{%macro upload_progress_indicator()%}
+<div id="upload-progress-indicator" class="modal fade" tabindex="-1" role="dialog">
+ <div class="modal-dialog" role="document">
+ <div class="modal-content">
+ <div class="modal-header">
+ <h3 class="modal-title">Uploading file</h3>
+ </div>
+
+ <div class="modal-body">
+ <form id="frm-cancel-upload" style="border-style: none;">
+ <div class="form-group">
+ <span id="progress-filename" class="form-text">No file selected!</span>
+ <progress id="progress-bar" value="0" max="100" class="form-control">
+ 0</progress>
+ </div>
+
+ <div class="form-group">
+ <span class="form-text text-muted" id="progress-text">
+ Uploading 0%</span>
+ <span class="form-text text-muted" id="progress-extra-text">
+ Processing</span>
+ </div>
+ </form>
+ </div>
+
+ <div class="modal-footer">
+ <button id="btn-cancel-upload"
+ type="button"
+ class="btn btn-danger"
+ data-dismiss="modal">Cancel</button>
+ </div>
+ </div>
+ </div>
+</div>
+{%endmacro%}
diff --git a/uploader/templates/worker_failure.html b/uploader/templates/worker_failure.html
new file mode 100644
index 0000000..b65b140
--- /dev/null
+++ b/uploader/templates/worker_failure.html
@@ -0,0 +1,24 @@
+{%extends "base.html"%}
+
+{%block title%}Worker Failure{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Worker Failure</h1>
+
+<p>
+ There was a critical failure launching the job to parse your file.
+ This is our fault and (probably) has nothing to do with the file you uploaded.
+</p>
+
+<p>
+ Please notify the developers of this issue when you encounter it,
+ providing the link to this page, or the information below.
+</p>
+
+<h4>Debugging Information</h4>
+
+<ul>
+ <li><strong>job id</strong>: {{job_id}}</li>
+</ul>
+
+{%endblock%}
diff --git a/uploader/upload/__init__.py b/uploader/upload/__init__.py
new file mode 100644
index 0000000..5f120d4
--- /dev/null
+++ b/uploader/upload/__init__.py
@@ -0,0 +1,7 @@
+"""Package handling upload of files."""
+from flask import Blueprint
+
+from .rqtl2 import rqtl2
+
+upload = Blueprint("upload", __name__)
+upload.register_blueprint(rqtl2, url_prefix="/rqtl2")
diff --git a/uploader/upload/rqtl2.py b/uploader/upload/rqtl2.py
new file mode 100644
index 0000000..6aed1f7
--- /dev/null
+++ b/uploader/upload/rqtl2.py
@@ -0,0 +1,1157 @@
+"""Module to handle uploading of R/qtl2 bundles."""#pylint: disable=[too-many-lines]
+import sys
+import json
+import traceback
+from pathlib import Path
+from datetime import date
+from uuid import UUID, uuid4
+from functools import partial
+from zipfile import ZipFile, is_zipfile
+from typing import Union, Callable, Optional
+
+import MySQLdb as mdb
+from redis import Redis
+from MySQLdb.cursors import DictCursor
+from werkzeug.utils import secure_filename
+from flask import (
+ flash,
+ escape,
+ request,
+ jsonify,
+ url_for,
+ redirect,
+ Response,
+ Blueprint,
+ render_template,
+ current_app as app)
+
+from r_qtl import r_qtl2
+
+from uploader import jobs
+from uploader.files import save_file, fullpath
+from uploader.dbinsert import species as all_species
+from uploader.db_utils import with_db_connection, database_connection
+
+from uploader.db.platforms import platform_by_id, platforms_by_species
+from uploader.db.averaging import averaging_methods, averaging_method_by_id
+from uploader.db.tissues import all_tissues, tissue_by_id, create_new_tissue
+from uploader.db import (
+ species_by_id,
+ save_population,
+ populations_by_species,
+ population_by_species_and_id,)
+from uploader.db.datasets import (
+ geno_dataset_by_id,
+ geno_datasets_by_species_and_population,
+
+ probeset_study_by_id,
+ probeset_create_study,
+ probeset_dataset_by_id,
+ probeset_create_dataset,
+ probeset_datasets_by_study,
+ probeset_studies_by_species_and_population)
+
+rqtl2 = Blueprint("rqtl2", __name__)
+
+@rqtl2.route("/", methods=["GET", "POST"])
+@rqtl2.route("/select-species", methods=["GET", "POST"])
+def select_species():
+ """Select the species."""
+ if request.method == "GET":
+ return render_template("rqtl2/index.html", species=with_db_connection(all_species))
+
+ species_id = request.form.get("species_id")
+ species = with_db_connection(
+ lambda conn: species_by_id(conn, species_id))
+ if bool(species):
+ return redirect(url_for(
+ "upload.rqtl2.select_population", species_id=species_id))
+ flash("Invalid species or no species selected!", "alert-error error-rqtl2")
+ return redirect(url_for("upload.rqtl2.select_species"))
+
+
+@rqtl2.route("/upload/species/<int:species_id>/select-population",
+ methods=["GET", "POST"])
+def select_population(species_id: int):
+ """Select/Create the population to organise data under."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ if not bool(species):
+ flash("Invalid species selected!", "alert-error error-rqtl2")
+ return redirect(url_for("upload.rqtl2.select_species"))
+
+ if request.method == "GET":
+ return render_template(
+ "rqtl2/select-population.html",
+ species=species,
+ populations=populations_by_species(conn, species_id))
+
+ population = population_by_species_and_id(
+ conn, species["SpeciesId"], request.form.get("inbredset_id"))
+ if not bool(population):
+ flash("Invalid Population!", "alert-error error-rqtl2")
+ return redirect(
+ url_for("upload.rqtl2.select_population", pgsrc="error"),
+ code=307)
+
+ return redirect(url_for("upload.rqtl2.upload_rqtl2_bundle",
+ species_id=species["SpeciesId"],
+ population_id=population["InbredSetId"]))
+
+
+@rqtl2.route("/upload/species/<int:species_id>/create-population",
+ methods=["POST"])
+def create_population(species_id: int):
+ """Create a new population for the given species."""
+ population_page = redirect(url_for("upload.rqtl2.select_population",
+ species_id=species_id))
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ population_name = request.form.get("inbredset_name", "").strip()
+ population_fullname = request.form.get("inbredset_fullname", "").strip()
+ if not bool(species):
+ flash("Invalid species!", "alert-error error-rqtl2")
+ return redirect(url_for("upload.rqtl2.select_species"))
+ if not bool(population_name):
+ flash("Invalid Population Name!", "alert-error error-rqtl2")
+ return population_page
+ if not bool(population_fullname):
+ flash("Invalid Population Full Name!", "alert-error error-rqtl2")
+ return population_page
+ new_population = save_population(conn, {
+ "SpeciesId": species["SpeciesId"],
+ "Name": population_name,
+ "InbredSetName": population_fullname,
+ "FullName": population_fullname,
+ "Family": request.form.get("inbredset_family") or None,
+ "Description": request.form.get("description") or None
+ })
+
+ flash("Population created successfully.", "alert-success")
+ return redirect(
+ url_for("upload.rqtl2.upload_rqtl2_bundle",
+ species_id=species_id,
+ population_id=new_population["population_id"],
+ pgsrc="create-population"),
+ code=307)
+
+
+class __RequestError__(Exception): #pylint: disable=[invalid-name]
+ """Internal class to avoid pylint's `too-many-return-statements` error."""
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle"),
+ methods=["GET", "POST"])
+def upload_rqtl2_bundle(species_id: int, population_id: int):
+ """Allow upload of R/qtl2 bundle."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ species = species_by_id(conn, species_id)
+ population = population_by_species_and_id(
+ conn, species["SpeciesId"], population_id)
+ if not bool(species):
+ flash("Invalid species!", "alert-error error-rqtl2")
+ return redirect(url_for("upload.rqtl2.select_species"))
+ if not bool(population):
+ flash("Invalid Population!", "alert-error error-rqtl2")
+ return redirect(
+ url_for("upload.rqtl2.select_population", pgsrc="error"),
+ code=307)
+ if request.method == "GET" or (
+ request.method == "POST"
+ and bool(request.args.get("pgsrc"))):
+ return render_template("rqtl2/upload-rqtl2-bundle-step-01.html",
+ species=species,
+ population=population)
+
+ try:
+ app.logger.debug("Files in the form: %s", request.files)
+ the_file = save_file(request.files["rqtl2_bundle_file"],
+ Path(app.config["UPLOAD_FOLDER"]))
+ except AssertionError:
+ app.logger.debug(traceback.format_exc())
+ flash("Please provide a valid R/qtl2 zip bundle.",
+ "alert-error error-rqtl2")
+ return redirect(url_for("upload.rqtl2.upload_rqtl2_bundle",
+ species_id=species_id,
+ population_id=population_id))
+
+ if not is_zipfile(str(the_file)):
+ app.logger.debug("The file is not a zip file.")
+ raise __RequestError__("Invalid file! Expected a zip file.")
+
+ jobid = trigger_rqtl2_bundle_qc(
+ species_id,
+ population_id,
+ the_file,
+ request.files["rqtl2_bundle_file"].filename)#type: ignore[arg-type]
+ return redirect(url_for(
+ "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid))
+
+
+def trigger_rqtl2_bundle_qc(
+ species_id: int,
+ population_id: int,
+ rqtl2bundle: Path,
+ originalfilename: str
+) -> UUID:
+ """Trigger QC on the R/qtl2 bundle."""
+ redisuri = app.config["REDIS_URL"]
+ with Redis.from_url(redisuri, decode_responses=True) as rconn:
+ jobid = uuid4()
+ redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"]
+ jobs.launch_job(
+ jobs.initialise_job(
+ rconn,
+ jobs.jobsnamespace(),
+ str(jobid),
+ [sys.executable, "-m", "scripts.qc_on_rqtl2_bundle",
+ app.config["SQL_URI"], app.config["REDIS_URL"],
+ jobs.jobsnamespace(), str(jobid), str(species_id),
+ str(population_id), "--redisexpiry",
+ str(redis_ttl_seconds)],
+ "rqtl2-bundle-qc-job",
+ redis_ttl_seconds,
+ {"job-metadata": json.dumps({
+ "speciesid": species_id,
+ "populationid": population_id,
+ "rqtl2-bundle-file": str(rqtl2bundle.absolute()),
+ "original-filename": originalfilename})}),
+ redisuri,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors")
+ return jobid
+
+
+def chunk_name(uploadfilename: str, chunkno: int) -> str:
+ """Generate chunk name from original filename and chunk number"""
+ if uploadfilename == "":
+ raise ValueError("Name cannot be empty!")
+ if chunkno < 1:
+ raise ValueError("Chunk number must be greater than zero")
+ return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}"
+
+
+def chunks_directory(uniqueidentifier: str) -> Path:
+ """Compute the directory where chunks are temporarily stored."""
+ if uniqueidentifier == "":
+ raise ValueError("Unique identifier cannot be empty!")
+ return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}")
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle-chunked"),
+ methods=["GET"])
+def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"]
+ species_id: int,
+ population_id: int
+):
+ """
+ Extension to the `upload_rqtl2_bundle` endpoint above that provides a way
+ for testing whether all the chunks have been uploaded and to assist with
+ resuming a failed upload.
+ """
+ fileid = request.args.get("resumableIdentifier", type=str) or ""
+ filename = request.args.get("resumableFilename", type=str) or ""
+ chunk = request.args.get("resumableChunkNumber", type=int) or 0
+ if not(fileid or filename or chunk):
+ return jsonify({
+ "message": "At least one required query parameter is missing.",
+ "error": "BadRequest",
+ "statuscode": 400
+ }), 400
+
+ if Path(chunks_directory(fileid),
+ chunk_name(filename, chunk)).exists():
+ return "OK"
+
+ return jsonify({
+ "message": f"Chunk {chunk} was not found.",
+ "error": "NotFound",
+ "statuscode": 404
+ }), 404
+
+
+def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path:
+ """Merge the chunks into a single file."""
+ with open(targetfile, "ab") as _target:
+ for chunkfile in chunkpaths:
+ with open(chunkfile, "rb") as _chunkdata:
+ _target.write(_chunkdata.read())
+
+ chunkfile.unlink()
+ return targetfile
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle-chunked"),
+ methods=["POST"])
+def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int):
+ """
+ Extension to the `upload_rqtl2_bundle` endpoint above that allows large
+ files to be uploaded in chunks.
+
+ This should hopefully speed up uploads, and if done right, even enable
+ resumable uploads
+ """
+ _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0
+ _chunk = request.form.get("resumableChunkNumber", default=1, type=int)
+ _uploadfilename = request.form.get(
+ "resumableFilename", default="", type=str) or ""
+ _fileid = request.form.get(
+ "resumableIdentifier", default="", type=str) or ""
+ _targetfile = Path(app.config["UPLOAD_FOLDER"], _fileid)
+
+ if _targetfile.exists():
+ return jsonify({
+ "message": (
+ "A file with a similar unique identifier has previously been "
+ "uploaded and possibly is/has being/been processed."),
+ "error": "BadRequest",
+ "statuscode": 400
+ }), 400
+
+ try:
+ # save chunk data
+ chunks_directory(_fileid).mkdir(exist_ok=True, parents=True)
+ request.files["file"].save(Path(chunks_directory(_fileid),
+ chunk_name(_uploadfilename, _chunk)))
+
+ # Check whether upload is complete
+ chunkpaths = tuple(
+ Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk))
+ for _achunk in range(1, _totalchunks+1))
+ if all(_file.exists() for _file in chunkpaths):
+ # merge_files and clean up chunks
+ __merge_chunks__(_targetfile, chunkpaths)
+ chunks_directory(_fileid).rmdir()
+ jobid = trigger_rqtl2_bundle_qc(
+ species_id, population_id, _targetfile, _uploadfilename)
+ return url_for(
+ "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid)
+ except Exception as exc:# pylint: disable=[broad-except]
+ msg = "Error processing uploaded file chunks."
+ app.logger.error(msg, exc_info=True, stack_info=True)
+ return jsonify({
+ "message": msg,
+ "error": type(exc).__name__,
+ "error-description": " ".join(str(arg) for arg in exc.args),
+ "error-trace": traceback.format_exception(exc)
+ }), 500
+
+ return "OK"
+
+
+@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>",
+ methods=["GET", "POST"])
+def rqtl2_bundle_qc_status(jobid: UUID):
+ """Check the status of the QC jobs."""
+ with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn,
+ database_connection(app.config["SQL_URI"]) as dbconn):
+ try:
+ thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid)
+ messagelistname = thejob.get("log-messagelist")
+ logmessages = (rconn.lrange(messagelistname, 0, -1)
+ if bool(messagelistname) else [])
+ jobstatus = thejob["status"]
+ if jobstatus == "error":
+ return render_template("rqtl2/rqtl2-qc-job-error.html",
+ job=thejob,
+ errorsgeneric=json.loads(
+ thejob.get("errors-generic", "[]")),
+ errorsgeno=json.loads(
+ thejob.get("errors-geno", "[]")),
+ errorspheno=json.loads(
+ thejob.get("errors-pheno", "[]")),
+ errorsphenose=json.loads(
+ thejob.get("errors-phenose", "[]")),
+ errorsphenocovar=json.loads(
+ thejob.get("errors-phenocovar", "[]")),
+ messages=logmessages)
+ if jobstatus == "success":
+ jobmeta = json.loads(thejob["job-metadata"])
+ species = species_by_id(dbconn, jobmeta["speciesid"])
+ return render_template(
+ "rqtl2/rqtl2-qc-job-results.html",
+ species=species,
+ population=population_by_species_and_id(
+ dbconn, species["SpeciesId"], jobmeta["populationid"]),
+ rqtl2bundle=Path(jobmeta["rqtl2-bundle-file"]).name,
+ rqtl2bundleorig=jobmeta["original-filename"])
+
+ def compute_percentage(thejob, filetype) -> Union[str, None]:
+ if f"{filetype}-linecount" in thejob:
+ return "100"
+ if f"{filetype}-filesize" in thejob:
+ percent = ((int(thejob.get(f"{filetype}-checked", 0))
+ /
+ int(thejob.get(f"{filetype}-filesize", 1)))
+ * 100)
+ return f"{percent:.2f}"
+ return None
+
+ return render_template(
+ "rqtl2/rqtl2-qc-job-status.html",
+ job=thejob,
+ geno_percent=compute_percentage(thejob, "geno"),
+ pheno_percent=compute_percentage(thejob, "pheno"),
+ phenose_percent=compute_percentage(thejob, "phenose"),
+ messages=logmessages)
+ except jobs.JobNotFound:
+ return render_template("rqtl2/no-such-job.html", jobid=jobid)
+
+
+def redirect_on_error(flaskroute, **kwargs):
+ """Utility to redirect on error"""
+ return redirect(url_for(flaskroute, **kwargs, pgsrc="error"),
+ code=(307 if request.method == "POST" else 302))
+
+
+def check_species(conn: mdb.Connection, formargs: dict) -> Optional[
+ tuple[str, Response]]:
+ """
+ Check whether the 'species_id' value is provided, and whether a
+ corresponding species exists in the database.
+
+ Maybe give the function a better name..."""
+ speciespage = redirect_on_error("upload.rqtl2.select_species")
+ if "species_id" not in formargs:
+ return "You MUST provide the Species identifier.", speciespage
+
+ if not bool(species_by_id(conn, formargs["species_id"])):
+ return "No species with the provided identifier exists.", speciespage
+
+ return None
+
+
+def check_population(conn: mdb.Connection,
+ formargs: dict,
+ species_id) -> Optional[tuple[str, Response]]:
+ """
+ Check whether the 'population_id' value is provided, and whether a
+ corresponding population exists in the database.
+
+ Maybe give the function a better name..."""
+ poppage = redirect_on_error(
+ "upload.rqtl2.select_species", species_id=species_id)
+ if "population_id" not in formargs:
+ return "You MUST provide the Population identifier.", poppage
+
+ if not bool(population_by_species_and_id(
+ conn, species_id, formargs["population_id"])):
+ return "No population with the provided identifier exists.", poppage
+
+ return None
+
+
+def check_r_qtl2_bundle(formargs: dict,
+ species_id,
+ population_id) -> Optional[tuple[str, Response]]:
+ """Check for the existence of the R/qtl2 bundle."""
+ fileuploadpage = redirect_on_error("upload.rqtl2.upload_rqtl2_bundle",
+ species_id=species_id,
+ population_id=population_id)
+ if not "rqtl2_bundle_file" in formargs:
+ return (
+ "You MUST provide a R/qtl2 zip bundle for upload.", fileuploadpage)
+
+ if not Path(fullpath(formargs["rqtl2_bundle_file"])).exists():
+ return "No R/qtl2 bundle with the given name exists.", fileuploadpage
+
+ return None
+
+
+def check_geno_dataset(conn: mdb.Connection,
+ formargs: dict,
+ species_id,
+ population_id) -> Optional[tuple[str, Response]]:
+ """Check for the Genotype dataset."""
+ genodsetpg = redirect_on_error("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id)
+ if not bool(formargs.get("geno-dataset-id")):
+ return (
+ "You MUST provide a valid Genotype dataset identifier", genodsetpg)
+
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s",
+ (formargs["geno-dataset-id"],))
+ results = cursor.fetchall()
+ if not bool(results):
+ return ("No genotype dataset with the provided identifier exists.",
+ genodsetpg)
+ if len(results) > 1:
+ return (
+ "Data corruption: More than one genotype dataset with the same "
+ "identifier.",
+ genodsetpg)
+
+ return None
+
+def check_tissue(
+ conn: mdb.Connection,formargs: dict) -> Optional[tuple[str, Response]]:
+ """Check for tissue/organ/biological material."""
+ selectdsetpg = redirect_on_error("upload.rqtl2.select_dataset_info",
+ species_id=formargs["species_id"],
+ population_id=formargs["population_id"])
+ if not bool(formargs.get("tissueid", "").strip()):
+ return ("No tissue/organ/biological material provided.", selectdsetpg)
+
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM Tissue WHERE Id=%s",
+ (formargs["tissueid"],))
+ results = cursor.fetchall()
+ if not bool(results):
+ return ("No tissue/organ with the provided identifier exists.",
+ selectdsetpg)
+
+ if len(results) > 1:
+ return (
+ "Data corruption: More than one tissue/organ with the same "
+ "identifier.",
+ selectdsetpg)
+
+ return None
+
+
+def check_probe_study(conn: mdb.Connection,
+ formargs: dict,
+ species_id,
+ population_id) -> Optional[tuple[str, Response]]:
+ """Check for the ProbeSet study."""
+ dsetinfopg = redirect_on_error("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id)
+ if not bool(formargs.get("probe-study-id")):
+ return "No probeset study was selected!", dsetinfopg
+
+ if not bool(probeset_study_by_id(conn, formargs["probe-study-id"])):
+ return ("No probeset study with the provided identifier exists",
+ dsetinfopg)
+
+ return None
+
+
+def check_probe_dataset(conn: mdb.Connection,
+ formargs: dict,
+ species_id,
+ population_id) -> Optional[tuple[str, Response]]:
+ """Check for the ProbeSet dataset."""
+ dsetinfopg = redirect_on_error("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id)
+ if not bool(formargs.get("probe-dataset-id")):
+ return "No probeset dataset was selected!", dsetinfopg
+
+ if not bool(probeset_dataset_by_id(conn, formargs["probe-dataset-id"])):
+ return ("No probeset dataset with the provided identifier exists",
+ dsetinfopg)
+
+ return None
+
+
+def with_errors(endpointthunk: Callable, *checkfns):
+ """Run 'endpointthunk' with error checking."""
+ formargs = {**dict(request.args), **dict(request.form)}
+ errors = tuple(item for item in (_fn(formargs=formargs) for _fn in checkfns)
+ if item is not None)
+ if len(errors) > 0:
+ flash(errors[0][0], "alert-error error-rqtl2")
+ return errors[0][1]
+
+ return endpointthunk()
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/select-geno-dataset"),
+ methods=["POST"])
+def select_geno_dataset(species_id: int, population_id: int):
+ """Select from existing geno datasets."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ geno_dset = geno_datasets_by_species_and_population(
+ conn, species_id, population_id)
+ if not bool(geno_dset):
+ flash("No genotype dataset was provided!",
+ "alert-error error-rqtl2")
+ return redirect(url_for("upload.rqtl2.select_geno_dataset",
+ species_id=species_id,
+ population_id=population_id,
+ pgsrc="error"),
+ code=307)
+
+ flash("Genotype accepted", "alert-success error-rqtl2")
+ return redirect(url_for("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id,
+ pgsrc="upload.rqtl2.select_geno_dataset"),
+ code=307)
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population, conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/create-geno-dataset"),
+ methods=["POST"])
+def create_geno_dataset(species_id: int, population_id: int):
+ """Create a new geno dataset."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ sgeno_page = redirect(url_for("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id,
+ pgsrc="error"),
+ code=307)
+ errorclasses = "alert-error error-rqtl2 error-rqtl2-create-geno-dataset"
+ if not bool(request.form.get("dataset-name")):
+ flash("You must provide the dataset name", errorclasses)
+ return sgeno_page
+ if not bool(request.form.get("dataset-fullname")):
+ flash("You must provide the dataset full name", errorclasses)
+ return sgeno_page
+ public = 2 if request.form.get("dataset-public") == "on" else 0
+
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ datasetname = request.form["dataset-name"]
+ new_dataset = {
+ "name": datasetname,
+ "fname": request.form.get("dataset-fullname"),
+ "sname": request.form.get("dataset-shortname") or datasetname,
+ "today": date.today().isoformat(),
+ "pub": public,
+ "isetid": population_id
+ }
+ cursor.execute("SELECT * FROM GenoFreeze WHERE Name=%s",
+ (datasetname,))
+ results = cursor.fetchall()
+ if bool(results):
+ flash(
+ f"A genotype dataset with name '{escape(datasetname)}' "
+ "already exists.",
+ errorclasses)
+ return redirect(url_for("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id,
+ pgsrc="error"),
+ code=307)
+ cursor.execute(
+ "INSERT INTO GenoFreeze("
+ "Name, FullName, ShortName, CreateTime, public, InbredSetId"
+ ") "
+ "VALUES("
+ "%(name)s, %(fname)s, %(sname)s, %(today)s, %(pub)s, %(isetid)s"
+ ")",
+ new_dataset)
+ flash("Created dataset successfully.", "alert-success")
+ return render_template(
+ "rqtl2/create-geno-dataset-success.html",
+ species=species_by_id(conn, species_id),
+ population=population_by_species_and_id(
+ conn, species_id, population_id),
+ rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
+ geno_dataset={**new_dataset, "id": cursor.lastrowid})
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population, conn=conn, species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/select-tissue"),
+ methods=["POST"])
+def select_tissue(species_id: int, population_id: int):
+ """Select from existing tissues."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ if not bool(request.form.get("tissueid", "").strip()):
+ flash("Invalid tissue selection!",
+ "alert-error error-select-tissue error-rqtl2")
+
+ return redirect(url_for("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id,
+ pgsrc="upload.rqtl2.select_geno_dataset"),
+ code=307)
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/create-tissue"),
+ methods=["POST"])
+def create_tissue(species_id: int, population_id: int):
+ """Add new tissue, organ or biological material to the system."""
+ form = request.form
+ datasetinfopage = redirect(
+ url_for("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id,
+ pgsrc="upload.rqtl2.select_geno_dataset"),
+ code=307)
+ with database_connection(app.config["SQL_URI"]) as conn:
+ tissuename = form.get("tissuename", "").strip()
+ tissueshortname = form.get("tissueshortname", "").strip()
+ if not bool(tissuename):
+ flash("Organ/Tissue name MUST be provided.",
+ "alert-error error-create-tissue error-rqtl2")
+ return datasetinfopage
+
+ if not bool(tissueshortname):
+ flash("Organ/Tissue short name MUST be provided.",
+ "alert-error error-create-tissue error-rqtl2")
+ return datasetinfopage
+
+ try:
+ tissue = create_new_tissue(conn, tissuename, tissueshortname)
+ flash("Tissue created successfully!", "alert-success")
+ return render_template(
+ "rqtl2/create-tissue-success.html",
+ species=species_by_id(conn, species_id),
+ population=population_by_species_and_id(
+ conn, species_id, population_id),
+ rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
+ geno_dataset=geno_dataset_by_id(
+ conn,
+ int(request.form["geno-dataset-id"])),
+ tissue=tissue)
+ except mdb.IntegrityError as _ierr:
+ flash("Tissue/Organ with that short name already exists!",
+ "alert-error error-create-tissue error-rqtl2")
+ return datasetinfopage
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/select-probeset-study"),
+ methods=["POST"])
+def select_probeset_study(species_id: int, population_id: int):
+ """Select or create a probeset study."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ summary_page = redirect(url_for("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id),
+ code=307)
+ if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))):
+ flash("Invalid study selected!", "alert-error error-rqtl2")
+ return summary_page
+
+ return summary_page
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_tissue, conn=conn),
+ partial(check_probe_study,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/select-probeset-dataset"),
+ methods=["POST"])
+def select_probeset_dataset(species_id: int, population_id: int):
+ """Select or create a probeset dataset."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ summary_page = redirect(url_for("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id),
+ code=307)
+ if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))):
+ flash("Invalid study selected!", "alert-error error-rqtl2")
+ return summary_page
+
+ return summary_page
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_tissue, conn=conn),
+ partial(check_probe_study,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_probe_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/create-probeset-study"),
+ methods=["POST"])
+def create_probeset_study(species_id: int, population_id: int):
+ """Create a new probeset study."""
+ errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-study"
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ form = request.form
+ dataset_info_page = redirect(
+ url_for("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id),
+ code=307)
+
+ if not (bool(form.get("platformid")) and
+ bool(platform_by_id(conn, int(form["platformid"])))):
+ flash("Invalid platform selected.", errorclasses)
+ return dataset_info_page
+
+ if not (bool(form.get("tissueid")) and
+ bool(tissue_by_id(conn, int(form["tissueid"])))):
+ flash("Invalid tissue selected.", errorclasses)
+ return dataset_info_page
+
+ studyname = form["studyname"]
+ try:
+ study = probeset_create_study(
+ conn, population_id, int(form["platformid"]), int(form["tissueid"]),
+ studyname, form.get("studyfullname") or "",
+ form.get("studyshortname") or "")
+ except mdb.IntegrityError as _ierr:
+ flash(f"ProbeSet study with name '{escape(studyname)}' already "
+ "exists.",
+ errorclasses)
+ return dataset_info_page
+ return render_template(
+ "rqtl2/create-probe-study-success.html",
+ species=species_by_id(conn, species_id),
+ population=population_by_species_and_id(
+ conn, species_id, population_id),
+ rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
+ geno_dataset=geno_dataset_by_id(
+ conn,
+ int(request.form["geno-dataset-id"])),
+ study=study)
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_tissue, conn=conn))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/create-probeset-dataset"),
+ methods=["POST"])
+def create_probeset_dataset(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements]
+ """Create a new probeset dataset."""
+ errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-dataset"
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():#pylint: disable=[too-many-return-statements]
+ form = request.form
+ summary_page = redirect(url_for("upload.rqtl2.select_dataset_info",
+ species_id=species_id,
+ population_id=population_id),
+ code=307)
+ if not bool(form.get("averageid")):
+ flash("Averaging method not selected!", errorclasses)
+ return summary_page
+ if not bool(form.get("datasetname")):
+ flash("Dataset name not provided!", errorclasses)
+ return summary_page
+ if not bool(form.get("datasetfullname")):
+ flash("Dataset full name not provided!", errorclasses)
+ return summary_page
+
+ tissue = tissue_by_id(conn, form.get("tissueid", "").strip())
+
+ study = probeset_study_by_id(conn, int(form["probe-study-id"]))
+ if not bool(study):
+ flash("Invalid ProbeSet study provided!", errorclasses)
+ return summary_page
+
+ avgmethod = averaging_method_by_id(conn, int(form["averageid"]))
+ if not bool(avgmethod):
+ flash("Invalid averaging method provided!", errorclasses)
+ return summary_page
+
+ try:
+ dset = probeset_create_dataset(conn,
+ int(form["probe-study-id"]),
+ int(form["averageid"]),
+ form["datasetname"],
+ form["datasetfullname"],
+ form["datasetshortname"],
+ form["datasetpublic"] == "on",
+ form.get(
+ "datasetdatascale", "log2"))
+ except mdb.IntegrityError as _ierr:
+ app.logger.debug("Possible integrity error: %s", traceback.format_exc())
+ flash(("IntegrityError: The data you provided has some errors: "
+ f"{_ierr.args}"),
+ errorclasses)
+ return summary_page
+ except Exception as _exc:# pylint: disable=[broad-except]
+ app.logger.debug("Error creating ProbeSet dataset: %s",
+ traceback.format_exc())
+ flash(("There was a problem creating your dataset. Please try "
+ "again."),
+ errorclasses)
+ return summary_page
+ return render_template(
+ "rqtl2/create-probe-dataset-success.html",
+ species=species_by_id(conn, species_id),
+ population=population_by_species_and_id(
+ conn, species_id, population_id),
+ rqtl2_bundle_file=request.form["rqtl2_bundle_file"],
+ geno_dataset=geno_dataset_by_id(
+ conn,
+ int(request.form["geno-dataset-id"])),
+ tissue=tissue,
+ study=study,
+ avgmethod=avgmethod,
+ dataset=dset)
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_tissue, conn=conn),
+ partial(check_probe_study,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/dataset-info"),
+ methods=["POST"])
+def select_dataset_info(species_id: int, population_id: int):
+ """
+ If `geno` files exist in the R/qtl2 bundle, prompt user to provide the
+ dataset the genotypes belong to.
+ """
+ form = request.form
+ with database_connection(app.config["SQL_URI"]) as conn:
+ def __thunk__():
+ species = species_by_id(conn, species_id)
+ population = population_by_species_and_id(
+ conn, species_id, population_id)
+ thefile = fullpath(form["rqtl2_bundle_file"])
+ with ZipFile(str(thefile), "r") as zfile:
+ cdata = r_qtl2.control_data(zfile)
+
+ geno_dataset = geno_dataset_by_id(
+ conn,form.get("geno-dataset-id", "").strip())
+ if "geno" in cdata and not bool(form.get("geno-dataset-id")):
+ return render_template(
+ "rqtl2/select-geno-dataset.html",
+ species=species,
+ population=population,
+ rqtl2_bundle_file=thefile.name,
+ datasets=geno_datasets_by_species_and_population(
+ conn, species_id, population_id))
+
+ tissue = tissue_by_id(conn, form.get("tissueid", "").strip())
+ if "pheno" in cdata and not bool(tissue):
+ return render_template(
+ "rqtl2/select-tissue.html",
+ species=species,
+ population=population,
+ rqtl2_bundle_file=thefile.name,
+ geno_dataset=geno_dataset,
+ studies=probeset_studies_by_species_and_population(
+ conn, species_id, population_id),
+ platforms=platforms_by_species(conn, species_id),
+ tissues=all_tissues(conn))
+
+ probeset_study = probeset_study_by_id(
+ conn, form.get("probe-study-id", "").strip())
+ if "pheno" in cdata and not bool(probeset_study):
+ return render_template(
+ "rqtl2/select-probeset-study-id.html",
+ species=species,
+ population=population,
+ rqtl2_bundle_file=thefile.name,
+ geno_dataset=geno_dataset,
+ studies=probeset_studies_by_species_and_population(
+ conn, species_id, population_id),
+ platforms=platforms_by_species(conn, species_id),
+ tissue=tissue)
+ probeset_study = probeset_study_by_id(
+ conn, int(form["probe-study-id"]))
+
+ probeset_dataset = probeset_dataset_by_id(
+ conn, form.get("probe-dataset-id", "").strip())
+ if "pheno" in cdata and not bool(probeset_dataset):
+ return render_template(
+ "rqtl2/select-probeset-dataset.html",
+ species=species,
+ population=population,
+ rqtl2_bundle_file=thefile.name,
+ geno_dataset=geno_dataset,
+ probe_study=probeset_study,
+ tissue=tissue,
+ datasets=probeset_datasets_by_study(
+ conn, int(form["probe-study-id"])),
+ avgmethods=averaging_methods(conn))
+
+ return render_template("rqtl2/summary-info.html",
+ species=species,
+ population=population,
+ rqtl2_bundle_file=thefile.name,
+ geno_dataset=geno_dataset,
+ tissue=tissue,
+ probe_study=probeset_study,
+ probe_dataset=probeset_dataset)
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
+ "/rqtl2-bundle/confirm-bundle-details"),
+ methods=["POST"])
+def confirm_bundle_details(species_id: int, population_id: int):
+ """Confirm the details and trigger R/qtl2 bundle processing..."""
+ redisuri = app.config["REDIS_URL"]
+ with (database_connection(app.config["SQL_URI"]) as conn,
+ Redis.from_url(redisuri, decode_responses=True) as rconn):
+ def __thunk__():
+ redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"]
+ jobid = str(uuid4())
+ _job = jobs.launch_job(
+ jobs.initialise_job(
+ rconn,
+ jobs.jobsnamespace(),
+ jobid,
+ [
+ sys.executable, "-m", "scripts.process_rqtl2_bundle",
+ app.config["SQL_URI"], app.config["REDIS_URL"],
+ jobs.jobsnamespace(), jobid, "--redisexpiry",
+ str(redis_ttl_seconds)],
+ "R/qtl2 Bundle Upload",
+ redis_ttl_seconds,
+ {
+ "bundle-metadata": json.dumps({
+ "speciesid": species_id,
+ "populationid": population_id,
+ "rqtl2-bundle-file": str(fullpath(
+ request.form["rqtl2_bundle_file"])),
+ "geno-dataset-id": request.form.get(
+ "geno-dataset-id", ""),
+ "probe-study-id": request.form.get(
+ "probe-study-id", ""),
+ "probe-dataset-id": request.form.get(
+ "probe-dataset-id", ""),
+ **({
+ "platformid": probeset_study_by_id(
+ conn,
+ int(request.form["probe-study-id"]))["ChipId"]
+ } if bool(request.form.get("probe-study-id")) else {})
+ })
+ }),
+ redisuri,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+ return redirect(url_for("upload.rqtl2.rqtl2_processing_status",
+ jobid=jobid))
+
+ return with_errors(__thunk__,
+ partial(check_species, conn=conn),
+ partial(check_population,
+ conn=conn,
+ species_id=species_id),
+ partial(check_r_qtl2_bundle,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_geno_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_probe_study,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id),
+ partial(check_probe_dataset,
+ conn=conn,
+ species_id=species_id,
+ population_id=population_id))
+
+
+@rqtl2.route("/status/<uuid:jobid>")
+def rqtl2_processing_status(jobid: UUID):
+ """Retrieve the status of the job processing the uploaded R/qtl2 bundle."""
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+ try:
+ thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid)
+
+ messagelistname = thejob.get("log-messagelist")
+ logmessages = (rconn.lrange(messagelistname, 0, -1)
+ if bool(messagelistname) else [])
+
+ if thejob["status"] == "error":
+ return render_template(
+ "rqtl2/rqtl2-job-error.html", job=thejob, messages=logmessages)
+ if thejob["status"] == "success":
+ return render_template("rqtl2/rqtl2-job-results.html",
+ job=thejob,
+ messages=logmessages)
+
+ return render_template(
+ "rqtl2/rqtl2-job-status.html", job=thejob, messages=logmessages)
+ except jobs.JobNotFound as _exc:
+ return render_template("rqtl2/no-such-job.html", jobid=jobid)