diff options
Diffstat (limited to 'uploader')
84 files changed, 6623 insertions, 0 deletions
diff --git a/uploader/__init__.py b/uploader/__init__.py new file mode 100644 index 0000000..3ee8aa0 --- /dev/null +++ b/uploader/__init__.py @@ -0,0 +1,48 @@ +"""The Quality-Control Web Application entry point""" +import os +import logging +from pathlib import Path + +from flask import Flask, request + +from .entry import entrybp +from .upload import upload +from .parse import parsebp +from .samples import samples +from .base_routes import base +from .dbinsert import dbinsertbp +from .errors import register_error_handlers + +def override_settings_with_envvars( + app: Flask, ignore: tuple[str, ...]=tuple()) -> None: + """Override settings in `app` with those in ENVVARS""" + for setting in (key for key in app.config if key not in ignore): + app.config[setting] = os.environ.get(setting) or app.config[setting] + + +def create_app(): + """The application factory""" + app = Flask(__name__) + app.config.from_pyfile( + Path(__file__).parent.joinpath("default_settings.py")) + if "QCAPP_CONF" in os.environ: + app.config.from_envvar("QCAPP_CONF") # Override defaults with instance path + + override_settings_with_envvars(app, ignore=tuple()) + + if "QCAPP_SECRETS" in os.environ: + app.config.from_envvar("QCAPP_SECRETS") + + # setup jinja2 symbols + app.jinja_env.globals.update(request_url=lambda : request.url) + + # setup blueprints + app.register_blueprint(base, url_prefix="/") + app.register_blueprint(entrybp, url_prefix="/") + app.register_blueprint(parsebp, url_prefix="/parse") + app.register_blueprint(upload, url_prefix="/upload") + app.register_blueprint(dbinsertbp, url_prefix="/dbinsert") + app.register_blueprint(samples, url_prefix="/samples") + + register_error_handlers(app) + return app diff --git a/uploader/base_routes.py b/uploader/base_routes.py new file mode 100644 index 0000000..9daf439 --- /dev/null +++ b/uploader/base_routes.py @@ -0,0 +1,29 @@ +"""Basic routes required for all pages""" +import os +from flask import Blueprint, send_from_directory + +base = Blueprint("base", __name__) + +def appenv(): + """Get app's guix environment path.""" + return os.environ.get("GN_UPLOADER_ENVIRONMENT") + +@base.route("/bootstrap/<path:filename>") +def bootstrap(filename): + """Fetch bootstrap files.""" + return send_from_directory( + appenv(), f"share/genenetwork2/javascript/bootstrap/{filename}") + + +@base.route("/jquery/<path:filename>") +def jquery(filename): + """Fetch jquery files.""" + return send_from_directory( + appenv(), f"share/genenetwork2/javascript/jquery/{filename}") + + +@base.route("/node-modules/<path:filename>") +def node_modules(filename): + """Fetch node-js modules.""" + return send_from_directory( + appenv(), f"lib/node_modules/{filename}") diff --git a/uploader/check_connections.py b/uploader/check_connections.py new file mode 100644 index 0000000..2561e55 --- /dev/null +++ b/uploader/check_connections.py @@ -0,0 +1,28 @@ +"""Check the various connection used in the application""" +import sys +import traceback + +import redis +import MySQLdb + +from uploader.db_utils import database_connection + +def check_redis(uri: str): + "Check the redis connection" + try: + with redis.Redis.from_url(uri) as rconn: + rconn.ping() + except redis.exceptions.ConnectionError as conn_err: + print(conn_err, file=sys.stderr) + print(traceback.format_exc(), file=sys.stderr) + sys.exit(1) + +def check_db(uri: str): + "Check the mysql connection" + try: + with database_connection(uri) as dbconn: # pylint: disable=[unused-variable] + pass + except MySQLdb.OperationalError as op_err: + print(op_err, file=sys.stderr) + print(traceback.format_exc(), file=sys.stderr) + sys.exit(1) diff --git a/uploader/db/__init__.py b/uploader/db/__init__.py new file mode 100644 index 0000000..36e93e8 --- /dev/null +++ b/uploader/db/__init__.py @@ -0,0 +1,8 @@ +"""Database functions""" +from .species import species, species_by_id +from .populations import ( + save_population, + population_by_id, + populations_by_species, + population_by_species_and_id) +from .datasets import geno_datasets_by_species_and_population diff --git a/uploader/db/averaging.py b/uploader/db/averaging.py new file mode 100644 index 0000000..62bbe67 --- /dev/null +++ b/uploader/db/averaging.py @@ -0,0 +1,23 @@ +"""Functions for db interactions for averaging methods""" +from typing import Optional + +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +def averaging_methods(conn: mdb.Connection) -> tuple[dict, ...]: + """Fetch all available averaging methods""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM AvgMethod") + return tuple(dict(row) for row in cursor.fetchall()) + +def averaging_method_by_id( + conn: mdb.Connection, averageid: int) -> Optional[dict]: + """Fetch the averaging method by its ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM AvgMethod WHERE Id=%s", + (averageid,)) + result = cursor.fetchone() + if bool(result): + return dict(result) + + return None diff --git a/uploader/db/datasets.py b/uploader/db/datasets.py new file mode 100644 index 0000000..767ec41 --- /dev/null +++ b/uploader/db/datasets.py @@ -0,0 +1,133 @@ +"""Functions for accessing the database relating to datasets.""" +from datetime import date +from typing import Optional + +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +def geno_datasets_by_species_and_population( + conn: mdb.Connection, + speciesid: int, + populationid: int) -> tuple[dict, ...]: + """Retrieve all genotypes datasets by species and population""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT gf.* FROM InbredSet AS iset INNER JOIN GenoFreeze AS gf " + "ON iset.InbredSetId=gf.InbredSetId " + "WHERE iset.SpeciesId=%(sid)s AND iset.InbredSetId=%(pid)s", + {"sid": speciesid, "pid": populationid}) + return tuple(dict(row) for row in cursor.fetchall()) + +def geno_dataset_by_id(conn: mdb.Connection, dataset_id) -> Optional[dict]: + """Retrieve genotype dataset by ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s", (dataset_id,)) + _dataset = cursor.fetchone() + return dict(_dataset) if bool(_dataset) else None + +def probeset_studies_by_species_and_population( + conn: mdb.Connection, + speciesid: int, + populationid: int) -> tuple[dict, ...]: + """Retrieve all probesets""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT pf.* FROM InbredSet AS iset INNER JOIN ProbeFreeze AS pf " + "ON iset.InbredSetId=pf.InbredSetId " + "WHERE iset.SpeciesId=%(sid)s AND iset.InbredSetId=%(pid)s", + {"sid": speciesid, "pid": populationid}) + return tuple(dict(row) for row in cursor.fetchall()) + +def probeset_datasets_by_study(conn: mdb.Connection, + studyid: int) -> tuple[dict, ...]: + """Retrieve all probeset databases by study.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s", + (studyid,)) + return tuple(dict(row) for row in cursor.fetchall()) + +def probeset_study_by_id(conn: mdb.Connection, studyid) -> Optional[dict]: + """Retrieve ProbeSet study by ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM ProbeFreeze WHERE Id=%s", (studyid,)) + _study = cursor.fetchone() + return dict(_study) if bool(_study) else None + +def probeset_create_study(conn: mdb.Connection,#pylint: disable=[too-many-arguments] + populationid: int, + platformid: int, + tissueid: int, + studyname: str, + studyfullname: str = "", + studyshortname: str = ""): + """Create a new ProbeSet study.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + studydata = { + "platid": platformid, + "tissueid": tissueid, + "name": studyname, + "fname": studyfullname or studyname, + "sname": studyshortname, + "today": date.today().isoformat(), + "popid": populationid + } + cursor.execute( + """ + INSERT INTO ProbeFreeze( + ChipId, TissueId, Name, FullName, ShortName, CreateTime, + InbredSetId + ) VALUES ( + %(platid)s, %(tissueid)s, %(name)s, %(fname)s, %(sname)s, + %(today)s, %(popid)s + ) + """, + studydata) + studyid = cursor.lastrowid + cursor.execute("UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s", + (studyid, studyid)) + return {**studydata, "studyid": studyid} + +def probeset_create_dataset(conn: mdb.Connection,#pylint: disable=[too-many-arguments] + studyid: int, + averageid: int, + datasetname: str, + datasetfullname: str, + datasetshortname: str="", + public: bool = True, + datascale="log2") -> dict: + """Create a new ProbeSet dataset.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + dataset = { + "studyid": studyid, + "averageid": averageid, + "name2": datasetname, + "fname": datasetfullname, + "name": datasetshortname, + "sname": datasetshortname, + "today": date.today().isoformat(), + "public": 2 if public else 0, + "authorisedusers": "williamslab", + "datascale": datascale + } + cursor.execute( + """ + INSERT INTO ProbeSetFreeze( + ProbeFreezeId, AvgId, Name, Name2, FullName, ShortName, + CreateTime, public, AuthorisedUsers, DataScale) + VALUES( + %(studyid)s, %(averageid)s, %(name)s, %(name2)s, %(fname)s, + %(sname)s, %(today)s, %(public)s, %(authorisedusers)s, + %(datascale)s) + """, + dataset) + return {**dataset, "datasetid": cursor.lastrowid} + +def probeset_dataset_by_id(conn: mdb.Connection, datasetid) -> Optional[dict]: + """Fetch a ProbeSet dataset by its ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Id=%s", (datasetid,)) + result = cursor.fetchone() + if bool(result): + return dict(result) + + return None diff --git a/uploader/db/platforms.py b/uploader/db/platforms.py new file mode 100644 index 0000000..cb527a7 --- /dev/null +++ b/uploader/db/platforms.py @@ -0,0 +1,25 @@ +"""Handle db interactions for platforms.""" +from typing import Optional + +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +def platforms_by_species( + conn: mdb.Connection, speciesid: int) -> tuple[dict, ...]: + """Retrieve platforms by the species""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GeneChip WHERE SpeciesId=%s " + "ORDER BY GeneChipName ASC", + (speciesid,)) + return tuple(dict(row) for row in cursor.fetchall()) + +def platform_by_id(conn: mdb.Connection, platformid: int) -> Optional[dict]: + """Retrieve a platform by its ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GeneChip WHERE Id=%s", + (platformid,)) + result = cursor.fetchone() + if bool(result): + return dict(result) + + return None diff --git a/uploader/db/populations.py b/uploader/db/populations.py new file mode 100644 index 0000000..4485e52 --- /dev/null +++ b/uploader/db/populations.py @@ -0,0 +1,54 @@ +"""Functions for accessing the database relating to species populations.""" +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +def population_by_id(conn: mdb.Connection, population_id) -> dict: + """Get the grouping/population by id.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM InbredSet WHERE InbredSetId=%s", + (population_id,)) + return cursor.fetchone() + +def population_by_species_and_id( + conn: mdb.Connection, species_id, population_id) -> dict: + """Retrieve a population by its identifier and species.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM InbredSet WHERE SpeciesId=%s AND Id=%s", + (species_id, population_id)) + return cursor.fetchone() + +def populations_by_species(conn: mdb.Connection, speciesid) -> tuple: + "Retrieve group (InbredSet) information from the database." + with conn.cursor(cursorclass=DictCursor) as cursor: + query = "SELECT * FROM InbredSet WHERE SpeciesId=%s" + cursor.execute(query, (speciesid,)) + return tuple(cursor.fetchall()) + + return tuple() + +def save_population(conn: mdb.Connection, population_details: dict) -> dict: + """Save the population details to the db.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "INSERT INTO InbredSet(" + "InbredSetId, InbredSetName, Name, SpeciesId, FullName, " + "MenuOrderId, Description" + ") " + "VALUES (" + "%(InbredSetId)s, %(InbredSetName)s, %(Name)s, %(SpeciesId)s, " + "%(FullName)s, %(MenuOrderId)s, %(Description)s" + ")", + { + "MenuOrderId": 0, + "InbredSetId": 0, + **population_details + }) + new_id = cursor.lastrowid + cursor.execute("UPDATE InbredSet SET InbredSetId=%s WHERE Id=%s", + (new_id, new_id)) + return { + **population_details, + "Id": new_id, + "InbredSetId": new_id, + "population_id": new_id + } diff --git a/uploader/db/species.py b/uploader/db/species.py new file mode 100644 index 0000000..653e59b --- /dev/null +++ b/uploader/db/species.py @@ -0,0 +1,22 @@ +"""Database functions for species.""" +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +def species(conn: mdb.Connection) -> tuple: + "Retrieve the species from the database." + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT SpeciesId, SpeciesName, LOWER(Name) AS Name, MenuName, " + "FullName FROM Species") + return tuple(cursor.fetchall()) + + return tuple() + +def species_by_id(conn: mdb.Connection, speciesid) -> dict: + "Retrieve the species from the database by id." + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT SpeciesId, SpeciesName, LOWER(Name) AS Name, MenuName, " + "FullName FROM Species WHERE SpeciesId=%s", + (speciesid,)) + return cursor.fetchone() diff --git a/uploader/db/tissues.py b/uploader/db/tissues.py new file mode 100644 index 0000000..9fe7bab --- /dev/null +++ b/uploader/db/tissues.py @@ -0,0 +1,50 @@ +"""Handle db interactions for tissue.""" +from typing import Union, Optional + +import MySQLdb as mdb +from MySQLdb.cursors import DictCursor + +def all_tissues(conn: mdb.Connection) -> tuple[dict, ...]: + """All available tissue.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Tissue ORDER BY TissueName") + return tuple(dict(row) for row in cursor.fetchall()) + + +def tissue_by_id(conn: mdb.Connection, tissueid) -> Optional[dict]: + """Retrieve a tissue by its ID""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Tissue WHERE Id=%s", (tissueid,)) + result = cursor.fetchone() + if bool(result): + return dict(result) + + return None + + +def create_new_tissue( + conn: mdb.Connection, + name: str, + shortname: str, + birnlexid: Optional[str] = None, + birnlexname: Optional[str] = None +) -> dict[str, Union[int, str, None]]: + """Add a new tissue, organ or biological material to the database.""" + with conn.cursor() as cursor: + cursor.execute( + "INSERT INTO " + "Tissue(TissueName, Name, Short_Name, BIRN_lex_ID, BIRN_lex_Name) " + "VALUES (%s, %s, %s, %s, %s)", + (name, name, shortname, birnlexid, birnlexname)) + tissueid = cursor.lastrowid + cursor.execute("UPDATE Tissue SET TissueId=%s WHERE Id=%s", + (tissueid, tissueid)) + return { + "Id": tissueid, + "TissueId": tissueid, + "TissueName": name, + "Name": name, + "Short_Name": shortname, + "BIRN_lex_ID": birnlexid, + "BIRN_lex_Name": birnlexname + } diff --git a/uploader/db_utils.py b/uploader/db_utils.py new file mode 100644 index 0000000..ef26398 --- /dev/null +++ b/uploader/db_utils.py @@ -0,0 +1,46 @@ +"""module contains all db related stuff""" +import logging +import traceback +import contextlib +from urllib.parse import urlparse +from typing import Any, Tuple, Optional, Iterator, Callable + +import MySQLdb as mdb +from redis import Redis +from flask import current_app as app + +def parse_db_url(db_url) -> Tuple: + """ + Parse SQL_URI configuration variable. + """ + parsed_db = urlparse(db_url) + return (parsed_db.hostname, parsed_db.username, + parsed_db.password, parsed_db.path[1:], parsed_db.port) + + +@contextlib.contextmanager +def database_connection(db_url: Optional[str] = None) -> Iterator[mdb.Connection]: + """function to create db connector""" + host, user, passwd, db_name, db_port = parse_db_url( + db_url or app.config["SQL_URI"]) + connection = mdb.connect( + host, user, passwd, db_name, port=(db_port or 3306)) + try: + yield connection + connection.commit() + except mdb.Error as _mdb_err: + logging.error(traceback.format_exc()) + connection.rollback() + finally: + connection.close() + +def with_db_connection(func: Callable[[mdb.Connection], Any]) -> Any: + """Call `func` with a MySQDdb database connection.""" + with database_connection(app.config["SQL_URI"]) as conn: + return func(conn) + +def with_redis_connection(func: Callable[[Redis], Any]) -> Any: + """Call `func` with a redis connection.""" + redisuri = app.config["REDIS_URL"] + with Redis.from_url(redisuri, decode_responses=True) as rconn: + return func(rconn) diff --git a/uploader/dbinsert.py b/uploader/dbinsert.py new file mode 100644 index 0000000..88d16ef --- /dev/null +++ b/uploader/dbinsert.py @@ -0,0 +1,397 @@ +"Handle inserting data into the database" +import os +import json +from typing import Union +from functools import reduce +from datetime import datetime + +from redis import Redis +from MySQLdb.cursors import DictCursor +from flask import ( + flash, request, url_for, Blueprint, redirect, render_template, + current_app as app) + +from uploader.db_utils import with_db_connection, database_connection +from uploader.db import species, species_by_id, populations_by_species + +from . import jobs + +dbinsertbp = Blueprint("dbinsert", __name__) + +def render_error(error_msg): + "Render the generic error page" + return render_template("dbupdate_error.html", error_message=error_msg), 400 + +def make_menu_items_grouper(grouping_fn=lambda item: item): + "Build function to be used to group menu items." + def __grouper__(acc, row): + grouping = grouping_fn(row[2]) + row_values = (row[0].strip(), row[1].strip()) + if acc.get(grouping) is None: + return {**acc, grouping: (row_values,)} + return {**acc, grouping: (acc[grouping] + (row_values,))} + return __grouper__ + +def genechips(): + "Retrieve the genechip information from the database" + def __organise_by_species__(acc, chip): + speciesid = chip["SpeciesId"] + if acc.get(speciesid) is None: + return {**acc, speciesid: (chip,)} + return {**acc, speciesid: acc[speciesid] + (chip,)} + + with database_connection() as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GeneChip ORDER BY GeneChipName ASC") + return reduce(__organise_by_species__, cursor.fetchall(), {}) + + return {} + +def platform_by_id(genechipid:int) -> Union[dict, None]: + "Retrieve the gene platform by id" + with database_connection() as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT * FROM GeneChip WHERE GeneChipId=%s", + (genechipid,)) + return cursor.fetchone() + +def studies_by_species_and_platform(speciesid:int, genechipid:int) -> tuple: + "Retrieve the studies by the related species and gene platform" + with database_connection() as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + query = ( + "SELECT Species.SpeciesId, ProbeFreeze.* " + "FROM Species INNER JOIN InbredSet " + "ON Species.SpeciesId=InbredSet.SpeciesId " + "INNER JOIN ProbeFreeze " + "ON InbredSet.InbredSetId=ProbeFreeze.InbredSetId " + "WHERE Species.SpeciesId = %s " + "AND ProbeFreeze.ChipId = %s") + cursor.execute(query, (speciesid, genechipid)) + return tuple(cursor.fetchall()) + + return tuple() + +def organise_groups_by_family(acc:dict, group:dict) -> dict: + "Organise the group (InbredSet) information by the group field" + family = group["Family"] + if acc.get(family): + return {**acc, family: acc[family] + (group,)} + return {**acc, family: (group,)} + +def tissues() -> tuple: + "Retrieve type (Tissue) information from the database." + with database_connection() as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Tissue ORDER BY Name") + return tuple(cursor.fetchall()) + + return tuple() + +@dbinsertbp.route("/platform", methods=["POST"]) +def select_platform(): + "Select the platform (GeneChipId) used for the data." + job_id = request.form["job_id"] + with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, + database_connection(app.config["SQL_URI"]) as conn): + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + if job: + filename = job["filename"] + filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + if os.path.exists(filepath): + default_species = 1 + gchips = genechips() + return render_template( + "select_platform.html", filename=filename, + filetype=job["filetype"], totallines=int(job["currentline"]), + default_species=default_species, species=species(conn), + genechips=gchips[default_species], + genechips_data=json.dumps(gchips)) + return render_error(f"File '{filename}' no longer exists.") + return render_error(f"Job '{job_id}' no longer exists.") + return render_error("Unknown error") + +@dbinsertbp.route("/study", methods=["POST"]) +def select_study(): + "View to select/create the study (ProbeFreeze) associated with the data." + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + + speciesid = form["species"] + genechipid = form["genechipid"] + + the_studies = studies_by_species_and_platform(speciesid, genechipid) + the_groups = reduce( + organise_groups_by_family, + with_db_connection( + lambda conn: populations_by_species(conn, speciesid)), + {}) + return render_template( + "select_study.html", filename=form["filename"], + filetype=form["filetype"], totallines=form["totallines"], + species=speciesid, genechipid=genechipid, studies=the_studies, + groups=the_groups, tissues = tissues(), + selected_group=int(form.get("inbredsetid", -13)), + selected_tissue=int(form.get("tissueid", -13))) + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/create-study", methods=["POST"]) +def create_study(): + "Create a new study (ProbeFreeze)." + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyname"), "study name" + assert form.get("inbredsetid"), "group" + assert form.get("tissueid"), "type/tissue" + + with database_connection() as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + values = ( + form["genechipid"], + form["tissueid"], + form["studyname"], + form.get("studyfullname", ""), + form.get("studyshortname", ""), + datetime.now().date().strftime("%Y-%m-%d"), + form["inbredsetid"]) + query = ( + "INSERT INTO ProbeFreeze(" + "ChipId, TissueId, Name, FullName, ShortName, CreateTime, " + "InbredSetId" + ") VALUES (%s, %s, %s, %s, %s, %s, %s)") + cursor.execute(query, values) + new_studyid = cursor.lastrowid + cursor.execute( + "UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s", + (new_studyid, new_studyid)) + flash("Study created successfully", "alert-success") + return render_template( + "continue_from_create_study.html", + filename=form["filename"], filetype=form["filetype"], + totallines=form["totallines"], species=form["species"], + genechipid=form["genechipid"], studyid=new_studyid) + except AssertionError as aserr: + flash(f"Missing data: {aserr.args[0]}", "alert-error") + return redirect(url_for("dbinsert.select_study"), code=307) + +def datasets_by_study(studyid:int) -> tuple: + "Retrieve datasets associated with a study with the ID `studyid`." + with database_connection() as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + query = "SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s" + cursor.execute(query, (studyid,)) + return tuple(cursor.fetchall()) + + return tuple() + +def averaging_methods() -> tuple: + "Retrieve averaging methods from database" + with database_connection() as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM AvgMethod") + return tuple(cursor.fetchall()) + + return tuple() + +def dataset_datascales() -> tuple: + "Retrieve datascales from database" + with database_connection() as conn: + with conn.cursor() as cursor: + cursor.execute( + 'SELECT DISTINCT DataScale FROM ProbeSetFreeze ' + 'WHERE DataScale IS NOT NULL AND DataScale != ""') + return tuple( + item for item in + (res[0].strip() for res in cursor.fetchall()) + if (item is not None and item != "")) + + return tuple() + +@dbinsertbp.route("/dataset", methods=["POST"]) +def select_dataset(): + "Select the dataset to add the file contents against" + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + + studyid = form["studyid"] + datasets = datasets_by_study(studyid) + return render_template( + "select_dataset.html", **{**form, "studyid": studyid}, + datasets=datasets, avgmethods=averaging_methods(), + datascales=dataset_datascales()) + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/create-dataset", methods=["POST"]) +def create_dataset(): + "Select the dataset to add the file contents against" + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + assert form.get("avgid"), "averaging method" + assert form.get("datasetname2"), "Dataset Name 2" + assert form.get("datasetfullname"), "Dataset Full Name" + assert form.get("datasetshortname"), "Dataset Short Name" + assert form.get("datasetpublic"), "Dataset public specification" + assert form.get("datasetconfidentiality"), "Dataset confidentiality" + assert form.get("datasetdatascale"), "Dataset Datascale" + + with database_connection() as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + datasetname = form["datasetname"] + cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Name=%s", + (datasetname,)) + results = cursor.fetchall() + if bool(results): + flash("A dataset with that name already exists.", + "alert-error") + return redirect(url_for("dbinsert.select_dataset"), code=307) + values = ( + form["studyid"], form["avgid"], + datasetname, form["datasetname2"], + form["datasetfullname"], form["datasetshortname"], + datetime.now().date().strftime("%Y-%m-%d"), + form["datasetpublic"], form["datasetconfidentiality"], + "williamslab", form["datasetdatascale"]) + query = ( + "INSERT INTO ProbeSetFreeze(" + "ProbeFreezeId, AvgID, Name, Name2, FullName, " + "ShortName, CreateTime, OrderList, public, " + "confidentiality, AuthorisedUsers, DataScale) " + "VALUES" + "(%s, %s, %s, %s, %s, %s, %s, NULL, %s, %s, %s, %s)") + cursor.execute(query, values) + new_datasetid = cursor.lastrowid + return render_template( + "continue_from_create_dataset.html", + filename=form["filename"], filetype=form["filetype"], + species=form["species"], genechipid=form["genechipid"], + studyid=form["studyid"], datasetid=new_datasetid, + totallines=form["totallines"]) + except AssertionError as aserr: + flash(f"Missing data {aserr.args[0]}", "alert-error") + return redirect(url_for("dbinsert.select_dataset"), code=307) + +def study_by_id(studyid:int) -> Union[dict, None]: + "Get a study by its Id" + with database_connection() as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT * FROM ProbeFreeze WHERE Id=%s", + (studyid,)) + return cursor.fetchone() + +def dataset_by_id(datasetid:int) -> Union[dict, None]: + "Retrieve a dataset by its id" + with database_connection() as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + ("SELECT AvgMethod.Name AS AvgMethodName, ProbeSetFreeze.* " + "FROM ProbeSetFreeze INNER JOIN AvgMethod " + "ON ProbeSetFreeze.AvgId=AvgMethod.AvgMethodId " + "WHERE ProbeSetFreeze.Id=%s"), + (datasetid,)) + return cursor.fetchone() + +def selected_keys(original: dict, keys: tuple) -> dict: + "Return a new dict from the `original` dict with only `keys` present." + return {key: value for key,value in original.items() if key in keys} + +@dbinsertbp.route("/final-confirmation", methods=["POST"]) +def final_confirmation(): + "Preview the data before triggering entry into the database" + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + assert form.get("datasetid"), "dataset" + + speciesid = form["species"] + genechipid = form["genechipid"] + studyid = form["studyid"] + datasetid=form["datasetid"] + return render_template( + "final_confirmation.html", filename=form["filename"], + filetype=form["filetype"], totallines=form["totallines"], + species=speciesid, genechipid=genechipid, studyid=studyid, + datasetid=datasetid, the_species=selected_keys( + with_db_connection(lambda conn: species_by_id(conn, speciesid)), + ("SpeciesName", "Name", "MenuName")), + platform=selected_keys( + platform_by_id(genechipid), + ("GeneChipName", "Name", "GeoPlatform", "Title", "GO_tree_value")), + study=selected_keys( + study_by_id(studyid), ("Name", "FullName", "ShortName")), + dataset=selected_keys( + dataset_by_id(datasetid), + ("AvgMethodName", "Name", "Name2", "FullName", "ShortName", + "DataScale"))) + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/insert-data", methods=["POST"]) +def insert_data(): + "Trigger data insertion" + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + assert form.get("datasetid"), "dataset" + + filename = form["filename"] + filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + redisurl = app.config["REDIS_URL"] + if os.path.exists(filepath): + with Redis.from_url(redisurl, decode_responses=True) as rconn: + job = jobs.launch_job( + jobs.data_insertion_job( + rconn, filepath, form["filetype"], form["totallines"], + form["species"], form["genechipid"], form["datasetid"], + app.config["SQL_URI"], redisurl, + app.config["JOBS_TTL_SECONDS"]), + redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors") + + return redirect(url_for("dbinsert.insert_status", job_id=job["jobid"])) + return render_error(f"File '{filename}' no longer exists.") + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/status/<job_id>", methods=["GET"]) +def insert_status(job_id: str): + "Retrieve status of data insertion." + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + job_status = job["status"] + if job_status == "success": + return render_template("insert_success.html", job=job) + if job["status"] == "error": + return render_template("insert_error.html", job=job) + return render_template("insert_progress.html", job=job) + return render_template("no_such_job.html", job_id=job_id), 400 diff --git a/uploader/default_settings.py b/uploader/default_settings.py new file mode 100644 index 0000000..7a9da0f --- /dev/null +++ b/uploader/default_settings.py @@ -0,0 +1,14 @@ +""" +The default configuration file. The values here should be overridden in the +actual configuration file used for the production and staging systems. +""" + +import os + +LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") +SECRET_KEY = b"<Please! Please! Please! Change This!>" +UPLOAD_FOLDER = "/tmp/qc_app_files" +REDIS_URL = "redis://" +JOBS_TTL_SECONDS = 1209600 # 14 days +GNQC_REDIS_PREFIX="GNQC" +SQL_URI = "" diff --git a/uploader/entry.py b/uploader/entry.py new file mode 100644 index 0000000..4a02f1e --- /dev/null +++ b/uploader/entry.py @@ -0,0 +1,127 @@ +"""Entry-point module""" +import os +import mimetypes +from typing import Tuple +from zipfile import ZipFile, is_zipfile + +from werkzeug.utils import secure_filename +from flask import ( + flash, + request, + url_for, + redirect, + Blueprint, + render_template, + current_app as app, + send_from_directory) + +from uploader.db import species +from uploader.db_utils import with_db_connection + +entrybp = Blueprint("entry", __name__) + +@entrybp.route("/favicon.ico", methods=["GET"]) +def favicon(): + """Return the favicon.""" + return send_from_directory(os.path.join(app.root_path, "static"), + "images/CITGLogo.png", + mimetype="image/png") + + +def errors(rqst) -> Tuple[str, ...]: + """Return a tuple of the errors found in the request `rqst`. If no error is + found, then an empty tuple is returned.""" + def __filetype_error__(): + return ( + ("Invalid file type provided.",) + if rqst.form.get("filetype") not in ("average", "standard-error") + else tuple()) + + def __file_missing_error__(): + return ( + ("No file was uploaded.",) + if ("qc_text_file" not in rqst.files or + rqst.files["qc_text_file"].filename == "") + else tuple()) + + def __file_mimetype_error__(): + text_file = rqst.files["qc_text_file"] + return ( + ( + ("Invalid file! Expected a tab-separated-values file, or a zip " + "file of the a tab-separated-values file."),) + if text_file.mimetype not in ( + "text/plain", "text/tab-separated-values", + "application/zip") + else tuple()) + + return ( + __filetype_error__() + + (__file_missing_error__() or __file_mimetype_error__())) + +def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]: + """Check the uploaded zip file for errors.""" + zfile_errors: Tuple[str, ...] = tuple() + if is_zipfile(filepath): + with ZipFile(filepath, "r") as zfile: + infolist = zfile.infolist() + if len(infolist) != 1: + zfile_errors = zfile_errors + ( + ("Expected exactly one (1) member file within the uploaded zip " + f"file. Got {len(infolist)} member files."),) + if len(infolist) == 1 and infolist[0].is_dir(): + zfile_errors = zfile_errors + ( + ("Expected a member text file in the uploaded zip file. Got a " + "directory/folder."),) + + if len(infolist) == 1 and not infolist[0].is_dir(): + zfile.extract(infolist[0], path=upload_dir) + mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}") + if mime[0] != "text/tab-separated-values": + zfile_errors = zfile_errors + ( + ("Expected the member text file in the uploaded zip file to" + " be a tab-separated file."),) + + return zfile_errors + +@entrybp.route("/", methods=["GET"]) +def index(): + """Load the landing page""" + return render_template("index.html") + +@entrybp.route("/upload", methods=["GET", "POST"]) +def upload_file(): + """Enables uploading the files""" + if request.method == "GET": + return render_template( + "select_species.html", species=with_db_connection(species)) + + upload_dir = app.config["UPLOAD_FOLDER"] + request_errors = errors(request) + if request_errors: + for error in request_errors: + flash(error, "alert-danger error-expr-data") + return redirect(url_for("entry.upload_file")) + + filename = secure_filename(request.files["qc_text_file"].filename) + if not os.path.exists(upload_dir): + os.mkdir(upload_dir) + + filepath = os.path.join(upload_dir, filename) + request.files["qc_text_file"].save(os.path.join(upload_dir, filename)) + + zip_errors = zip_file_errors(filepath, upload_dir) + if zip_errors: + for error in zip_errors: + flash(error, "alert-danger error-expr-data") + return redirect(url_for("entry.upload_file")) + + return redirect(url_for("parse.parse", + speciesid=request.form["speciesid"], + filename=filename, + filetype=request.form["filetype"])) + +@entrybp.route("/data-review", methods=["GET"]) +def data_review(): + """Provide some help on data expectations to the user.""" + return render_template("data_review.html") diff --git a/uploader/errors.py b/uploader/errors.py new file mode 100644 index 0000000..3e7c893 --- /dev/null +++ b/uploader/errors.py @@ -0,0 +1,29 @@ +"""Application error handling.""" +import traceback +from werkzeug.exceptions import HTTPException + +import MySQLdb as mdb +from flask import Flask, request, render_template, current_app as app + +def handle_general_exception(exc: Exception): + """Handle generic exceptions.""" + trace = traceback.format_exc() + app.logger.error( + "Error (%s.%s): Generic unhandled exception!! (URI: %s)\n%s", + exc.__class__.__module__, exc.__class__.__name__, request.url, trace) + return render_template("unhandled_exception.html", trace=trace), 500 + +def handle_http_exception(exc: HTTPException): + """Handle HTTP exceptions.""" + app.logger.error( + "HTTP Error %s: %s", exc.code, exc.description, exc_info=True) + return render_template("http-error.html", + request_url=request.url, + exc=exc, + trace=traceback.format_exception(exc)), exc.code + +def register_error_handlers(appl: Flask): + """Register top-level error/exception handlers.""" + appl.register_error_handler(Exception, handle_general_exception) + appl.register_error_handler(HTTPException, handle_http_exception) + appl.register_error_handler(mdb.MySQLError, handle_general_exception) diff --git a/uploader/files.py b/uploader/files.py new file mode 100644 index 0000000..b163612 --- /dev/null +++ b/uploader/files.py @@ -0,0 +1,26 @@ +"""Utilities to deal with uploaded files.""" +import hashlib +from pathlib import Path +from datetime import datetime +from flask import current_app + +from werkzeug.utils import secure_filename +from werkzeug.datastructures import FileStorage + +def save_file(fileobj: FileStorage, upload_dir: Path) -> Path: + """Save the uploaded file and return the path.""" + assert bool(fileobj), "Invalid file object!" + hashed_name = hashlib.sha512( + f"{fileobj.filename}::{datetime.now().isoformat()}".encode("utf8") + ).hexdigest() + filename = Path(secure_filename(hashed_name)) # type: ignore[arg-type] + if not upload_dir.exists(): + upload_dir.mkdir() + + filepath = Path(upload_dir, filename) + fileobj.save(filepath) + return filepath + +def fullpath(filename: str): + """Get a file's full path. This makes use of `flask.current_app`.""" + return Path(current_app.config["UPLOAD_FOLDER"], filename).absolute() diff --git a/uploader/input_validation.py b/uploader/input_validation.py new file mode 100644 index 0000000..9abe742 --- /dev/null +++ b/uploader/input_validation.py @@ -0,0 +1,27 @@ +"""Input validation utilities""" +from typing import Any + +def is_empty_string(value: str) -> bool: + """Check whether as string is empty""" + return (isinstance(value, str) and value.strip() == "") + +def is_empty_input(value: Any) -> bool: + """Check whether user provided an empty value.""" + return (value is None or is_empty_string(value)) + +def is_integer_input(value: Any) -> bool: + """ + Check whether user provided a value that can be parsed into an integer. + """ + def __is_int__(val, base): + try: + int(val, base=base) + except ValueError: + return False + return True + return isinstance(value, int) or ( + (not is_empty_input(value)) and ( + isinstance(value, str) and ( + __is_int__(value, 10) + or __is_int__(value, 8) + or __is_int__(value, 16)))) diff --git a/uploader/jobs.py b/uploader/jobs.py new file mode 100644 index 0000000..21889da --- /dev/null +++ b/uploader/jobs.py @@ -0,0 +1,130 @@ +"""Handle jobs""" +import os +import sys +import shlex +import subprocess +from uuid import UUID, uuid4 +from datetime import timedelta +from typing import Union, Optional + +from redis import Redis +from flask import current_app as app + +JOBS_PREFIX = "JOBS" + +class JobNotFound(Exception): + """Raised if we try to retrieve a non-existent job.""" + +def jobsnamespace(): + """ + Return the jobs namespace prefix. It depends on app configuration. + + Calling this function outside of an application context will cause an + exception to be raised. It is mostly a convenience utility to use within the + application. + """ + return f"{app.config['GNQC_REDIS_PREFIX']}:{JOBS_PREFIX}" + +def job_key(namespaceprefix: str, jobid: Union[str, UUID]) -> str: + """Build the key by appending it to the namespace prefix.""" + return f"{namespaceprefix}:{jobid}" + +def raise_jobnotfound(rprefix:str, jobid: Union[str,UUID]): + """Utility to raise a `NoSuchJobError`""" + raise JobNotFound(f"Could not retrieve job '{jobid}' from '{rprefix}.") + +def error_filename(jobid, error_dir): + "Compute the path of the file where errors will be dumped." + return f"{error_dir}/job_{jobid}.error" + +def initialise_job(# pylint: disable=[too-many-arguments] + rconn: Redis, rprefix: str, jobid: str, command: list, job_type: str, + ttl_seconds: int = 86400, extra_meta: Optional[dict] = None) -> dict: + "Initialise a job 'object' and put in on redis" + the_job = { + "jobid": jobid, "command": shlex.join(command), "status": "pending", + "percent": 0, "job-type": job_type, **(extra_meta or {}) + } + rconn.hset(job_key(rprefix, jobid), mapping=the_job) + rconn.expire( + name=job_key(rprefix, jobid), time=timedelta(seconds=ttl_seconds)) + return the_job + +def build_file_verification_job(#pylint: disable=[too-many-arguments] + redis_conn: Redis, + dburi: str, + redisuri: str, + speciesid: int, + filepath: str, + filetype: str, + ttl_seconds: int): + "Build a file verification job" + jobid = str(uuid4()) + command = [ + sys.executable, "-m", "scripts.validate_file", + dburi, redisuri, jobsnamespace(), jobid, + "--redisexpiry", str(ttl_seconds), + str(speciesid), filetype, filepath, + ] + return initialise_job( + redis_conn, jobsnamespace(), jobid, command, "file-verification", + ttl_seconds, { + "filetype": filetype, + "filename": os.path.basename(filepath), "percent": 0 + }) + +def data_insertion_job(# pylint: disable=[too-many-arguments] + redis_conn: Redis, filepath: str, filetype: str, totallines: int, + speciesid: int, platformid: int, datasetid: int, databaseuri: str, + redisuri: str, ttl_seconds: int) -> dict: + "Build a data insertion job" + jobid = str(uuid4()) + command = [ + sys.executable, "-m", "scripts.insert_data", filetype, filepath, + speciesid, platformid, datasetid, databaseuri, redisuri + ] + return initialise_job( + redis_conn, jobsnamespace(), jobid, command, "data-insertion", + ttl_seconds, { + "filename": os.path.basename(filepath), "filetype": filetype, + "totallines": totallines + }) + +def launch_job(the_job: dict, redisurl: str, error_dir): + """Launch a job in the background""" + if not os.path.exists(error_dir): + os.mkdir(error_dir) + + jobid = the_job["jobid"] + with open(error_filename(jobid, error_dir), + "w", + encoding="utf-8") as errorfile: + subprocess.Popen( # pylint: disable=[consider-using-with] + [sys.executable, "-m", "scripts.worker", redisurl, jobsnamespace(), + jobid], + stderr=errorfile, + env={"PYTHONPATH": ":".join(sys.path)}) + + return the_job + +def job(rconn: Redis, rprefix: str, jobid: Union[str,UUID]): + "Retrieve the job" + thejob = (rconn.hgetall(job_key(rprefix, jobid)) or + raise_jobnotfound(rprefix, jobid)) + return thejob + +def update_status( + rconn: Redis, rprefix: str, jobid: Union[str, UUID], status: str): + """Update status of job in redis.""" + rconn.hset(name=job_key(rprefix, jobid), key="status", value=status) + +def update_stdout_stderr(rconn: Redis, + rprefix: str, + jobid: Union[str, UUID], + bytes_read: bytes, + stream: str): + "Update the stdout/stderr keys according to the value of `stream`." + thejob = job(rconn, rprefix, jobid) + contents = thejob.get(stream, '') + new_contents = contents + bytes_read.decode("utf-8") + rconn.hset(name=job_key(rprefix, jobid), key=stream, value=new_contents) diff --git a/uploader/parse.py b/uploader/parse.py new file mode 100644 index 0000000..865dae2 --- /dev/null +++ b/uploader/parse.py @@ -0,0 +1,175 @@ +"""File parsing module""" +import os + +import jsonpickle +from redis import Redis +from flask import flash, request, url_for, redirect, Blueprint, render_template +from flask import current_app as app + +from quality_control.errors import InvalidValue, DuplicateHeading + +from uploader import jobs +from uploader.dbinsert import species_by_id +from uploader.db_utils import with_db_connection + +parsebp = Blueprint("parse", __name__) + +def isinvalidvalue(item): + """Check whether item is of type InvalidValue""" + return isinstance(item, InvalidValue) + +def isduplicateheading(item): + """Check whether item is of type DuplicateHeading""" + return isinstance(item, DuplicateHeading) + +@parsebp.route("/parse", methods=["GET"]) +def parse(): + """Trigger file parsing""" + errors = False + speciesid = request.args.get("speciesid") + filename = request.args.get("filename") + filetype = request.args.get("filetype") + if speciesid is None: + flash("No species selected", "alert-error error-expr-data") + errors = True + else: + try: + speciesid = int(speciesid) + species = with_db_connection( + lambda con: species_by_id(con, speciesid)) + if not bool(species): + flash("No such species.", "alert-error error-expr-data") + errors = True + except ValueError: + flash("Invalid speciesid provided. Expected an integer.", + "alert-error error-expr-data") + errors = True + + if filename is None: + flash("No file provided", "alert-error error-expr-data") + errors = True + + if filetype is None: + flash("No filetype provided", "alert-error error-expr-data") + errors = True + + if filetype not in ("average", "standard-error"): + flash("Invalid filetype provided", "alert-error error-expr-data") + errors = True + + if filename: + filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename) + if not os.path.exists(filepath): + flash("Selected file does not exist (any longer)", + "alert-error error-expr-data") + errors = True + + if errors: + return redirect(url_for("entry.upload_file")) + + redisurl = app.config["REDIS_URL"] + with Redis.from_url(redisurl, decode_responses=True) as rconn: + job = jobs.launch_job( + jobs.build_file_verification_job( + rconn, app.config["SQL_URI"], redisurl, + speciesid, filepath, filetype, + app.config["JOBS_TTL_SECONDS"]), + redisurl, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + + return redirect(url_for("parse.parse_status", job_id=job["jobid"])) + +@parsebp.route("/status/<job_id>", methods=["GET"]) +def parse_status(job_id: str): + "Retrieve the status of the job" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + except jobs.JobNotFound as _exc: + return render_template("no_such_job.html", job_id=job_id), 400 + + error_filename = jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + if os.path.exists(error_filename): + stat = os.stat(error_filename) + if stat.st_size > 0: + return redirect(url_for("parse.fail", job_id=job_id)) + + job_id = job["jobid"] + progress = float(job["percent"]) + status = job["status"] + filename = job.get("filename", "uploaded file") + errors = jsonpickle.decode( + job.get("errors", jsonpickle.encode(tuple()))) + if status in ("success", "aborted"): + return redirect(url_for("parse.results", job_id=job_id)) + + if status == "parse-error": + return redirect(url_for("parse.fail", job_id=job_id)) + + app.jinja_env.globals.update( + isinvalidvalue=isinvalidvalue, + isduplicateheading=isduplicateheading) + return render_template( + "job_progress.html", + job_id = job_id, + job_status = status, + progress = progress, + message = job.get("message", ""), + job_name = f"Parsing '{filename}'", + errors=errors) + +@parsebp.route("/results/<job_id>", methods=["GET"]) +def results(job_id: str): + """Show results of parsing...""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + filename = job["filename"] + errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple()))) + app.jinja_env.globals.update( + isinvalidvalue=isinvalidvalue, + isduplicateheading=isduplicateheading) + return render_template( + "parse_results.html", + errors=errors, + job_name = f"Parsing '{filename}'", + user_aborted = job.get("user_aborted"), + job_id=job["jobid"]) + + return render_template("no_such_job.html", job_id=job_id) + +@parsebp.route("/fail/<job_id>", methods=["GET"]) +def fail(job_id: str): + """Handle parsing failure""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + error_filename = jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + if os.path.exists(error_filename): + stat = os.stat(error_filename) + if stat.st_size > 0: + return render_template( + "worker_failure.html", job_id=job_id) + + return render_template("parse_failure.html", job=job) + + return render_template("no_such_job.html", job_id=job_id) + +@parsebp.route("/abort", methods=["POST"]) +def abort(): + """Handle user request to abort file processing""" + job_id = request.form["job_id"] + + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id), + key="user_aborted", + value=int(True)) + + return redirect(url_for("parse.parse_status", job_id=job_id)) diff --git a/uploader/samples.py b/uploader/samples.py new file mode 100644 index 0000000..9c95770 --- /dev/null +++ b/uploader/samples.py @@ -0,0 +1,354 @@ +"""Code regarding samples""" +import os +import sys +import csv +import uuid +from pathlib import Path +from typing import Iterator + +import MySQLdb as mdb +from redis import Redis +from MySQLdb.cursors import DictCursor +from flask import ( + flash, + request, + url_for, + redirect, + Blueprint, + render_template, + current_app as app) + +from functional_tools import take + +from uploader import jobs +from uploader.files import save_file +from uploader.input_validation import is_integer_input +from uploader.db_utils import ( + with_db_connection, + database_connection, + with_redis_connection) +from uploader.db import ( + species_by_id, + save_population, + population_by_id, + populations_by_species, + species as fetch_species) + +samples = Blueprint("samples", __name__) + +@samples.route("/upload/species", methods=["GET", "POST"]) +def select_species(): + """Select the species.""" + if request.method == "GET": + return render_template("samples/select-species.html", + species=with_db_connection(fetch_species)) + + index_page = redirect(url_for("entry.upload_file")) + species_id = request.form.get("species_id") + if bool(species_id): + species_id = int(species_id) + species = with_db_connection( + lambda conn: species_by_id(conn, species_id)) + if bool(species): + return redirect(url_for( + "samples.select_population", species_id=species_id)) + flash("Invalid species selected!", "alert-error") + flash("You need to select a species", "alert-error") + return index_page + +@samples.route("/upload/species/<int:species_id>/create-population", + methods=["POST"]) +def create_population(species_id: int): + """Create new grouping/population.""" + if not is_integer_input(species_id): + flash("You did not provide a valid species. Please select one to " + "continue.", + "alert-danger") + return redirect(url_for("samples.select_species")) + species = with_db_connection(lambda conn: species_by_id(conn, species_id)) + if not bool(species): + flash("Species with given ID was not found.", "alert-danger") + return redirect(url_for("samples.select_species")) + + species_page = redirect(url_for("samples.select_species"), code=307) + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + pop_name = request.form.get("inbredset_name", "").strip() + pop_fullname = request.form.get("inbredset_fullname", "").strip() + + if not bool(species): + flash("Invalid species!", "alert-error error-create-population") + return species_page + if (not bool(pop_name)) or (not bool(pop_fullname)): + flash("You *MUST* provide a grouping/population name", + "alert-error error-create-population") + return species_page + + pop = save_population(conn, { + "SpeciesId": species["SpeciesId"], + "Name": pop_name, + "InbredSetName": pop_fullname, + "FullName": pop_fullname, + "Family": request.form.get("inbredset_family") or None, + "Description": request.form.get("description") or None + }) + + flash("Grouping/Population created successfully.", "alert-success") + return redirect(url_for("samples.upload_samples", + species_id=species_id, + population_id=pop["population_id"])) + +@samples.route("/upload/species/<int:species_id>/population", + methods=["GET", "POST"]) +def select_population(species_id: int): + """Select from existing groupings/populations.""" + if not is_integer_input(species_id): + flash("You did not provide a valid species. Please select one to " + "continue.", + "alert-danger") + return redirect(url_for("samples.select_species")) + species = with_db_connection(lambda conn: species_by_id(conn, species_id)) + if not bool(species): + flash("Species with given ID was not found.", "alert-danger") + return redirect(url_for("samples.select_species")) + + if request.method == "GET": + return render_template( + "samples/select-population.html", + species=species, + populations=with_db_connection( + lambda conn: populations_by_species(conn, species_id))) + + population_page = redirect(url_for( + "samples.select_population", species_id=species_id), code=307) + _population_id = request.form.get("inbredset_id") + if not is_integer_input(_population_id): + flash("You did not provide a valid population. Please select one to " + "continue.", + "alert-danger") + return population_page + population = with_db_connection( + lambda conn: population_by_id(conn, _population_id)) + if not bool(population): + flash("Invalid grouping/population!", + "alert-error error-select-population") + return population_page + + return redirect(url_for("samples.upload_samples", + species_id=species_id, + population_id=_population_id), + code=307) + +def read_samples_file(filepath, separator: str, firstlineheading: bool, **kwargs) -> Iterator[dict]: + """Read the samples file.""" + with open(filepath, "r", encoding="utf-8") as inputfile: + reader = csv.DictReader( + inputfile, + fieldnames=( + None if firstlineheading + else ("Name", "Name2", "Symbol", "Alias")), + delimiter=separator, + quotechar=kwargs.get("quotechar", '"')) + for row in reader: + yield row + +def save_samples_data(conn: mdb.Connection, + speciesid: int, + file_data: Iterator[dict]): + """Save the samples to DB.""" + data = ({**row, "SpeciesId": speciesid} for row in file_data) + total = 0 + with conn.cursor() as cursor: + while True: + batch = take(data, 5000) + if len(batch) == 0: + break + cursor.executemany( + "INSERT INTO Strain(Name, Name2, SpeciesId, Symbol, Alias) " + "VALUES(" + " %(Name)s, %(Name2)s, %(SpeciesId)s, %(Symbol)s, %(Alias)s" + ") ON DUPLICATE KEY UPDATE Name=Name", + batch) + total += len(batch) + print(f"\tSaved {total} samples total so far.") + +def cross_reference_samples(conn: mdb.Connection, + species_id: int, + population_id: int, + strain_names: Iterator[str]): + """Link samples to their population.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT MAX(OrderId) AS loid FROM StrainXRef WHERE InbredSetId=%s", + (population_id,)) + last_order_id = (cursor.fetchone()["loid"] or 10) + total = 0 + while True: + batch = take(strain_names, 5000) + if len(batch) == 0: + break + params_str = ", ".join(["%s"] * len(batch)) + ## This query is slow -- investigate. + cursor.execute( + "SELECT s.Id FROM Strain AS s LEFT JOIN StrainXRef AS sx " + "ON s.Id = sx.StrainId WHERE s.SpeciesId=%s AND s.Name IN " + f"({params_str}) AND sx.StrainId IS NULL", + (species_id,) + tuple(batch)) + strain_ids = (sid["Id"] for sid in cursor.fetchall()) + params = tuple({ + "pop_id": population_id, + "strain_id": strain_id, + "order_id": last_order_id + (order_id * 10), + "mapping": "N", + "pedigree": None + } for order_id, strain_id in enumerate(strain_ids, start=1)) + cursor.executemany( + "INSERT INTO StrainXRef( " + " InbredSetId, StrainId, OrderId, Used_for_mapping, PedigreeStatus" + ")" + "VALUES (" + " %(pop_id)s, %(strain_id)s, %(order_id)s, %(mapping)s, " + " %(pedigree)s" + ")", + params) + last_order_id += (len(params) * 10) + total += len(batch) + print(f"\t{total} total samples cross-referenced to the population " + "so far.") + +def build_sample_upload_job(# pylint: disable=[too-many-arguments] + speciesid: int, + populationid: int, + samplesfile: Path, + separator: str, + firstlineheading: bool, + quotechar: str): + """Define the async command to run the actual samples data upload.""" + return [ + sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"], + str(speciesid), str(populationid), str(samplesfile.absolute()), + separator, f"--redisuri={app.config['REDIS_URL']}", + f"--quotechar={quotechar}" + ] + (["--firstlineheading"] if firstlineheading else []) + +@samples.route("/upload/species/<int:species_id>/populations/<int:population_id>/samples", + methods=["GET", "POST"]) +def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements] + """Upload the samples.""" + samples_uploads_page = redirect(url_for("samples.upload_samples", + species_id=species_id, + population_id=population_id)) + if not is_integer_input(species_id): + flash("You did not provide a valid species. Please select one to " + "continue.", + "alert-danger") + return redirect(url_for("samples.select_species")) + species = with_db_connection(lambda conn: species_by_id(conn, species_id)) + if not bool(species): + flash("Species with given ID was not found.", "alert-danger") + return redirect(url_for("samples.select_species")) + + if not is_integer_input(population_id): + flash("You did not provide a valid population. Please select one " + "to continue.", + "alert-danger") + return redirect(url_for("samples.select_population", + species_id=species_id), + code=307) + population = with_db_connection( + lambda conn: population_by_id(conn, int(population_id))) + if not bool(population): + flash("Invalid grouping/population!", "alert-error") + return redirect(url_for("samples.select_population", + species_id=species_id), + code=307) + + if request.method == "GET" or request.files.get("samples_file") is None: + return render_template("samples/upload-samples.html", + species=species, + population=population) + + try: + samples_file = save_file(request.files["samples_file"], + Path(app.config["UPLOAD_FOLDER"])) + except AssertionError: + flash("You need to provide a file with the samples data.", + "alert-error") + return samples_uploads_page + + firstlineheading = (request.form.get("first_line_heading") == "on") + + separator = request.form.get("separator", ",") + if separator == "other": + separator = request.form.get("other_separator", ",") + if not bool(separator): + flash("You need to provide a separator character.", "alert-error") + return samples_uploads_page + + quotechar = (request.form.get("field_delimiter", '"') or '"') + + redisuri = app.config["REDIS_URL"] + with Redis.from_url(redisuri, decode_responses=True) as rconn: + the_job = jobs.launch_job( + jobs.initialise_job( + rconn, + jobs.jobsnamespace(), + str(uuid.uuid4()), + build_sample_upload_job( + species["SpeciesId"], + population["InbredSetId"], + samples_file, + separator, + firstlineheading, + quotechar), + "samples_upload", + app.config["JOBS_TTL_SECONDS"], + {"job_name": f"Samples Upload: {samples_file.name}"}), + redisuri, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + return redirect(url_for( + "samples.upload_status", job_id=the_job["jobid"])) + +@samples.route("/upload/status/<uuid:job_id>", methods=["GET"]) +def upload_status(job_id: uuid.UUID): + """Check on the status of a samples upload job.""" + job = with_redis_connection(lambda rconn: jobs.job( + rconn, jobs.jobsnamespace(), job_id)) + if job: + status = job["status"] + if status == "success": + return render_template("samples/upload-success.html", job=job) + + if status == "error": + return redirect(url_for("samples.upload_failure", job_id=job_id)) + + error_filename = Path(jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) + if error_filename.exists(): + stat = os.stat(error_filename) + if stat.st_size > 0: + return redirect(url_for( + "samples.upload_failure", job_id=job_id)) + + return render_template( + "samples/upload-progress.html", + job=job) # maybe also handle this? + + return render_template("no_such_job.html", job_id=job_id), 400 + +@samples.route("/upload/failure/<uuid:job_id>", methods=["GET"]) +def upload_failure(job_id: uuid.UUID): + """Display the errors of the samples upload failure.""" + job = with_redis_connection(lambda rconn: jobs.job( + rconn, jobs.jobsnamespace(), job_id)) + if not bool(job): + return render_template("no_such_job.html", job_id=job_id), 400 + + error_filename = Path(jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) + if error_filename.exists(): + stat = os.stat(error_filename) + if stat.st_size > 0: + return render_template("worker_failure.html", job_id=job_id) + + return render_template("samples/upload-failure.html", job=job) diff --git a/uploader/static/css/custom-bootstrap.css b/uploader/static/css/custom-bootstrap.css new file mode 100644 index 0000000..67f1199 --- /dev/null +++ b/uploader/static/css/custom-bootstrap.css @@ -0,0 +1,23 @@ +/** Customize some bootstrap selectors **/ +.btn { + text-transform: capitalize; +} + +.navbar-inverse { + background-color: #336699; + border-color: #080808; + color: #FFFFFF; + background-image: none; +} + +.navbar-inverse .navbar-nav>li>a { + color: #FFFFFF; +} + +.navbar-nav > li > a { + padding: 5px; +} + +.navbar { + min-height: 30px; +} diff --git a/uploader/static/css/styles.css b/uploader/static/css/styles.css new file mode 100644 index 0000000..a88c229 --- /dev/null +++ b/uploader/static/css/styles.css @@ -0,0 +1,7 @@ +.heading { + text-transform: capitalize; +} + +label { + text-transform: capitalize; +} diff --git a/uploader/static/css/two-column-with-separator.css b/uploader/static/css/two-column-with-separator.css new file mode 100644 index 0000000..b6efd46 --- /dev/null +++ b/uploader/static/css/two-column-with-separator.css @@ -0,0 +1,27 @@ +.two-column-with-separator { + display: grid; + grid-template-columns: 9fr 1fr 9fr; +} + +.two-col-sep-col1 { + grid-column: 1 / 2; +} + +.two-col-sep-separator { + grid-column: 2 / 3; + text-align: center; + color: #FE3535; + font-weight: bolder; +} + +.two-col-sep-col2 { + grid-column: 3 / 4; +} + +.two-col-sep-col1, .two-col-sep-col2 { + border-style: solid; + border-color: #FE3535; + border-width: 1px; + border-radius: 2em; + padding: 2em 3em 2em 3em; +} diff --git a/uploader/static/images/CITGLogo.png b/uploader/static/images/CITGLogo.png Binary files differnew file mode 100644 index 0000000..ae99fed --- /dev/null +++ b/uploader/static/images/CITGLogo.png diff --git a/uploader/static/js/select_platform.js b/uploader/static/js/select_platform.js new file mode 100644 index 0000000..4fdd865 --- /dev/null +++ b/uploader/static/js/select_platform.js @@ -0,0 +1,70 @@ +function radio_column(chip) { + col = document.createElement("td"); + radio = document.createElement("input"); + radio.setAttribute("type", "radio"); + radio.setAttribute("name", "genechipid"); + radio.setAttribute("value", chip["GeneChipId"]); + radio.setAttribute("required", "required"); + col.appendChild(radio); + return col; +} + +function setup_genechips(genechip_data) { + columns = ["GeneChipId", "GeneChipName"] + submit_button = document.querySelector( + "#select-platform-form button[type='submit']"); + elt = document.getElementById( + "genechips-table").getElementsByTagName("tbody")[0]; + remove_children(elt); + if((genechip_data === undefined) || genechip_data.length === 0) { + row = document.createElement("tr"); + col = document.createElement("td"); + col.setAttribute("colspan", "3"); + text = document.createTextNode("No chips found for selected species"); + col.appendChild(text); + row.appendChild(col); + elt.appendChild(row); + submit_button.setAttribute("disabled", true); + return false; + } + + submit_button.removeAttribute("disabled") + genechip_data.forEach(chip => { + row = document.createElement("tr"); + row.appendChild(radio_column(chip)); + columns.forEach(column => { + col = document.createElement("td"); + content = document.createTextNode(chip[column]); + col.appendChild(content); + row.appendChild(col); + }); + elt.appendChild(row); + }); +} + +function genechips() { + return JSON.parse( + document.getElementById("select-platform-form").getAttribute( + "data-genechips")); +} + +function update_genechips(event) { + genec = genechips(); + + species_elt = document.getElementById("species"); + + if(event.target == species_elt) { + setup_genechips(genec[species_elt.value.toLowerCase()]); + } +} + +function select_row_radio(row) { + radio = row.getElementsByTagName( + "td")[0].getElementsByTagName( + "input")[0]; + if(radio === undefined) { + return false; + } + radio.setAttribute("checked", "checked"); + return true; +} diff --git a/uploader/static/js/upload_progress.js b/uploader/static/js/upload_progress.js new file mode 100644 index 0000000..9638b36 --- /dev/null +++ b/uploader/static/js/upload_progress.js @@ -0,0 +1,97 @@ +function make_processing_indicator(elt) { + var count = 0; + return function() { + var message = "Finalising upload and saving file: " + if(count > 5) { + count = 1; + } + for(i = 0; i < count; i++) { + message = message + "."; + } + elt.innerHTML = message + count = count + 1 + }; +} + +function make_progress_updater(file, indicator_elt) { + var progress_bar = indicator_elt.querySelector("#progress-bar"); + var progress_text = indicator_elt.querySelector("#progress-text"); + var extra_text = indicator_elt.querySelector("#progress-extra-text"); + return function(event) { + if(event.loaded <= file.size) { + var percent = Math.round((event.loaded / file.size) * 100); + progress_bar.value = percent + progress_text.innerHTML = "Uploading: " + percent + "%"; + extra_text.setAttribute("class", "hidden") + } + + if(event.loaded == event.total) { + progress_bar.value = 100; + progress_text.innerHTML = "Uploaded: 100%"; + extra_text.setAttribute("class", null); + intv = setInterval(make_processing_indicator(extra_text), 400); + setTimeout(function() {clearTimeout(intv);}, 20000); + } + }; +} + +function setup_cancel_upload(request, indicator_elt) { + document.getElementById("btn-cancel-upload").addEventListener( + "click", function(event) { + event.preventDefault(); + request.abort(); + }); +} + +function setup_request(file, progress_indicator_elt) { + var request = new XMLHttpRequest(); + var updater = make_progress_updater(file, progress_indicator_elt); + request.upload.addEventListener("progress", updater); + request.onload = function(event) { + document.location.assign(request.responseURL); + }; + setup_cancel_upload(request, progress_indicator_elt) + return request; +} + +function selected_filetype(radios) { + for(idx = 0; idx < radios.length; idx++) { + if(radios[idx].checked) { + return radios[idx].value; + } + } +} + +function make_data_uploader(setup_formdata) { + return function(event) { + event.preventDefault(); + + var pindicator = document.getElementById("upload-progress-indicator"); + + var form = event.target; + var the_file = form.querySelector("input[type='file']").files[0]; + if(the_file === undefined) { + form.querySelector("input[type='file']").parentElement.setAttribute( + "class", "invalid-input"); + error_elt = form.querySelector("#no-file-error"); + if(error_elt !== undefined) { + error_elt.setAttribute("style", "display: block;"); + } + return false; + } + var formdata = setup_formdata(form); + + document.getElementById("progress-filename").innerHTML = the_file.name; + var request = setup_request(the_file, pindicator); + request.open(form.getAttribute("method"), form.getAttribute("action")); + request.send(formdata); + return false; + } +} + + +function setup_upload_handlers(formid, datauploader) { + console.info("Setting up the upload handlers.") + upload_form = document.getElementById(formid); + upload_form.addEventListener("submit", datauploader); +} diff --git a/uploader/static/js/upload_samples.js b/uploader/static/js/upload_samples.js new file mode 100644 index 0000000..aed536f --- /dev/null +++ b/uploader/static/js/upload_samples.js @@ -0,0 +1,132 @@ +/* + * Read the file content and set the `data-preview-content` attribute on the + * file element + */ +function read_first_n_lines(event, + fileelement, + numlines, + firstlineheading = true) { + var thefile = fileelement.files[0]; + var reader = new FileReader(); + reader.addEventListener("load", (event) => { + var filecontent = event.target.result.split( + "\n").slice( + 0, (numlines + (firstlineheading ? 1 : 0))).map( + (line) => {return line.trim("\r");}); + fileelement.setAttribute( + "data-preview-content", JSON.stringify(filecontent)); + display_preview(event); + }) + reader.readAsText(thefile); +} + +function remove_rows(preview_table) { + var table_body = preview_table.getElementsByTagName("tbody")[0]; + while(table_body.children.length > 0) { + table_body.removeChild(table_body.children.item(0)); + } +} + +/* + * Display error row + */ +function display_error_row(preview_table, error_message) { + remove_rows(preview_table); + row = document.createElement("tr"); + cell = document.createElement("td"); + cell.setAttribute("colspan", 4); + cell.innerHTML = error_message; + row.appendChild(cell); + preview_table.getElementsByTagName("tbody")[0].appendChild(row); +} + +function strip(str, chars) { + var end = str.length; + var start = 0 + for(var j = str.length; j > 0; j--) { + if(!chars.includes(str[j - 1])) { + break; + } + end = end - 1; + } + for(var i = 0; i < end; i++) { + if(!chars.includes(str[i])) { + break; + } + start = start + 1; + } + return str.slice(start, end); +} + +function process_preview_data(preview_data, separator, delimiter) { + return preview_data.map((line) => { + return line.split(separator).map((field) => { + return strip(field, delimiter); + }); + }); +} + +function render_preview(preview_table, preview_data) { + remove_rows(preview_table); + var table_body = preview_table.getElementsByTagName("tbody")[0]; + preview_data.forEach((line) => { + var row = document.createElement("tr"); + line.forEach((field) => { + var cell = document.createElement("td"); + cell.innerHTML = field; + row.appendChild(cell); + }); + table_body.appendChild(row); + }); +} + +/* + * Display a preview of the data, relying on the user's selection. + */ +function display_preview(event) { + var data_preview_table = document.getElementById("tbl:samples-preview"); + remove_rows(data_preview_table); + + var separator = document.getElementById("select:separator").value; + if(separator === "other") { + separator = document.getElementById("txt:separator").value; + } + if(separator == "") { + display_error_row(data_preview_table, "Please provide a separator."); + return false; + } + + var delimiter = document.getElementById("txt:delimiter").value; + + var firstlineheading = document.getElementById("chk:heading").checked; + + var fileelement = document.getElementById("file:samples"); + var preview_data = JSON.parse( + fileelement.getAttribute("data-preview-content") || "[]"); + if(preview_data.length == 0) { + display_error_row( + data_preview_table, + "No file data to preview. Check that file is provided."); + } + + render_preview(data_preview_table, process_preview_data( + preview_data.slice(0 + (firstlineheading ? 1 : 0)), + separator, + delimiter)); +} + +document.getElementById("chk:heading").addEventListener( + "change", display_preview); +document.getElementById("select:separator").addEventListener( + "change", display_preview); +document.getElementById("txt:separator").addEventListener( + "keyup", display_preview); +document.getElementById("txt:delimiter").addEventListener( + "keyup", display_preview); +document.getElementById("file:samples").addEventListener( + "change", (event) => { + read_first_n_lines(event, + document.getElementById("file:samples"), + 30, + document.getElementById("chk:heading").checked); + }); diff --git a/uploader/static/js/utils.js b/uploader/static/js/utils.js new file mode 100644 index 0000000..045dd47 --- /dev/null +++ b/uploader/static/js/utils.js @@ -0,0 +1,10 @@ +function remove_children(element) { + Array.from(element.children).forEach(child => { + element.removeChild(child); + }); +} + +function trigger_change_event(element) { + evt = new Event("change"); + element.dispatchEvent(evt); +} diff --git a/uploader/templates/base.html b/uploader/templates/base.html new file mode 100644 index 0000000..eb5e6b7 --- /dev/null +++ b/uploader/templates/base.html @@ -0,0 +1,51 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="UTF-8" /> + <meta application-name="GeneNetwork Quality-Control Application" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + {%block extrameta%}{%endblock%} + + <title>GN Uploader: {%block title%}{%endblock%}</title> + + <link rel="stylesheet" type="text/css" + href="{{url_for('base.bootstrap', + filename='css/bootstrap.min.css')}}" /> + <link rel="stylesheet" type="text/css" + href="{{url_for('base.bootstrap', + filename='css/bootstrap-theme.min.css')}}" /> + + + <link rel="shortcut icon" type="image/png" sizes="64x64" + href="{{url_for('static', filename='images/CITGLogo.png')}}" /> + + <link rel="stylesheet" type="text/css" href="/static/css/custom-bootstrap.css" /> + <link rel="stylesheet" type="text/css" href="/static/css/styles.css" /> + + {%block css%}{%endblock%} + </head> + + <body> + <div class="navbar navbar-inverse navbar-static-top pull-left" + role="navigation" + style="width: 100%;min-width: 850px;white-space: nowrap;"> + <div class="container-fluid" style="width: 100%"> + <ul class="nav navbar-nav"> + <li><a href="/" style="font-weight: bold">GN Uploader</a></li> + <li> + <a href="{{gnuri or 'https://genenetwork.org'}}">GeneNetwork</a> + </li> + </ul> + </div> + </div> + <div class="container"> + {%block contents%}{%endblock%} + </div> + + <script src="{{url_for('base.jquery', + filename='jquery.min.js')}}"></script> + <script src="{{url_for('base.bootstrap', + filename='js/bootstrap.min.js')}}"></script> + {%block javascript%}{%endblock%} + </body> +</html> diff --git a/uploader/templates/cli-output.html b/uploader/templates/cli-output.html new file mode 100644 index 0000000..33fb73b --- /dev/null +++ b/uploader/templates/cli-output.html @@ -0,0 +1,8 @@ +{%macro cli_output(job, stream)%} + +<h4>{{stream | upper}} Output</h4> +<div class="cli-output"> + <pre>{{job.get(stream, "")}}</pre> +</div> + +{%endmacro%} diff --git a/uploader/templates/continue_from_create_dataset.html b/uploader/templates/continue_from_create_dataset.html new file mode 100644 index 0000000..03bb49c --- /dev/null +++ b/uploader/templates/continue_from_create_dataset.html @@ -0,0 +1,52 @@ +{%extends "base.html"%} +{%from "dbupdate_hidden_fields.html" import hidden_fields%} + +{%block title%}Create Study{%endblock%} + +{%block css%} +<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> +{%endblock%} + +{%block contents%} +<h2 class="heading">{{filename}}: create study</h2> + +{%with messages = get_flashed_messages(with_categories=true)%} +{%if messages:%} +<ul> + {%for category, message in messages:%} + <li class="{{category}}">{{message}}</li> + {%endfor%} +</ul> +{%endif%} +{%endwith%} + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.final_confirmation')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col1"> + <legend>continue with new dataset</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, datasetid=datasetid, totallines=totallines)}} + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +<div class="row"> + <p class="two-col-sep-separator">OR</p> +</div> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.select_dataset')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col2"> + <legend>Select from existing dataset</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, datasetid=datasetid, totallines=totallines)}} + + <button type="submit" class="btn btn-primary">go back</button> + </form> +</div> +{%endblock%} diff --git a/uploader/templates/continue_from_create_study.html b/uploader/templates/continue_from_create_study.html new file mode 100644 index 0000000..34e6e5e --- /dev/null +++ b/uploader/templates/continue_from_create_study.html @@ -0,0 +1,52 @@ +{%extends "base.html"%} +{%from "dbupdate_hidden_fields.html" import hidden_fields%} + +{%block title%}Create Study{%endblock%} + +{%block css%} +<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> +{%endblock%} + +{%block contents%} +<h2 class="heading">{{filename}}: create study</h2> + +{%with messages = get_flashed_messages(with_categories=true)%} +{%if messages:%} +<ul> + {%for category, message in messages:%} + <li class="{{category}}">{{message}}</li> + {%endfor%} +</ul> +{%endif%} +{%endwith%} + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.select_dataset')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col1"> + <legend>continue with new study</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, totallines=totallines)}} + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +<div class="row"> + <p class="two-col-sep-separator">OR</p> +</div> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.select_study')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col2"> + <legend>Select from existing study</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, totallines=totallines)}} + + <button type="submit" class="btn btn-primary">go back</button> + </form> +</div> +{%endblock%} diff --git a/uploader/templates/data_review.html b/uploader/templates/data_review.html new file mode 100644 index 0000000..b7528fd --- /dev/null +++ b/uploader/templates/data_review.html @@ -0,0 +1,85 @@ +{%extends "base.html"%} + +{%block title%}Data Review{%endblock%} + +{%block contents%} +<h1 class="heading">data review</h1> + +<div class="row"> + <h2 id="data-concerns">Data Concerns</h2> + <p>The following are some of the requirements that the data in your file + <strong>MUST</strong> fulfil before it is considered valid for this system: + </p> + + <ol> + <li>File headings + <ul> + <li>The first row in the file should contains the headings. The number of + headings in this first row determines the number of columns expected for + all other lines in the file.</li> + <li>Each heading value in the first row MUST appear in the first row + <strong>ONE AND ONLY ONE</strong> time</li> + <li>The sample/cases (previously 'strains') headers in your first row will be + against those in the <a href="https://genenetwork.org" + title="Link to the GeneNetwork service"> + GeneNetwork</a> database.<br /> + <small class="text-muted"> + If you encounter an error saying your sample(s)/case(s) do not exist + in the GeneNetwork database, then you will have to use the + <a href="{{url_for('samples.select_species')}}" + title="Upload samples/cases feature">Upload Samples/Cases</a> + option on this system to upload them. + </small> + </ul> + </li> + + <li>Data + <ol> + <li><strong>NONE</strong> of the data cells/fields is allowed to be empty. + All fields/cells <strong>MUST</strong> contain a value.</li> + <li>The first column of the data rows will be considered a textual field, + holding the "identifier" for that row<li> + <li>Except for the first column/field for each data row, + <strong>NONE</strong> of the data columns/cells/fields should contain + spurious characters like `eeeee`, `5.555iloveguix`, etc...<br /> + All of them should be decimal values</li> + <li>decimal numbers must conform to the following criteria: + <ul> + <li>when checking an average file decimal numbers must have exactly three + decimal places to the right of the decimal point.</li> + <li>when checking a standard error file decimal numbers must have six or + greater decimal places to the right of the decimal point.</li> + <li>there must be a number to the left side of the decimal place + (e.g. 0.55555 is allowed but .55555 is not).</li> + </ul> + </li> + </ol> + </li> + </ol> +</div> + + +<div class="row"> + <h2 id="file-types">Supported File Types</h2> + We support the following file types: + + <ul> + <li>Tab-Separated value files (.tsv) + <ul> + <li>The <strong>TAB</strong> character is used to separate the fields of each + column</li> + <li>The values of each field <strong>ARE NOT</strong> quoted.</li> + <li>Here is an + <a href="https://gitlab.com/fredmanglis/gnqc_py/-/blob/main/tests/test_data/no_data_errors.tsv"> + example file</a> with a single data row.</li> + </ul> + </li> + <li>.txt files: Content has the same format as .tsv file above</li> + <li>.zip files: each zip file should contain + <strong>ONE AND ONLY ONE</strong> file of the .tsv or .txt type above. + <br />Any zip file with more than one file is invalid, and so is an empty + zip file.</li> + </ul> + +</div> +{%endblock%} diff --git a/uploader/templates/dbupdate_error.html b/uploader/templates/dbupdate_error.html new file mode 100644 index 0000000..e1359d2 --- /dev/null +++ b/uploader/templates/dbupdate_error.html @@ -0,0 +1,12 @@ +{%extends "base.html"%} + +{%block title%}DB Update Error{%endblock%} + +{%block contents%} +<h1 class="heading">database update error</h2> + +<p class="alert-danger"> + <strong>Database Update Error</strong>: {{error_message}} +</p> + +{%endblock%} diff --git a/uploader/templates/dbupdate_hidden_fields.html b/uploader/templates/dbupdate_hidden_fields.html new file mode 100644 index 0000000..ccbc299 --- /dev/null +++ b/uploader/templates/dbupdate_hidden_fields.html @@ -0,0 +1,29 @@ +{%macro hidden_fields(filename, filetype):%} + +<!-- {{kwargs}}: mostly for accessing the kwargs in macro --> + +<input type="hidden" name="filename" value="{{filename}}" /> +<input type="hidden" name="filetype" value="{{filetype}}" /> +{%if kwargs.get("totallines")%} +<input type="hidden" name="totallines" value="{{kwargs['totallines']}}" /> +{%endif%} +{%if kwargs.get("species"):%} +<input type="hidden" name="species" value="{{kwargs['species']}}" /> +{%endif%} +{%if kwargs.get("genechipid"):%} +<input type="hidden" name="genechipid" value="{{kwargs['genechipid']}}" /> +{%endif%} +{%if kwargs.get("inbredsetid"):%} +<input type="hidden" name="inbredsetid" value="{{kwargs['inbredsetid']}}" /> +{%endif%} +{%if kwargs.get("tissueid"):%} +<input type="hidden" name="tissueid" value="{{kwargs['tissueid']}}" /> +{%endif%} +{%if kwargs.get("studyid"):%} +<input type="hidden" name="studyid" value="{{kwargs['studyid']}}" /> +{%endif%} +{%if kwargs.get("datasetid"):%} +<input type="hidden" name="datasetid" value="{{kwargs['datasetid']}}" /> +{%endif%} + +{%endmacro%} diff --git a/uploader/templates/errors_display.html b/uploader/templates/errors_display.html new file mode 100644 index 0000000..715cfcf --- /dev/null +++ b/uploader/templates/errors_display.html @@ -0,0 +1,47 @@ +{%macro errors_display(errors, no_error_msg, error_message, complete)%} + +{%if errors | length == 0 %} +<span {%if complete%}class="alert-success"{%endif%}>{{no_error_msg}}</span> +{%else %} +<p class="alert-danger">{{error_message}}</p> + +<table class="table reports-table"> + <thead> + <tr> + <th>line number</th> + <th>column(s)</th> + <th>error</th> + <th>error message</th> + </tr> + </thead> + + <tbody> + {%for error in errors%} + <tr> + <td>{{error["line"]}}</td> + <td> + {%if isinvalidvalue(error):%} + {{error.column}} + {%elif isduplicateheading(error): %} + {{error.columns}} + {%else: %} + - + {%endif %} + </td> + <td> + {%if isinvalidvalue(error):%} + Invalid Value + {%elif isduplicateheading(error): %} + Duplicate Header + {%else%} + Inconsistent Columns + {%endif %} + </td> + <td>{{error["message"]}}</td> + </tr> + {%endfor%} + </tbody> +</table> +{%endif%} + +{%endmacro%} diff --git a/uploader/templates/final_confirmation.html b/uploader/templates/final_confirmation.html new file mode 100644 index 0000000..0727fc8 --- /dev/null +++ b/uploader/templates/final_confirmation.html @@ -0,0 +1,47 @@ +{%extends "base.html"%} +{%from "dbupdate_hidden_fields.html" import hidden_fields%} + +{%block title%}Confirmation{%endblock%} + +{%macro display_item(item_name, item_data):%} +<li> + <strong>{{item_name}}</strong> + {%if item_data%} + <ul> + {%for term,value in item_data.items():%} + <li><strong>{{term}}:</strong> {{value}}</li> + {%endfor%} + </ul> + {%endif%} +</li> +{%endmacro%} + +{%block contents%} +<h2 class="heading">Final Confirmation</h2> + +<div class="two-col-sep-col1"> + <p><strong>Selected Data</strong></p> + <ul> + <li><strong>File</strong> + <ul> + <li><strong>Filename</strong>: {{filename}}</li> + <li><strong>File Type</strong>: {{filetype}}</li> + </ul> + </li> + {{display_item("Species", the_species)}} + {{display_item("Platform", platform)}} + {{display_item("Study", study)}} + {{display_item("Dataset", dataset)}} + </ul> +</div> + +<form method="POST" action="{{url_for('dbinsert.insert_data')}}"> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid,datasetid=datasetid, totallines=totallines)}} + <fieldset> + <input type="submit" class="btn btn-primary" value="confirm" /> + </fieldset> +</form> +</div> +{%endblock%} diff --git a/uploader/templates/flash_messages.html b/uploader/templates/flash_messages.html new file mode 100644 index 0000000..b7af178 --- /dev/null +++ b/uploader/templates/flash_messages.html @@ -0,0 +1,25 @@ +{%macro flash_all_messages()%} +{%with messages = get_flashed_messages(with_categories=true)%} +{%if messages:%} +<ul> + {%for category, message in messages:%} + <li class="{{category}}">{{message}}</li> + {%endfor%} +</ul> +{%endif%} +{%endwith%} +{%endmacro%} + +{%macro flash_messages(filter_class)%} +{%with messages = get_flashed_messages(with_categories=true)%} +{%if messages:%} +<ul> + {%for category, message in messages:%} + {%if filter_class in category%} + <li class="{{category}}">{{message}}</li> + {%endif%} + {%endfor%} +</ul> +{%endif%} +{%endwith%} +{%endmacro%} diff --git a/uploader/templates/http-error.html b/uploader/templates/http-error.html new file mode 100644 index 0000000..374fb86 --- /dev/null +++ b/uploader/templates/http-error.html @@ -0,0 +1,18 @@ +{%extends "base.html"%} + +{%block title%}HTTP Error: {{exc.code}}{%endblock%} + +{%block contents%} +<h1>{{exc.code}}: {{exc.description}}</h1> + +<div class="row"> + <p> + You attempted to access {{request_url}} which failed with the following + error: + </p> +</div> + +<div class="row"> + <pre>{{"\n".join(trace)}}</pre> +</div> +{%endblock%} diff --git a/uploader/templates/index.html b/uploader/templates/index.html new file mode 100644 index 0000000..89d2ae9 --- /dev/null +++ b/uploader/templates/index.html @@ -0,0 +1,81 @@ +{%extends "base.html"%} + +{%block title%}Data Upload{%endblock%} + +{%block contents%} +<div class="row"> + <h1 class="heading">data upload</h1> + + <div class="explainer"> + <p>Each of the sections below gives you a different option for data upload. + Please read the documentation for each section carefully to understand what + each section is about.</p> + </div> +</div> + +<div class="row"> + <h2 class="heading">R/qtl2 Bundles</h2> + + <div class="explainer"> + <p>This feature combines and extends the two upload methods below. Instead of + uploading one item at a time, the R/qtl2 bundle you upload can contain both + the genotypes data (samples/individuals/cases and their data) and the + expression data.</p> + <p>The R/qtl2 bundle, additionally, can contain extra metadata, that neither + of the methods below can handle.</p> + + <a href="{{url_for('upload.rqtl2.select_species')}}" + title="Upload a zip bundle of R/qtl2 files"> + <button class="btn btn-primary">upload R/qtl2 bundle</button></a> + </div> +</div> + + +<div class="row"> + <h2 class="heading">Expression Data</h2> + + <div class="explainer"> + <p>This feature enables you to upload expression data. It expects the data to + be in <strong>tab-separated values (TSV)</strong> files. The data should be + a simple matrix of <em>phenotype × sample</em>, i.e. The first column is a + list of the <em>phenotypes</em> and the first row is a list of + <em>samples/cases</em>.</p> + + <p>If you haven't done so please go to this page to learn the requirements for + file formats and helpful suggestions to enter your data in a fast and easy + way.</p> + + <ol> + <li><strong>PLEASE REVIEW YOUR DATA.</strong>Make sure your data complies + with our system requirements. ( + <a href="{{url_for('entry.data_review')}}#data-concerns" + title="Details for the data expectations.">Help</a> + )</li> + <li><strong>UPLOAD YOUR DATA FOR DATA VERIFICATION.</strong> We accept + <strong>.csv</strong>, <strong>.txt</strong> and <strong>.zip</strong> + files (<a href="{{url_for('entry.data_review')}}#file-types" + title="Details for the data expectations.">Help</a>)</li> + </ol> + </div> + + <a href="{{url_for('entry.upload_file')}}" + title="Upload your expression data" + class="btn btn-primary">upload expression data</a> +</div> + +<div class="row"> + <h2 class="heading">samples/cases</h2> + + <div class="explainer"> + <p>For the expression data above, you need the samples/cases in your file to + already exist in the GeneNetwork database. If there are any samples that do + not already exist the upload of the expression data will fail.</p> + <p>This section gives you the opportunity to upload any missing samples</p> + </div> + + <a href="{{url_for('samples.select_species')}}" + title="Upload samples/cases/individuals for your data" + class="btn btn-primary">upload Samples/Cases</a> +</div> + +{%endblock%} diff --git a/uploader/templates/insert_error.html b/uploader/templates/insert_error.html new file mode 100644 index 0000000..5301288 --- /dev/null +++ b/uploader/templates/insert_error.html @@ -0,0 +1,32 @@ +{%extends "base.html"%} + +{%block title%}Data Insertion Failure{%endblock%} + +{%block contents%} +<h1 class="heading">Insertion Failure</h1> + +<div class="row"> + <p> + There was an error inserting data into the database + </p> + + <p> + Please notify the developers of this issue when you encounter it, + providing the information below. + </p> + + <h4>Debugging Information</h4> + + <ul> + <li><strong>job id</strong>: {{job["jobid"]}}</li> + </ul> +</div> + +<div class="row"> + <h4>STDERR Output</h4> + <pre class="cli-output"> + {{job["stderr"]}} + </pre> +</div> + +{%endblock%} diff --git a/uploader/templates/insert_progress.html b/uploader/templates/insert_progress.html new file mode 100644 index 0000000..52177d6 --- /dev/null +++ b/uploader/templates/insert_progress.html @@ -0,0 +1,46 @@ +{%extends "base.html"%} +{%from "stdout_output.html" import stdout_output%} + +{%block extrameta%} +<meta http-equiv="refresh" content="5"> +{%endblock%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">{{job_name}}</h1> + +<div class="row"> + <form> + <div class="form-group"> + <label for="job_status" class="form-label">status:</label> + <span class="form-text">{{job_status}}: {{message}}</span> + </div> + +{%if job.get("stdout", "").split("\n\n") | length < 3 %} +{%set lines = 0%} +{%else%} +{%set lines = (job.get("stdout", "").split("\n\n") | length / 3) %} +{%endif%} +{%set totallines = job.get("totallines", lines+3) | int %} +{%if totallines > 1000 %} +{%set fraction = ((lines*1000)/totallines) %} +{%else%} +{%set fraction = (lines/totallines)%} +{%endif%} + + <div class="form-group"> + <label for="job_{{job_id}}" class="form-label">inserting: </label> + <progress id="jobs_{{job_id}}" + value="{{(fraction)}}" + class="form-control">{{fraction*100}}</progress> + <span class="form-text text-muted"> + {{"%.2f" | format(fraction * 100 | float)}}%</span> + </div> + </form> +</div> + + +{{stdout_output(job)}} + +{%endblock%} diff --git a/uploader/templates/insert_success.html b/uploader/templates/insert_success.html new file mode 100644 index 0000000..7e1fa8d --- /dev/null +++ b/uploader/templates/insert_success.html @@ -0,0 +1,19 @@ +{%extends "base.html"%} +{%from "stdout_output.html" import stdout_output%} + +{%block title%}Insertion Success{%endblock%} + +{%block contents%} +<h1 class="heading">Insertion Success</h1> + +<div class="row"> +<p>Data inserted successfully!</p> + +<p>The following queries were run:</p> +</div> + +<div class="row"> + {{stdout_output(job)}} +</div> + +{%endblock%} diff --git a/uploader/templates/job_progress.html b/uploader/templates/job_progress.html new file mode 100644 index 0000000..1af0763 --- /dev/null +++ b/uploader/templates/job_progress.html @@ -0,0 +1,40 @@ +{%extends "base.html"%} +{%from "errors_display.html" import errors_display%} + +{%block extrameta%} +<meta http-equiv="refresh" content="5"> +{%endblock%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">{{job_name}}</h2> + +<div class="row"> + <form action="{{url_for('parse.abort')}}" method="POST"> + <legend class="heading">Status</legend> + <div class="form-group"> + <label for="job_status" class="form-label">status:</label> + <span class="form-text">{{job_status}}: {{message}}</span><br /> + </div> + + <div class="form-group"> + <label for="job_{{job_id}}" class="form-label">parsing: </label> + <progress id="job_{{job_id}}" + value="{{progress/100}}" + class="form-control"> + {{progress}}</progress> + <span class="form-text text-muted">{{"%.2f" | format(progress)}}%</span> + </div> + + <input type="hidden" name="job_id" value="{{job_id}}" /> + + <button type="submit" class="btn btn-danger">Abort</button> + </form> +</div> + +<div class="row"> + {{errors_display(errors, "No errors found so far", "We have found the following errors so far", False)}} +</div> + +{%endblock%} diff --git a/uploader/templates/no_such_job.html b/uploader/templates/no_such_job.html new file mode 100644 index 0000000..42a2d48 --- /dev/null +++ b/uploader/templates/no_such_job.html @@ -0,0 +1,14 @@ +{%extends "base.html"%} + +{%block extrameta%} +<meta http-equiv="refresh" content="5;url={{url_for('entry.upload_file')}}"> +{%endblock%} + +{%block title%}No Such Job{%endblock%} + +{%block contents%} +<h1 class="heading">No Such Job: {{job_id}}</h2> + +<p>No job, with the id '<em>{{job_id}}</em>' was found!</p> + +{%endblock%} diff --git a/uploader/templates/parse_failure.html b/uploader/templates/parse_failure.html new file mode 100644 index 0000000..31f6be8 --- /dev/null +++ b/uploader/templates/parse_failure.html @@ -0,0 +1,26 @@ +{%extends "base.html"%} + +{%block title%}Worker Failure{%endblock%} + +{%block contents%} +<h1 class="heading">Worker Failure</h1> + +<p> + There was an error while parsing your file. +</p> + +<p> + Please notify the developers of this issue when you encounter it, + providing the information below. +</p> + +<h4>Debugging Information</h4> + +<ul> + <li><strong>job id</strong>: {{job["job_id"]}}</li> + <li><strong>filename</strong>: {{job["filename"]}}</li> + <li><strong>line number</strong>: {{job["line_number"]}}</li> + <li><strong>Progress</strong>: {{job["percent"]}} %</li> +</ul> + +{%endblock%} diff --git a/uploader/templates/parse_results.html b/uploader/templates/parse_results.html new file mode 100644 index 0000000..e2bf7f0 --- /dev/null +++ b/uploader/templates/parse_results.html @@ -0,0 +1,30 @@ +{%extends "base.html"%} +{%from "errors_display.html" import errors_display%} + +{%block title%}Parse Results{%endblock%} + +{%block contents%} +<h1 class="heading">{{job_name}}: parse results</h2> + +{%if user_aborted%} +<span class="alert-warning">Job aborted by the user</span> +{%endif%} + +{{errors_display(errors, "No errors found in the file", "We found the following errors", True)}} + +{%if errors | length == 0 and not user_aborted %} +<form method="post" action="{{url_for('dbinsert.select_platform')}}"> + <input type="hidden" name="job_id" value="{{job_id}}" /> + <input type="submit" value="update database" class="btn btn-primary" /> +</form> +{%endif%} + +{%if errors | length > 0 or user_aborted %} +<br /> +<a href="{{url_for('entry.upload_file')}}" title="Back to index page." + class="btn btn-primary"> + Go back +</a> +{%endif%} + +{%endblock%} diff --git a/uploader/templates/rqtl2/create-geno-dataset-success.html b/uploader/templates/rqtl2/create-geno-dataset-success.html new file mode 100644 index 0000000..1b50221 --- /dev/null +++ b/uploader/templates/rqtl2/create-geno-dataset-success.html @@ -0,0 +1,55 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Select Genotypes Dataset</h2> + +<div class="explainer"> + <p>You successfully created the genotype dataset with the following + information. + <dl> + <dt>ID</dt> + <dd>{{geno_dataset.id}}</dd> + + <dt>Name</dt> + <dd>{{geno_dataset.name}}</dd> + + <dt>Full Name</dt> + <dd>{{geno_dataset.fname}}</dd> + + <dt>Short Name</dt> + <dd>{{geno_dataset.sname}}</dd> + + <dt>Created On</dt> + <dd>{{geno_dataset.today}}</dd> + + <dt>Public?</dt> + <dd>{%if geno_dataset.public == 0%}No{%else%}Yes{%endif%}</dd> + </dl> + </p> +</div> + +<div class="row"> + <form id="frm-upload-rqtl2-bundle" + action="{{url_for('upload.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + <legend class="heading">select from existing genotype datasets</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" + value="{{geno_dataset.id}}" /> + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/create-probe-dataset-success.html b/uploader/templates/rqtl2/create-probe-dataset-success.html new file mode 100644 index 0000000..790d174 --- /dev/null +++ b/uploader/templates/rqtl2/create-probe-dataset-success.html @@ -0,0 +1,59 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Create ProbeSet Dataset</h2> + +<div class="row"> + <p>You successfully created the ProbeSet dataset with the following + information. + <dl> + <dt>Averaging Method</dt> + <dd>{{avgmethod.Name}}</dd> + + <dt>ID</dt> + <dd>{{dataset.datasetid}}</dd> + + <dt>Name</dt> + <dd>{{dataset.name2}}</dd> + + <dt>Full Name</dt> + <dd>{{dataset.fname}}</dd> + + <dt>Short Name</dt> + <dd>{{dataset.sname}}</dd> + + <dt>Created On</dt> + <dd>{{dataset.today}}</dd> + + <dt>DataScale</dt> + <dd>{{dataset.datascale}}</dd> + </dl> + </p> +</div> + +<div class="row"> + <form id="frm-upload-rqtl2-bundle" + action="{{url_for('upload.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + <legend class="heading">Create ProbeSet dataset</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + <input type="hidden" name="probe-study-id" value="{{study.Id}}" /> + <input type="hidden" name="probe-dataset-id" value="{{dataset.datasetid}}" /> + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/create-probe-study-success.html b/uploader/templates/rqtl2/create-probe-study-success.html new file mode 100644 index 0000000..d0ee508 --- /dev/null +++ b/uploader/templates/rqtl2/create-probe-study-success.html @@ -0,0 +1,49 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Create ProbeSet Study</h2> + +<div class="row"> + <p>You successfully created the ProbeSet study with the following + information. + <dl> + <dt>ID</dt> + <dd>{{study.id}}</dd> + + <dt>Name</dt> + <dd>{{study.name}}</dd> + + <dt>Full Name</dt> + <dd>{{study.fname}}</dd> + + <dt>Short Name</dt> + <dd>{{study.sname}}</dd> + + <dt>Created On</dt> + <dd>{{study.today}}</dd> + </dl> + </p> + + <form id="frm-upload-rqtl2-bundle" + action="{{url_for('upload.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + <legend class="heading">Create ProbeSet study</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="probe-study-id" value="{{study.studyid}}" /> + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/create-tissue-success.html b/uploader/templates/rqtl2/create-tissue-success.html new file mode 100644 index 0000000..5f2c5a0 --- /dev/null +++ b/uploader/templates/rqtl2/create-tissue-success.html @@ -0,0 +1,106 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Select Tissue</h2> + +<div class="row"> + <p>You have successfully added a new tissue, organ or biological material with + the following details:</p> +</div> + +<div class="row"> + {{flash_all_messages()}} + + <form id="frm-create-tissue-display" + method="POST" + action="#"> + <legend class="heading">Create Tissue</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + + <div class="form-group"> + <label>Name</label> + <label>{{tissue.TissueName}}</label> + </div> + + <div class="form-group"> + <label>Short Name</label> + <label>{{tissue.Short_Name}}</label> + </div> + + {%if tissue.BIRN_lex_ID%} + <div class="form-group"> + <label>BIRN Lex ID</label> + <label>{{tissue.BIRN_lex_ID}}</label> + </div> + {%endif%} + + {%if tissue.BIRN_lex_Name%} + <div class="form-group"> + <label>BIRN Lex Name</label> + <label>{{tissue.BIRN_lex_Name}}</label> + </div> + {%endif%} + </form> + + <div id="action-buttons" + style="width:65ch;display:inline-grid;column-gap:5px;"> + + <form id="frm-create-tissue-success-continue" + method="POST" + action="{{url_for('upload.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + style="display: inline; width: 100%; grid-column: 1 / 2; + padding-top: 0.5em; text-align: center; border: none; + background-color: inherit;"> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + + <button type="submit" class="btn btn-primary">continue</button> + </form> + </div> +</div> + +<div class="row"> + <p style="display:inline;width:100%;grid-column:2/3;text-align:center; + color:#336699;font-weight:bold;"> + OR + </p> +</div> + +<div class="row"> + <form id="frm-create-tissue-success-select-existing" + method="POST" + action="{{url_for('upload.rqtl2.select_tissue', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + style="display: inline; width: 100%; grid-column: 3 / 4; + padding-top: 0.5em; text-align: center; border: none; + background-color: inherit;"> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + + <button type="submit" class="btn btn-primary"> + select from existing tissues</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/index.html b/uploader/templates/rqtl2/index.html new file mode 100644 index 0000000..f3329c2 --- /dev/null +++ b/uploader/templates/rqtl2/index.html @@ -0,0 +1,36 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Data Upload{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 data upload</h1> + +<h2>R/qtl2 Upload</h2> + +<form method="POST" action="{{url_for('upload.rqtl2.select_species')}}" + id="frm-rqtl2-upload"> + <legend class="heading">upload R/qtl2 bundle</legend> + {{flash_messages("error-rqtl2")}} + + <div class="form-group"> + <label for="select:species" class="form-label">Species</label> + <select id="select:species" + name="species_id" + required="required" + class="form-control"> + <option value="">Select species</option> + {%for spec in species%} + <option value="{{spec.SpeciesId}}">{{spec.MenuName}}</option> + {%endfor%} + </select> + <small class="form-text text-muted"> + Data that you upload to the system should belong to a know species. + Here you can select the species that you wish to upload data for. + </small> + </div> + + <button type="submit" class="btn btn-primary" />submit</button> +</form> + +{%endblock%} diff --git a/uploader/templates/rqtl2/no-such-job.html b/uploader/templates/rqtl2/no-such-job.html new file mode 100644 index 0000000..b17004f --- /dev/null +++ b/uploader/templates/rqtl2/no-such-job.html @@ -0,0 +1,13 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 job status</h1> + +<h2>R/qtl2 Upload: No Such Job</h2> + +<p class="alert-danger">No job with ID {{jobid}} was found.</p> + +{%endblock%} diff --git a/uploader/templates/rqtl2/rqtl2-job-error.html b/uploader/templates/rqtl2/rqtl2-job-error.html new file mode 100644 index 0000000..9817518 --- /dev/null +++ b/uploader/templates/rqtl2/rqtl2-job-error.html @@ -0,0 +1,39 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 job status</h1> + +<h2>R/qtl2 Upload: Job Status</h2> + +<div class="explainer"> + <p>The processing of the R/qtl2 bundle you uploaded has failed. We have + provided some information below to help you figure out what the problem + could be.</p> + <p>If you find that you cannot figure out what the problem is on your own, + please contact the team running the system for assistance, providing the + following details: + <ul> + <li>R/qtl2 bundle you uploaded</li> + <li>This URL: <strong>{{request_url()}}</strong></li> + <li>(maybe) a screenshot of this page</li> + </ul> + </p> +</div> + +<h4>stdout</h4> +{{cli_output(job, "stdout")}} + +<h4>stderr</h4> +{{cli_output(job, "stderr")}} + +<h4>Log</h4> +<div class="cli-output"> + {%for msg in messages%} + {{msg}}<br /> + {%endfor%} +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/rqtl2-job-results.html b/uploader/templates/rqtl2/rqtl2-job-results.html new file mode 100644 index 0000000..4ecd415 --- /dev/null +++ b/uploader/templates/rqtl2/rqtl2-job-results.html @@ -0,0 +1,24 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 job status</h1> + +<h2>R/qtl2 Upload: Job Status</h2> + +<div class="explainer"> + <p>The processing of the R/qtl2 bundle you uploaded has completed + successfully.</p> + <p>You should now be able to use GeneNetwork to run analyses on your data.</p> +</div> + +<h4>Log</h4> +<div class="cli-output"> + {%for msg in messages%} + {{msg}}<br /> + {%endfor%} +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/rqtl2-job-status.html b/uploader/templates/rqtl2/rqtl2-job-status.html new file mode 100644 index 0000000..e896f88 --- /dev/null +++ b/uploader/templates/rqtl2/rqtl2-job-status.html @@ -0,0 +1,20 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Job Status{%endblock%} + +{%block extrameta%} +<meta http-equiv="refresh" content="3"> +{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 job status</h1> + +<h2>R/qtl2 Upload: Job Status</h2> + +<h4>Log</h4> +<div class="cli-output"> + <pre>{{"\n".join(messages)}}</pre> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/rqtl2-qc-job-error.html b/uploader/templates/rqtl2/rqtl2-qc-job-error.html new file mode 100644 index 0000000..90e8887 --- /dev/null +++ b/uploader/templates/rqtl2/rqtl2-qc-job-error.html @@ -0,0 +1,120 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block title%}R/qtl2 bundle: QC Job Error{%endblock%} + +{%macro errors_table(tableid, errors)%} +<table id="{{tableid}}" class="table error-table"> + <caption>{{caption}}</caption> + <thead> + <tr> + <th>Line</th> + <th>Column</th> + <th>Value</th> + <th>Message</th> + </tr> + </thead> + <tbody> + {%for error in errors%} + <tr> + <td>{{error.line}}</td> + <td>{{error.column}}</td> + <td>{{error.value}}</td> + <td>{{error.message}}</td> + </tr> + {%else%} + <tr> + <td colspan="4">No errors to display here.</td> + </tr> + {%endfor%} + </tbody> +</table> +{%endmacro%} + +{%block contents%} +<h1 class="heading">R/qtl2 bundle: QC job Error</h1> + +<div class="explainer"> + <p>The R/qtl2 bundle has failed some <emph>Quality Control</emph> checks.</p> + <p>We list below some of the errors that need to be fixed before the data can + be uploaded onto GeneNetwork.</p> +</div> + +{%if errorsgeneric | length > 0%} +<h2 class="heading">Generic Errors ({{errorsgeneric | length}})</h3> +<div class="explainer"> + We found the following generic errors in your R/qtl2 bundle: +</div> + +<h3>Missing Files</h3> +<div class="explainer"> + <p>These files are listed in the bundle's control file, but do not actually + exist in the bundle</p> +</div> +<table id="tbl-errors-missing-files" class="table error-table"> + <thead> + <tr> + <th>Control File Key</th> + <th>Bundle File Name</th> + <th>Message</th> + </tr> + </thead> + <tbody> + {%for error in (errorsgeneric | selectattr("type", "equalto", "MissingFile"))%} + <tr> + <td>{{error.controlfilekey}}</td> + <td>{{error.filename}}</td> + <td>{{error.message}}</td> + </tr> + {%endfor%} + </tbody> +</table> + +<h3>Other Generic Errors</h3> +{{errors_table("tbl-errors-generic", errorsgeneric| selectattr("type", "ne", "MissingFile"))}} +{%endif%} + +{%if errorsgeno | length > 0%} +<h2 class="heading">Geno Errors ({{errorsgeno | length}})</h3> +<div class="explainer"> + We found the following errors in the 'geno' file in your R/qtl2 bundle: +</div> +{{errors_table("tbl-errors-geno", errorsgeno[0:50])}} +{%endif%} + +{%if errorspheno | length > 0%} +<h2 class="heading">Pheno Errors ({{errorspheno | length}})</h3> +<div class="explainer"> + We found the following errors in the 'pheno' file in your R/qtl2 bundle: +</div> +{{errors_table("tbl-errors-pheno", errorspheno[0:50])}} +{%endif%} + +{%if errorsphenose | length > 0%} +<h2 class="heading">Phenose Errors ({{errorsphenose | length}})</h3> +<div class="explainer"> + We found the following errors in the 'phenose' file in your R/qtl2 bundle: +</div> +{{errors_table("tbl-errors-phenose", errorsphenose[0:50])}} +{%endif%} + +{%if errorsphenocovar | length > 0%} +<h2 class="heading">Phenocovar Errors ({{errorsphenocovar | length}})</h3> +<div class="explainer"> + We found the following errors in the 'phenocovar' file in your R/qtl2 bundle: +</div> +{{errorsphenocovar}} +{%endif%} + +<h4>stdout</h4> +{{cli_output(job, "stdout")}} + +<h4>stderr</h4> +{{cli_output(job, "stderr")}} + +<h4>Log</h4> +<div class="cli-output"> + <pre>{{"\n".join(messages)}}</pre> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/rqtl2-qc-job-results.html b/uploader/templates/rqtl2/rqtl2-qc-job-results.html new file mode 100644 index 0000000..59bc8cd --- /dev/null +++ b/uploader/templates/rqtl2/rqtl2-qc-job-results.html @@ -0,0 +1,66 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block title%}R/qtl2 bundle: QC job results{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 bundle: QC job results</h1> + +<div class="row"> + <p>The R/qtl2 bundle you uploaded has passed all automated quality-control + checks successfully.</p> + <p>You may now continue to load the data into GeneNetwork for the bundle, with + the following details:</p> +</div> + +<div class="row"> + <form id="form-qc-job-results" + action="{{url_for('upload.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.Id)}}" + method="POST"> + <div class="form-group"> + <legend>Species</legend> + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + + <span class="form-label">Name</span> + <span class="form-text">{{species.Name | capitalize}}</span> + + <span class="form-label">Scientific</span> + <span class="form-text">{{species.FullName | capitalize}}</span> + </div> + + <div class="form-group"> + <legend>population</legend> + <input type="hidden" name="population_id" value="{{population.Id}}" /> + + <span class="form-label">Name</span> + <span class="form-text">{{population.InbredSetName}}</span> + + <span class="form-label">Full Name</span> + <span class="form-text">{{population.FullName}}</span> + + <span class="form-label">Genetic Type</span> + <span class="form-text">{{population.GeneticType}}</span> + + <span class="form-label">Description</span> + <span class="form-text">{{population.Description or "-"}}</span> + </div> + + <div class="form-group"> + <legend>R/qtl2 Bundle File</legend> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2bundle}}" /> + <input type="hidden" name="original-filename" value="{{rqtl2bundleorig}}" /> + + <span class="form-label">Original Name</span> + <span class="form-text">{{rqtl2bundleorig}}</span> + + <span class="form-label">Internal Name</span> + <span class="form-text">{{rqtl2bundle[0:25]}}…</span> + </div> + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/rqtl2-qc-job-status.html b/uploader/templates/rqtl2/rqtl2-qc-job-status.html new file mode 100644 index 0000000..f4a6266 --- /dev/null +++ b/uploader/templates/rqtl2/rqtl2-qc-job-status.html @@ -0,0 +1,41 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Job Status{%endblock%} + +{%block extrameta%} +<meta http-equiv="refresh" content="3"> +{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 bundle: QC job status</h1> + +{%if geno_percent%} +<p> + <h2>Checking 'geno' file:</h2> + <progress id="prg-geno-checking" value="{{geno_percent}}" max="100"> + {{geno_percent}}%</progress> + {{geno_percent}}%</p> +{%endif%} + +{%if pheno_percent%} +<p> + <h2>Checking 'pheno' file:</h2> + <progress id="prg-pheno-checking" value="{{pheno_percent}}" max="100"> + {{pheno_percent}}%</progress> + {{pheno_percent}}%</p> +{%endif%} + +{%if phenose_percent%} +<p> + <h2>Checking 'phenose' file:</h2> + <progress id="prg-phenose-checking" value="{{phenose_percent}}" max="100"> + {{phenose_percent}}%</progress> + {{phenose_percent}}%</p> +{%endif%} + +<h4>Log</h4> +<div class="cli-output"> + <pre>{{"\n".join(messages)}}</pre> +</div> +{%endblock%} diff --git a/uploader/templates/rqtl2/rqtl2-qc-job-success.html b/uploader/templates/rqtl2/rqtl2-qc-job-success.html new file mode 100644 index 0000000..2861a04 --- /dev/null +++ b/uploader/templates/rqtl2/rqtl2-qc-job-success.html @@ -0,0 +1,37 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}R/qtl2 Bundle: Quality Control Successful{%endblock%} + +{%block contents%} +<h2 class="heading">R/qtl2 Bundle: Quality Control Successful</h2> + +<div class="row"> + <p>The R/qtl2 bundle you uploaded has passed <emph>all</emph> quality control + checks successfully, and is now ready for uploading into the database.</p> + <p>Click "Continue" below to proceed.</p> +</div> + +<!-- + The "action" on this form takes us to the next step, where we can + select all the other data necessary to enter the data into the database. + --> +<div class="row"> + <form id="frm-upload-rqtl2-bundle" + action="{{url_for('upload.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + {{flash_all_messages()}} + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/select-geno-dataset.html b/uploader/templates/rqtl2/select-geno-dataset.html new file mode 100644 index 0000000..873f9c3 --- /dev/null +++ b/uploader/templates/rqtl2/select-geno-dataset.html @@ -0,0 +1,144 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Select Genotypes Dataset</h2> + +<div class="row"> + <p>Your R/qtl2 files bundle contains a "geno" specification. You will + therefore need to select from one of the existing Genotype datasets or + create a new one.</p> + <p>This is the dataset where your data will be organised under.</p> +</div> + +<div class="row"> + <form id="frm-upload-rqtl2-bundle" + action="{{url_for('upload.rqtl2.select_geno_dataset', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + <legend class="heading">select from existing genotype datasets</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + + {{flash_messages("error-rqtl2-select-geno-dataset")}} + + <div class="form-group"> + <legend>Datasets</legend> + <label for="select:geno-datasets" class="form-label">Dataset</label> + <select id="select:geno-datasets" + name="geno-dataset-id" + required="required" + {%if datasets | length == 0%} + disabled="disabled" + {%endif%} + class="form-control" + aria-describedby="help-geno-dataset-select-dataset"> + <option value="">Select dataset</option> + {%for dset in datasets%} + <option value="{{dset['Id']}}">{{dset["Name"]}} ({{dset["FullName"]}})</option> + {%endfor%} + </select> + <span id="help-geno-dataset-select-dataset" class="form-text text-muted"> + Select from the existing genotype datasets for species + {{species.SpeciesName}} ({{species.FullName}}). + </span> + </div> + + <button type="submit" class="btn btn-primary">select dataset</button> + </form> +</div> + +<div class="row"> + <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> +</div> + +<div class="row"> + <form id="frm-upload-rqtl2-bundle" + action="{{url_for('upload.rqtl2.create_geno_dataset', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + <legend class="heading">create a new genotype dataset</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + + {{flash_messages("error-rqtl2-create-geno-dataset")}} + + <div class="form-group"> + <label for="txt:dataset-name" class="form-label">Name</label> + <input type="text" + id="txt:dataset-name" + name="dataset-name" + maxlength="100" + required="required" + class="form-control" + aria-describedby="help-geno-dataset-name" /> + <span id="help-geno-dataset-name" class="form-text text-muted"> + Provide the new name for the genotype dataset, e.g. "BXDGeno" + </span> + </div> + + <div class="form-group"> + <label for="txt:dataset-fullname" class="form-label">Full Name</label> + <input type="text" + id="txt:dataset-fullname" + name="dataset-fullname" + required="required" + maxlength="100" + class="form-control" + aria-describedby="help-geno-dataset-fullname" /> + + <span id="help-geno-dataset-fullname" class="form-text text-muted"> + Provide a longer name that better describes the genotype dataset, e.g. + "BXD Genotypes" + </span> + </div> + + <div class="form-group"> + <label for="txt:dataset-shortname" class="form-label">Short Name</label> + <input type="text" + id="txt:dataset-shortname" + name="dataset-shortname" + maxlength="100" + class="form-control" + aria-describedby="help-geno-dataset-shortname" /> + + <span id="help-geno-dataset-shortname" class="form-text text-muted"> + Provide a short name for the genotype dataset. This is optional. If not + provided, we'll default to the same value as the "Name" above. + </span> + </div> + + <div class="form-group"> + <input type="checkbox" + id="chk:dataset-public" + name="dataset-public" + checked="checked" + class="form-check" + aria-describedby="help-geno-datasent-public" /> + <label for="chk:dataset-public" class="form-check-label">Public?</label> + + <span id="help-geno-dataset-public" class="form-text text-muted"> + Specify whether the dataset will be available publicly. Check to make the + dataset publicly available and uncheck to limit who can access the dataset. + </span> + </div> + + <button type="submit" class="btn btn-primary">create new dataset</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/select-population.html b/uploader/templates/rqtl2/select-population.html new file mode 100644 index 0000000..37731f0 --- /dev/null +++ b/uploader/templates/rqtl2/select-population.html @@ -0,0 +1,136 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Select Grouping/Population{%endblock%} + +{%block contents%} +<h1 class="heading">Select grouping/population</h1> + +<div class="explainer"> + <p>The data is organised in a hierarchical form, beginning with + <em>species</em> at the very top. Under <em>species</em> the data is + organised by <em>population</em>, sometimes referred to as <em>grouping</em>. + (In some really old documents/systems, you might see this referred to as + <em>InbredSet</em>.)</p> + <p>In this section, you get to define what population your data is to be + organised by.</p> +</div> + +<form method="POST" + action="{{url_for('upload.rqtl2.select_population', species_id=species.SpeciesId)}}"> + <legend class="heading">select grouping/population</legend> + {{flash_messages("error-select-population")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + + <div class="form-group"> + <label for="select:inbredset" class="form-label">population</label> + <select id="select:inbredset" + name="inbredset_id" + required="required" + class="form-control"> + <option value="">Select a grouping/population</option> + {%for pop in populations%} + <option value="{{pop.InbredSetId}}"> + {{pop.InbredSetName}} ({{pop.FullName}})</option> + {%endfor%} + </select> + <span class="form-text text-muted">If you are adding data to an already existing + population, simply pick the population from this drop-down selector. If + you cannot find your population from this list, try the form below to + create a new one..</span> + </div> + + <button type="submit" class="btn btn-primary" />select population</button> +</form> + +<p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> + +<form method="POST" + action="{{url_for('upload.rqtl2.create_population', species_id=species.SpeciesId)}}"> + <legend class="heading">create new grouping/population</legend> + {{flash_messages("error-create-population")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + + <div class="form-group"> + <legend class="heading">mandatory</legend> + + <div class="form-group"> + <label for="txt:inbredset-name" class="form-label">name</label> + <input id="txt:inbredset-name" + name="inbredset_name" + type="text" + required="required" + maxlength="30" + placeholder="Enter grouping/population name" + class="form-control" /> + <span class="form-text text-muted">This is a short name that identifies the + population. Useful for menus, and quick scanning.</span> + </div> + + <div class="form-group"> + <label for="txt:" class="form-label">full name</label> + <input id="txt:inbredset-fullname" + name="inbredset_fullname" + type="text" + required="required" + maxlength="100" + placeholder="Enter the grouping/population's full name" + class="form-control" /> + <span class="form-text text-muted">This can be the same as the name above, or can + be longer. Useful for documentation, and human communication.</span> + </div> + </div> + + <div class="form-group"> + <legend class="heading">optional</legend> + + <div class="form-group"> + <label for="num:public" class="form-label">public?</label> + <select id="num:public" + name="public" + class="form-control"> + <option value="0">0 - Only accessible to authorised users</option> + <option value="1">1 - Publicly accessible to all users</option> + <option value="2" selected> + 2 - Publicly accessible to all users</option> + </select> + <span class="form-text text-muted">This determines whether the + population/grouping will appear on the menus for users.</span> + </div> + + <div class="form-group"> + <label for="txt:inbredset-family" class="form-label">family</label> + <input id="txt:inbredset-family" + name="inbredset_family" + type="text" + placeholder="I am not sure what this is about." + class="form-control" /> + <span class="form-text text-muted">I do not currently know what this is about. + This is a failure on my part to figure out what this is and provide a + useful description. Please feel free to remind me.</span> + </div> + + <div class="form-group"> + <label for="txtarea:" class="form-label">Description</label> + <textarea id="txtarea:description" + name="description" + rows="5" + placeholder="Enter a description of this grouping/population" + class="form-control"></textarea> + <span class="form-text text-muted"> + A long-form description of what the population consists of. Useful for + humans.</span> + </div> + </div> + + <button type="submit" class="btn btn-primary" /> + create grouping/population</button> +</form> + +{%endblock%} + + +{%block javascript%} +{%endblock%} diff --git a/uploader/templates/rqtl2/select-probeset-dataset.html b/uploader/templates/rqtl2/select-probeset-dataset.html new file mode 100644 index 0000000..26f52ed --- /dev/null +++ b/uploader/templates/rqtl2/select-probeset-dataset.html @@ -0,0 +1,191 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Phenotype(ProbeSet) Dataset</h2> + +<div class="row"> + <p>The R/qtl2 bundle you uploaded contains (a) "<strong>pheno</strong>" + file(s). This data needs to be organised under a dataset.</p> + <p>This page gives you the ability to do that.</p> +</div> + +{%if datasets | length > 0%} +<div class="row"> + <form method="POST" + action="{{url_for('upload.rqtl2.select_probeset_dataset', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:select-probeset-dataset"> + <legend class="heading">Select from existing ProbeSet datasets</legend> + {{flash_messages("error-rqtl2")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" /> + + <div class="form-group"> + <label class="form-label" for="select:probe-dataset">Dataset</label> + <select id="select:probe-dataset" + name="probe-dataset-id" + required="required" + {%if datasets | length == 0%}disabled="disabled"{%endif%} + class="form-control" + aria-describedby="help-probe-dataset"> + <option value="">Select a dataset</option> + {%for dataset in datasets%} + <option value={{dataset.Id}}> + {{dataset.Name}} + {%if dataset.FullName%} + -- ({{dataset.FullName}}) + {%endif%} + </option> + {%endfor%} + </select> + + <span id="help-probe-dataset" class="form-text text-muted"> + Select from existing ProbeSet datasets.</span> + </div> + + <button type="submit" class="btn btn-primary" />select dataset</button> +</form> +</div> + +<div class="row"> + <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> +</div> +{%endif%} + +<div class="row"> + <p>Create an entirely new ProbeSet dataset for your data.</p> +</div> + +<div class="row"> + <form method="POST" + action="{{url_for('upload.rqtl2.create_probeset_dataset', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:create-probeset-dataset"> + <legend class="heading">Create a new ProbeSet dataset</legend> + {{flash_messages("error-rqtl2-create-probeset-dataset")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" /> + + <div class="form-group"> + <label class="form-label" for="select:average">averaging method</label> + <select id="select:average" + name="averageid" + required="required" + class="form-control" + aria-describedby="help-average"> + <option value="">Select averaging method</option> + {%for avgmethod in avgmethods%} + <option value="{{avgmethod.Id}}"> + {{avgmethod.Name}} + {%if avgmethod.Normalization%} + ({{avgmethod.Normalization}}) + {%endif%} + </option> + {%endfor%} + </select> + + <span id="help-average" class="form-text text-muted"> + Select the averaging method used for your data. + </span> + </div> + + <div class="form-group"> + <label class="form-label" for="txt:datasetname">Name</label> + <input type="text" id="txt:datasetname" name="datasetname" + required="required" + maxlength="40" + title="Name of the dataset, e.g 'BXDMicroArray_ProbeSet_June03'. (Required)" + class="form-control" + aria-describedby="help-dataset-name" /> + + <span id="help-dataset-name" class="form-text text-muted"> + Provide a name for the dataset e.g. "BXDMicroArray_ProbeSet_June03". This + is mandatory <strong>MUST</strong> be provided. + </span> + </div> + + <div class="form-group"> + <label class="form-label" for="txt:datasetfullname">Full Name</label> + <input type="text" id="txt:datasetfullname" name="datasetfullname" + required="required" + maxlength="100" + title="A longer name for the dataset, e.g. 'UTHSC Brain mRNA U74Av2 (Jun03) MAS5'. (Required)" + class="form-control" + aria-describedby="help-dataset-fullname" /> + + <span id="help-dataset-fullname" class="form-text text-muted"> + Provide a longer, more descriptive name for the dataset e.g. + "UTHSC Brain mRNA U74Av2 (Jun03) MAS5". This is mandatory and + <strong>MUST</strong> be provided. + </span> + </div> + + <div class="form-group"> + <label class="form-label" for="txt:datasetshortname">Short Name</label> + <input type="text" id="txt:datasetshortname" name="datasetshortname" + maxlength="100" + title="An abbreviated name for the dataset, e.g 'Br_U_0603_M'. (Optional)" + class="form-control" + aria-describedby="help-dataset-shortname" /> + + <span id="help-dataset-shortname" class="form-text text-muted"> + Provide a longer, more descriptive name for the dataset e.g. "Br_U_0603_M". + This is optional. + </span> + </div> + + <div class="form-check"> + <input type="checkbox" id="chk:public" name="datasetpublic" + checked="checked" + title="Whether or not the dataset is accessible by the general public." + class="form-check-input" + aria-describedby="help-public" /> + <label class="form-check-label" for="chk:datasetpublic">Public?</label> + + <span id="help-public" class="form-text text-muted"> + Check to specify that the dataset will be publicly available. Uncheck to + limit access to the dataset. + </span> + </div> + + <div class="form-group"> + <label class="form-label" for="select:datasetdatascale">Data Scale</label> + <select id="select:datasetdatascale" + name="datasetdatascale" + required="required" + class="form-control" + aria-describedby="help-dataset-datascale"> + <option value="log2" selected="selected">log2</option> + <option value="z_score">z_score</option> + <option value="log2_ratio">log2_ratio</option> + <option value="linear">linear</option> + <option value="linear_positive">linear_positive</option> + </select> + + <span id="help-dataset-datascale" class="form-text text-muted"> + Select from a list of scaling methods. + </span> + </div> + + <button type="submit" class="btn btn-primary">create dataset</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/select-probeset-study-id.html b/uploader/templates/rqtl2/select-probeset-study-id.html new file mode 100644 index 0000000..b9bf52e --- /dev/null +++ b/uploader/templates/rqtl2/select-probeset-study-id.html @@ -0,0 +1,143 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages %} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Phenotype(ProbeSet) Study</h2> + +<div class="row"> + <p>The R/qtl2 bundle you uploaded contains (a) "<strong>pheno</strong>" + file(s). This data needs to be organised under a study.</p> + <p>In this page, you can either select from a existing dataset:</p> + + <form method="POST" + action="{{url_for('upload.rqtl2.select_probeset_study', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:select-probeset-study"> + <legend class="heading">Select from existing ProbeSet studies</legend> + {{flash_messages("error-rqtl2-select-probeset-study")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + + <div> + <label for="select:probe-study" class="form-label">Study</label> + <select id="select:probe-study" + name="probe-study-id" + required="required" + aria-describedby="help-select-probeset-study" + {%if studies | length == 0%}disabled="disabled"{%endif%} + class="form-control"> + <option value="">Select a study</option> + {%for study in studies%} + <option value={{study.Id}}> + {{study.Name}} + {%if study.FullName%} + -- ({{study.FullName}}) + {%endif%} + </option> + {%endfor%} + </select> + <small id="help-select-probeset-study" class="form-text text-muted"> + Select from existing ProbeSet studies. + </small> + </div> + + <button type="submit" class="btn btn-primary">select study</button> + </form> +</div> + +<div class="row"> + <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> +</div> + +<div class="row"> + + <p>Create a new ProbeSet dataset below:</p> + + <form method="POST" + action="{{url_for('upload.rqtl2.create_probeset_study', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:create-probeset-study"> + <legend class="heading">Create new ProbeSet study</legend> + + {{flash_messages("error-rqtl2-create-probeset-study")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> + + <div> + <label for="select:platform" class="form-label">Platform</label> + <select id="select:platform" + name="platformid" + required="required" + aria-describedby="help-select-platform" + {%if platforms | length == 0%}disabled="disabled"{%endif%} + class="form-control"> + <option value="">Select a platform</option> + {%for platform in platforms%} + <option value="{{platform.GeneChipId}}"> + {{platform.GeneChipName}} ({{platform.Name}}) + </option> + {%endfor%} + </select> + <small id="help-select-platform" class="form-text text-muted"> + Select from a list of known genomics platforms. + </small> + </div> + + <div class="form-group"> + <label for="txt:studyname" class="form-label">Study name</label> + <input type="text" id="txt:studyname" name="studyname" + placeholder="Name of the study. (Required)" + required="required" + maxlength="100" + class="form-control" /> + <span class="form-text text-muted" id="help-study-name"> + Provide a name for the study.</span> + </div> + + <div class="form-group"> + <label for="txt:studyfullname" class="form-label">Full Study Name</label> + <input type="text" + id="txt:studyfullname" + name="studyfullname" + placeholder="Longer name of the study. (Optional)" + maxlength="100" + class="form-control" /> + <span class="form-text text-muted" id="help-study-full-name"> + Provide a longer, more descriptive name for the study. This is optional + and you can leave it blank. + </span> + </div> + + <div class="form-group"> + <label for="txt:studyshortname" class="form-label">Short Study Name</label> + <input type="text" + id="txt:studyshortname" + name="studyshortname" + placeholder="Shorter name of the study. (Optional)" + maxlength="100" + class="form-control" /> + <span class="form-text text-muted" id="help-study-short-name"> + Provide a shorter name for the study. This is optional and you can leave + it blank. + </span> + </div> + + <button type="submit" class="btn btn-primary">create study</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/select-tissue.html b/uploader/templates/rqtl2/select-tissue.html new file mode 100644 index 0000000..34e1758 --- /dev/null +++ b/uploader/templates/rqtl2/select-tissue.html @@ -0,0 +1,115 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Tissue</h2> + +<div class="row"> + <p>The data you are uploading concerns a tissue, cell, organ, or other + biological material used in an experiment.</p> + <p>Select the appropriate biological material below</p> +</div> + +{%if tissues | length > 0%} +<div class="row"> + <form method="POST" + action="{{url_for('upload.rqtl2.select_tissue', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:select-probeset-dataset"> + <legend class="heading">Select from existing ProbeSet datasets</legend> + {{flash_messages("error-select-tissue")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + + <div class="form-group"> + <label class="form-label" for="select-tissue">Tissue</label> + <select id="select-tissue" + name="tissueid" + required="required" + {%if tissues | length == 0%}disabled="disabled"{%endif%} + class="form-control" + aria-describedby="help-select-tissue"> + <option value="">Select a tissue</option> + {%for tissue in tissues%} + <option value={{tissue.Id}}> + {{tissue.Name}} + {%if tissue.Short_Name%} + -- ({{tissue.Short_Name}}) + {%endif%} + </option> + {%endfor%} + </select> + + <span id="help-select-tissue" class="form-text text-muted"> + Select from existing biological material.</span> + </div> + + <button type="submit" class="btn btn-primary">use selected</button> + </form> +</div> + +<div class="row"> + <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> +</div> +{%endif%} + +<div class="row"> + <p>If you cannot find the biological material in the drop-down above, add it + to the system below.</p> + + <form method="POST" + action="{{url_for('upload.rqtl2.create_tissue', + species_id=species.SpeciesId, population_id=population.Id)}}" + id="frm:create-probeset-dataset"> + <legend class="heading">Add new tissue, organ or biological material</legend> + {{flash_messages("error-create-tissue")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + + <div class="form-group"> + <label class="form-label" for="tissue-name">name</label> + <input type="text" + id="txt-tissuename" + name="tissuename" + required="required" + title = "A name to identify the tissue, organ or biological material." + class="form-control" + aria-describedby="help-tissue-name" /> + + <span class="form-text text-muted" id="help-tissue-name"> + A name to identify the tissue, organ or biological material. + </span> + </div> + + <div class="form-group"> + <label for="txt-shortname" class="form-label">short name</label> + <input type="text" id="txt-tissueshortname" name="tissueshortname" + required="required" + maxlength="7" + title="A short name (e.g. 'Mam') for the biological material." + class="form-control" + aria-describedby="help-tissue-short-name" /> + + <span class="form-text text-muted" id="help-tissue-short-name"> + Provide a short name for the tissue, organ or biological material used in + the experiment. + </span> + </div> + + <button type="submit" class="btn btn-primary" />add new material</button> +</form> +</div> + +{%endblock%} diff --git a/uploader/templates/rqtl2/summary-info.html b/uploader/templates/rqtl2/summary-info.html new file mode 100644 index 0000000..1be87fa --- /dev/null +++ b/uploader/templates/rqtl2/summary-info.html @@ -0,0 +1,65 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Summary</h2> + +<div class="row"> + <p>This is the information you have provided to accompany the R/qtl2 bundle + you have uploaded. Please verify the information is correct before + proceeding.</p> +</div> + +<div class="row"> + <dl> + <dt>Species</dt> + <dd>{{species.SpeciesName}} ({{species.FullName}})</dd> + + <dt>Population</dt> + <dd>{{population.InbredSetName}}</dd> + + {%if geno_dataset%} + <dt>Genotype Dataset</dt> + <dd>{{geno_dataset.Name}} ({{geno_dataset.FullName}})</dd> + {%endif%} + + {%if tissue%} + <dt>Tissue</dt> + <dd>{{tissue.TissueName}} ({{tissue.Name}}, {{tissue.Short_Name}})</dd> + {%endif%} + + {%if probe_study%} + <dt>ProbeSet Study</dt> + <dd>{{probe_study.Name}} ({{probe_study.FullName}})</dd> + {%endif%} + + {%if probe_dataset%} + <dt>ProbeSet Dataset</dt> + <dd>{{probe_dataset.Name2}} ({{probe_dataset.FullName}})</dd> + {%endif%} + </dl> +</div> + +<div class="row"> + <form id="frm:confirm-rqtl2bundle-details" + action="{{url_for('upload.rqtl2.confirm_bundle_details', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + <legend class="heading">Create ProbeSet dataset</legend> + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> + <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> + <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" /> + <input type="hidden" name="probe-dataset-id" value="{{probe_dataset.Id}}" /> + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> +{%endblock%} diff --git a/uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html new file mode 100644 index 0000000..07c240f --- /dev/null +++ b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html @@ -0,0 +1,276 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "upload_progress_indicator.html" import upload_progress_indicator%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +{%macro rqtl2_file_help()%} +<span class="form-text text-muted"> + <p> + Provide a valid R/qtl2 zip file here. In particular, ensure your zip bundle + contains exactly one control file and the corresponding files mentioned in + the control file. + </p> + <p> + The control file can be either a YAML or JSON file. <em>ALL</em> other data + files in the zip bundle should be CSV files. + </p> + <p>See the + <a href="https://kbroman.org/qtl2/assets/vignettes/input_files.html" + target="_blank"> + R/qtl2 file format specifications + </a> + for more details. + </p> +</span> +{%endmacro%} +{{upload_progress_indicator()}} + +<div id="resumable-file-display-template" + class="panel panel-info" + style="display: none"> + <div class="panel-heading"></div> + <div class="panel-body"></div> +</div> + + +<h2 class="heading">Upload R/qtl2 Bundle</h2> + +<div id="resumable-drop-area" + style="display:none;background:#eeeeee;min-height:12em;border-radius:0.5em;padding:1em;"> + <p> + <a id="resumable-browse-button" href="#" + class="btn btn-info">Browse</a> + </p> + <p class="form-text text-muted"> + You can drag and drop your file here, or click the browse button. + Click on the file to remove it. + </p> + {{rqtl2_file_help()}} + <div id="resumable-selected-files" + style="display:flex;flex-direction:row;flex-wrap: wrap;justify-content:space-around;gap:10px 20px;"></div> + <div id="resumable-class-buttons" style="text-align: right;"> + <button id="resumable-upload-button" + class="btn btn-primary" + style="display: none">start upload</button> + <button id="resumable-cancel-upload-button" + class="btn btn-danger" + style="display: none">cancel upload</button> + </div> + <div id="resumable-progress-bar" class="progress" style="display: none"> + <div class="progress-bar" + role="progress-bar" + aria-valuenow="60" + aria-valuemin="0" + aria-valuemax="100" + style="width: 0%;"> + Uploading: 60% + </div> + </div> +</div> + +<form id="frm-upload-rqtl2-bundle" + action="{{url_for('upload.rqtl2.upload_rqtl2_bundle', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data" + data-resumable-target="{{url_for( + 'upload.rqtl2.upload_rqtl2_bundle_chunked_post', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}"> + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + + {{flash_all_messages()}} + + <div class="form-group"> + <legend class="heading">file upload</legend> + <label for="file-rqtl2-bundle" class="form-label">R/qtl2 bundle</label> + <input type="file" id="file-rqtl2-bundle" name="rqtl2_bundle_file" + accept="application/zip, .zip" + required="required" + class="form-control" /> + {{rqtl2_file_help()}} + </div> + + <button type="submit" + class="btn btn-primary" + data-toggle="modal" + data-target="#upload-progress-indicator">upload R/qtl2 bundle</button> +</form> + +{%endblock%} + +{%block javascript%} +<script src="{{url_for('base.node_modules', + filename='resumablejs/resumable.js')}}"></script> +<script type="text/javascript" src="/static/js/upload_progress.js"></script> +<script type="text/javascript"> + function readBinaryFile(file) { + return new Promise((resolve, reject) => { + var _reader = new FileReader(); + _reader.onload = (event) => {resolve(_reader.result);}; + _reader.readAsArrayBuffer(file); + }); + } + + function computeFileChecksum(file) { + return readBinaryFile(file) + .then((content) => { + return window.crypto.subtle.digest( + "SHA-256", new Uint8Array(content)); + }).then((digest) => { + return Uint8ArrayToHex(new Uint8Array(digest)) + }); + } + + function Uint8ArrayToHex(arr) { + var toHex = (val) => { + _hex = val.toString(16); + if(_hex.length < 2) { + return "0" + val; + } + return _hex; + }; + _hexstr = "" + arr.forEach((val) => {_hexstr += toHex(val)}); + return _hexstr + } + + var r = Resumable({ + target: $("#frm-upload-rqtl2-bundle").attr("data-resumable-target"), + fileType: ["zip"], + maxFiles: 1, + forceChunkSize: true, + generateUniqueIdentifier: (file, event) => { + return computeFileChecksum(file).then((checksum) => { + var _relativePath = (file.webkitRelativePath + || file.relativePath + || file.fileName + || file.name); + return checksum + "-" + _relativePath.replace( + /[^a-zA-Z0-9_-]/img, ""); + }); + } + }); + + if(r.support) { + //Hide form and display drag&drop UI + $("#frm-upload-rqtl2-bundle").css("display", "none"); + $("#resumable-drop-area").css("display", "block"); + + // Define UI elements for browse and drag&drop + r.assignDrop(document.getElementById("resumable-drop-area")); + r.assignBrowse(document.getElementById("resumable-browse-button")); + + // Event handlers + + function display_files(files) { + displayArea = $("#resumable-selected-files") + displayArea.empty(); + files.forEach((file) => { + var displayElement = $( + "#resumable-file-display-template").clone(); + displayElement.removeAttr("id"); + displayElement.css("display", ""); + displayElement.find(".panel-heading").text(file.fileName); + list = $("<ul></ul>"); + list.append($("<li><strong>Name</strong>: " + + (file.name + || file.fileName + || file.relativePath + || file.webkitRelativePath) + + "</li>")); + list.append($("<li><strong>Size</strong>: " + + (file.size / (1024*1024)).toFixed(2) + + " MB</li>")); + list.append($("<li><strong>Unique Identifier</strong>: " + + file.uniqueIdentifier + "</li>")); + list.append($("<li><strong>Mime</strong>: " + + file.file.type + + "</li>")); + displayElement.find(".panel-body").append(list); + displayElement.appendTo("#resumable-selected-files"); + }); + } + + r.on("filesAdded", function(files) { + display_files(files); + $("#resumable-upload-button").css("display", ""); + $("#resumable-upload-button").on("click", (event) => { + r.upload(); + }); + }); + + r.on("uploadStart", (event) => { + $("#resumable-upload-button").css("display", "none"); + $("#resumable-cancel-upload-button").css("display", ""); + $("#resumable-cancel-upload-button").on("click", (event) => { + r.files.forEach((file) => { + if(file.isUploading()) { + file.abort(); + } + }); + $("#resumable-cancel-upload-button").css("display", "none"); + $("#resumable-upload-button").on("click", (event) => { + r.files.forEach((file) => {file.retry();}); + }); + $("#resumable-upload-button").css("display", ""); + }); + }); + + r.on("progress", () => { + var progress = (r.progress() * 100).toFixed(2); + var pbar = $("#resumable-progress-bar > .progress-bar"); + $("#resumable-progress-bar").css("display", ""); + pbar.css("width", progress+"%"); + pbar.attr("aria-valuenow", progress); + pbar.text("Uploading: " + progress + "%"); + }) + + r.on("fileSuccess", (file, message) => { + if(message != "OK") { + var uri = (window.location.protocol + + "//" + + window.location.host + + message); + window.location.replace(uri); + } + }); + + r.on("error", (message, file) => { + filename = (file.webkitRelativePath + || file.relativePath + || file.fileName + || file.name); + jsonmsg = JSON.parse(message); + alert("There was an error while uploading your file '" + + filename + + "'. The error message was:\n\n\t" + + jsonmsg.error + + " (" + + jsonmsg.statuscode + + "): " + jsonmsg.message); + }) + } else { + setup_upload_handlers( + "frm-upload-rqtl2-bundle", make_data_uploader( + function (form) { + var formdata = new FormData(); + formdata.append( + "species_id", + form.querySelector('input[name="species_id"]').value); + formdata.append( + "population_id", + form.querySelector('input[name="population_id"]').value); + formdata.append( + "rqtl2_bundle_file", + form.querySelector("#file-rqtl2-bundle").files[0]); + return formdata; + })); + } +</script> +{%endblock%} diff --git a/uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html new file mode 100644 index 0000000..93b1dc9 --- /dev/null +++ b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html @@ -0,0 +1,33 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Upload R/qtl2 Bundle{%endblock%} + +{%block contents%} +<h2 class="heading">Upload R/qtl2 Bundle</h2> + +<div class="row"> + <p>You have successfully uploaded the zipped bundle of R/qtl2 files.</p> + <p>The next step is to select the various extra information we need to figure + out what to do with the data. You will select/create the relevant studies + and/or datasets to organise the data in the steps that follow.</p> + <p>Click "Continue" below to proceed.</p> + + <form id="frm-upload-rqtl2-bundle" + action="{{url_for('upload.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + {{flash_all_messages()}} + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + + <button type="submit" class="btn btn-primary">continue</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/samples/select-population.html b/uploader/templates/samples/select-population.html new file mode 100644 index 0000000..da19ddc --- /dev/null +++ b/uploader/templates/samples/select-population.html @@ -0,0 +1,99 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Select Grouping/Population{%endblock%} + +{%block contents%} +<h1 class="heading">Select grouping/population</h1> + +<div> + <p>We organise the samples/cases/strains in a hierarchichal form, starting + with <strong>species</strong> at the very top. Under species, we have a + grouping in terms of the relevant population + (e.g. Inbred populations, cell tissue, etc.)</p> +</div> + +<form method="POST" action="{{url_for('samples.select_population', + species_id=species.SpeciesId)}}"> + <legend class="heading">select grouping/population</legend> + {{flash_messages("error-select-population")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + + <div class="form-group"> + <label for="select:inbredset" class="form-label">grouping/population</label> + <select id="select:inbredset" + name="inbredset_id" + required="required" + class="form-control"> + <option value="">Select a grouping/population</option> + {%for pop in populations%} + <option value="{{pop.InbredSetId}}"> + {{pop.InbredSetName}} ({{pop.FullName}})</option> + {%endfor%} + </select> + </div> + + <button type="submit" class="btn btn-primary">select population</button> +</form> + +<p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> + +<form method="POST" action="{{url_for('samples.create_population', + species_id=species.SpeciesId)}}"> + <legend class="heading">create new grouping/population</legend> + {{flash_messages("error-create-population")}} + + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <div class="form-group"> + <legend>mandatory</legend> + + <label for="txt:inbredset-name" class="form-label">name</label> + <input id="txt:inbredset-name" + name="inbredset_name" + type="text" + required="required" + placeholder="Enter grouping/population name" + class="form-control" /> + + <label for="txt:" class="form-label">full name</label> + <input id="txt:inbredset-fullname" + name="inbredset_fullname" + type="text" + required = "required" + placeholder="Enter the grouping/population's full name" + class="form-control" /> + </div> + <div class="form-group"> + <legend>Optional</legend> + + <label for="num:public" class="form-label">public?</label> + <input id="num:public" + name="public" + type="number" + min="0" max="2" value="2" + class="form-control" /> + + <label for="txt:inbredset-family" class="form-label">family</label> + <input id="txt:inbredset-family" + name="inbredset_family" + type="text" + placeholder="I am not sure what this is about." + class="form-control" /> + + <label for="txtarea:" class="form-label">Description</label> + <textarea id="txtarea:description" + name="description" + rows="5" + placeholder="Enter a description of this grouping/population" + class="form-control"></textarea> + </div> + + <button type="submit" class="btn btn-primary">create grouping/population</button> +</form> + +{%endblock%} + + +{%block javascript%} +{%endblock%} diff --git a/uploader/templates/samples/select-species.html b/uploader/templates/samples/select-species.html new file mode 100644 index 0000000..edadc61 --- /dev/null +++ b/uploader/templates/samples/select-species.html @@ -0,0 +1,30 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Select Grouping/Population{%endblock%} + +{%block contents%} +<h2 class="heading">upload samples/cases</h2> + +<p>We need to know what species your data belongs to.</p> + +{{flash_all_messages()}} + +<form method="POST" action="{{url_for('samples.select_species')}}"> + <legend class="heading">upload samples</legend> + <div class="form-group"> + <label for="select_species02" class="form-label">Species</label> + <select id="select_species02" + name="species_id" + required="required" + class="form-control"> + <option value="">Select species</option> + {%for spec in species%} + <option value="{{spec.SpeciesId}}">{{spec.MenuName}}</option> + {%endfor%} + </select> + </div> + + <button type="submit" class="btn btn-primary">submit</button> +</form> +{%endblock%} diff --git a/uploader/templates/samples/upload-failure.html b/uploader/templates/samples/upload-failure.html new file mode 100644 index 0000000..09e2ecf --- /dev/null +++ b/uploader/templates/samples/upload-failure.html @@ -0,0 +1,27 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block title%}Samples Upload Failure{%endblock%} + +{%block contents%} +<h1 class="heading">{{job.job_name}}</h2> + +<p>There was a failure attempting to upload the samples.</p> + +<p>Here is some information to help with debugging the issue. Provide this + information to the developer/maintainer.</p> + +<h3>Debugging Information</h3> +<ul> + <li><strong>job id</strong>: {{job.job_id}}</li> + <li><strong>status</strong>: {{job.status}}</li> + <li><strong>job type</strong>: {{job["job-type"]}}</li> +</ul> + +<h4>stdout</h4> +{{cli_output(job, "stdout")}} + +<h4>stderr</h4> +{{cli_output(job, "stderr")}} + +{%endblock%} diff --git a/uploader/templates/samples/upload-progress.html b/uploader/templates/samples/upload-progress.html new file mode 100644 index 0000000..7bb02be --- /dev/null +++ b/uploader/templates/samples/upload-progress.html @@ -0,0 +1,22 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block extrameta%} +<meta http-equiv="refresh" content="5"> +{%endblock%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">{{job.job_name}}</h2> + +<p> +<strong>status</strong>: +<span>{{job["status"]}} ({{job.get("message", "-")}})</span><br /> +</p> + +<p>saving to database...</p> + +{{cli_output(job, "stdout")}} + +{%endblock%} diff --git a/uploader/templates/samples/upload-samples.html b/uploader/templates/samples/upload-samples.html new file mode 100644 index 0000000..e62de57 --- /dev/null +++ b/uploader/templates/samples/upload-samples.html @@ -0,0 +1,139 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Upload Samples{%endblock%} + +{%block css%}{%endblock%} + +{%block contents%} +<h1 class="heading">upload samples</h1> + +{{flash_messages("alert-success")}} + +<p>You can now upload a character-separated value (CSV) file that contains + details about your samples. The CSV file should have the following fields: + <dl> + <dt>Name</dt> + <dd>The primary name for the sample</dd> + + <dt>Name2</dt> + <dd>A secondary name for the sample. This can simply be the same as + <strong>Name</strong> above. This field <strong>MUST</strong> contain a + value.</dd> + + <dt>Symbol</dt> + <dd>A symbol for the sample. Can be an empty field.</dd> + + <dt>Alias</dt> + <dd>An alias for the sample. Can be an empty field.</dd> + </dl> +</p> + +<form id="form-samples" + method="POST" + action="{{url_for('samples.upload_samples', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + enctype="multipart/form-data"> + <legend class="heading">upload samples</legend> + + <div class="form-group"> + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <label class="form-label">species:</label> + <span class="form-text">{{species.SpeciesName}} [{{species.MenuName}}]</span> + </div> + + <div class="form-group"> + <input type="hidden" name="inbredset_id" value="{{population.InbredSetId}}" /> + <label class="form-label">grouping/population:</label> + <span class="form-text">{{population.Name}} [{{population.FullName}}]</span> + </div> + + <div class="form-group"> + <label for="file-samples" class="form-label">select file</label> + <input type="file" name="samples_file" id="file:samples" + accept="text/csv, text/tab-separated-values" + class="form-control" /> + </div> + + <div class="form-group"> + <label for="select:separator" class="form-label">field separator</label> + <select id="select:separator" + name="separator" + required="required" + class="form-control"> + <option value="">Select separator for your file: (default is comma)</option> + <option value="	">TAB</option> + <option value=" ">Space</option> + <option value=",">Comma</option> + <option value=";">Semicolon</option> + <option value="other">Other</option> + </select> + <input id="txt:separator" + type="text" + name="other_separator" + class="form-control" /> + <small class="form-text text-muted"> + If you select '<strong>Other</strong>' for the field separator value, + enter the character that separates the fields in your CSV file in the form + field below. + </small> + </div> + + <div class="form-group form-check"> + <input id="chk:heading" + type="checkbox" + name="first_line_heading" + class="form-check-input" /> + <label for="chk:heading" class="form-check-label"> + first line is a heading?</label> + <small class="form-text text-muted"> + Select this if the first line in your file contains headings for the + columns. + </small> + </div> + + <div class="form-group"> + <label for="txt:delimiter" class="form-label">field delimiter</label> + <input id="txt:delimiter" + type="text" + name="field_delimiter" + maxlength="1" + class="form-control" /> + <small class="form-text text-muted"> + If there is a character delimiting the string texts within particular + fields in your CSV, provide the character here. This can be left blank if + no such delimiters exist in your file. + </small> + </div> + + <button type="submit" + class="btn btn-primary">upload samples file</button> +</form> + +<table id="tbl:samples-preview" class="table"> + <caption class="heading">preview content</caption> + + <thead> + <tr> + <th>Name</th> + <th>Name2</th> + <th>Symbol</th> + <th>Alias</th> + </tr> + </thead> + + <tbody> + <tr id="default-row"> + <td colspan="4"> + Please make some selections to preview the data.</td> + </tr> + </tbody> +</table> + +{%endblock%} + + +{%block javascript%} +<script src="/static/js/upload_samples.js" type="text/javascript"></script> +{%endblock%} diff --git a/uploader/templates/samples/upload-success.html b/uploader/templates/samples/upload-success.html new file mode 100644 index 0000000..cb745c3 --- /dev/null +++ b/uploader/templates/samples/upload-success.html @@ -0,0 +1,18 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +<h1 class="heading">{{job.job_name}}</h2> + +<p> +<strong>status</strong>: +<span>{{job["status"]}} ({{job.get("message", "-")}})</span><br /> +</p> + +<p>Successfully uploaded the samples.</p> + +{{cli_output(job, "stdout")}} + +{%endblock%} diff --git a/uploader/templates/select_dataset.html b/uploader/templates/select_dataset.html new file mode 100644 index 0000000..2f07de8 --- /dev/null +++ b/uploader/templates/select_dataset.html @@ -0,0 +1,161 @@ +{%extends "base.html"%} +{%from "dbupdate_hidden_fields.html" import hidden_fields%} + +{%block title%}Select Dataset{%endblock%} + +{%block css%} +<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> +{%endblock%} + +{%block contents%} +<h2 class="heading">{{filename}}: select dataset</h2> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.final_confirmation')}}" + id="select-dataset-form" class="two-col-sep-col1"> + <legend class="heading">choose existing dataset</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, totallines=totallines)}} + + <div class="form-group"> + <label for="datasetid" class="form-label">dataset:</label> + <select id="datasetid" name="datasetid" class="form-control" + {%if datasets | length == 0:%} + disabled="disabled" + {%endif%}> + {%for dataset in datasets:%} + <option value="{{dataset['Id']}}"> + [{{dataset["Name"]}}] - {{dataset["FullName"]}} + </option> + {%endfor%} + </select> + </div> + + <button type="submit" class="btn btn-primary" + {%if datasets | length == 0:%} + disabled="disabled" + {%endif%} />update database</button> +</form> +</div> + +<div class="row"> + <p class="two-col-sep-separator">OR</p> +</div> + +<div class="row"> + <form method="POST" id="create-dataset-form" + action="{{url_for('dbinsert.create_dataset')}}" + class="two-col-sep-col2"> + <legend class="heading">create new dataset</legend> + {{hidden_fields( + filename, filetype, species=species, genechipid=genechipid, + studyid=studyid, totallines=totallines)}} + + {%with messages = get_flashed_messages(with_categories=true)%} + {%if messages:%} + <ul> + {%for category, message in messages:%} + <li class="{{category}}">{{message}}</li> + {%endfor%} + </ul> + {%endif%} + {%endwith%} + + <div class="form-group"> + <label for="avgid" class="form-label">average:</label> + <select id="avgid" name="avgid" required="required" class="form-control"> + <option value="">Select averaging method</option> + {%for method in avgmethods:%} + <option value="{{method['AvgMethodId']}}" + {%if avgid is defined and method['AvgMethodId'] | int == avgid | int%} + selected="selected" + {%endif%}> + {{method["Name"]}} + </option> + {%endfor%} + </select> + </div> + + <div class="form-group"> + <label for="datasetname" class="form-label">name:</label> + <input id="datasetname" name="datasetname" type="text" + class="form-control" + {%if datasetname is defined %} + value="{{datasetname}}" + {%endif%} /> + </div> + + <div class="form-group"> + <label for="datasetname2" class="form-label">name 2:</label> + <input id="datasetname2" name="datasetname2" type="text" + required="required" class="form-control" + {%if datasetname2 is defined %} + value="{{datasetname2}}" + {%endif%} /> + </div> + + <div class="form-group"> + <label for="datasetfullname" class="form-label">full name:</label> + <input id="datasetfullname" name="datasetfullname" type="text" + required="required" class="form-control" + {%if datasetfullname is defined %} + value="{{datasetfullname}}" + {%endif%} /> + </div> + + <div class="form-group"> + <label for="datasetshortname" class="form-label">short name:</label> + <input id="datasetshortname" name="datasetshortname" type="text" + required="required" class="form-control" + {%if datasetshortname is defined %} + value="{{datasetshortname}}" + {%endif%} /> + </div> + + <div class="form-group"> + <label for="datasetpublic" class="form-label">public:</label> + <input id="datasetpublic" name="datasetpublic" type="number" + required="required" min="0" max="2" + {%if datasetpublic is defined %} + value="{{datasetpublic | int}}" + {%else%} + value="0" + {%endif%} + class="form-control" /> + </div> + + <div class="form-group"> + <label for="datasetconfidentiality">confidentiality:</label> + <input id="datasetconfidentiality" name="datasetconfidentiality" + type="number" required="required" min="0" max="2" + {%if datasetconfidentiality is defined %} + value="{{datasetconfidentiality | int}}" + {%else%} + value="0" + {%endif%} + class="form-control" /> + </div> + + <div class="form-group"> + <label for="datasetdatascale" class="form-label">data scale:</label> + <select id="datasetdatascale" name="datasetdatascale" class="form-control"> + <option value="">None</option> + {%for dscale in datascales:%} + <option value="{{dscale}}" + {%if datasetdatascale is defined and dscale == datasetdatascale%} + selected="selected" + {%elif dscale == "log2":%} + selected="selected" + {%endif%}> + {{dscale}} + </option> + {%endfor%} + </select> + </div> + + <button type="submit" class="btn btn-primary">create new dataset</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/select_platform.html b/uploader/templates/select_platform.html new file mode 100644 index 0000000..d9bc68f --- /dev/null +++ b/uploader/templates/select_platform.html @@ -0,0 +1,82 @@ +{%extends "base.html"%} + +{%block title%}Select Dataset{%endblock%} + +{%block contents%} +<h2 class="heading">{{filename}}: select platform</h2> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.select_study')}}" + id="select-platform-form" data-genechips="{{genechips_data}}"> + <input type="hidden" name="filename" value="{{filename}}" /> + <input type="hidden" name="filetype" value="{{filetype}}" /> + <input type="hidden" name="totallines" value="{{totallines}}" /> + + <div class="form-group"> + <label for="species" class="form-label">species</label> + <select id="species" name="species" class="form-control"> + {%for row in species:%} + <option value="{{row['SpeciesId']}}" + {%if row["Name"] == default_species:%} + selected="selected" + {%endif%}> + {{row["MenuName"]}} + </option> + {%endfor%} + </select> + </div> + + <table id="genechips-table" class="table"> + <caption>select platform</caption> + <thead> + <tr> + <th>Select</th> + <th>GeneChip ID</th> + <th>GeneChip Name</th> + </tr> + </thead> + + <tbody> + {%for chip in genechips:%} + <tr> + <td> + <input type="radio" name="genechipid" value="{{chip['GeneChipId']}}" + required="required" /> + </td> + <td>{{chip["GeneChipId"]}}</td> + <td>{{chip["GeneChipName"]}}</td> + </tr> + {%else%} + <tr> + <td colspan="5">No chips found for selected species</td> + </tr> + {%endfor%} + </tbody> + </table> + + <button type="submit" class="btn btn-primary">submit platform</button> + </form> +</div> +{%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/utils.js"></script> +<script type="text/javascript" src="/static/js/select_platform.js"></script> +<script type="text/javascript"> + document.getElementById( + "species").addEventListener("change", update_genechips); + document.getElementById( + "genechips-table").getElementsByTagName( + "tbody")[0].addEventListener( + "click", + function(event) { + if(event.target.tagName.toLowerCase() == "td") { + return select_row_radio(event.target.parentElement); + } + if(event.target.tagName.toLowerCase() == "td") { + return select_row_radio(event.target); + } + return false; + }); +</script> +{%endblock%} diff --git a/uploader/templates/select_species.html b/uploader/templates/select_species.html new file mode 100644 index 0000000..3b1a8a9 --- /dev/null +++ b/uploader/templates/select_species.html @@ -0,0 +1,92 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} +{%from "upload_progress_indicator.html" import upload_progress_indicator%} + +{%block title%}expression data: select species{%endblock%} + +{%block contents%} +{{upload_progress_indicator()}} + +<h2 class="heading">expression data: select species</h2> + +<div class="row"> + <form action="{{url_for('entry.upload_file')}}" + method="POST" + enctype="multipart/form-data" + id="frm-upload-expression-data"> + <legend class="heading">upload expression data</legend> + {{flash_messages("error-expr-data")}} + + <div class="form-group"> + <label for="select_species01" class="form-label">Species</label> + <select id="select_species01" + name="speciesid" + required="required" + class="form-control"> + <option value="">Select species</option> + {%for aspecies in species%} + <option value="{{aspecies.SpeciesId}}">{{aspecies.MenuName}}</option> + {%endfor%} + </select> + </div> + + <div class="form-group"> + <legend class="heading">file type</legend> + + <div class="form-check"> + <input type="radio" name="filetype" value="average" id="filetype_average" + required="required" class="form-check-input" /> + <label for="filetype_average" class="form-check-label">average</label> + </div> + + <div class="form-check"> + <input type="radio" name="filetype" value="standard-error" + id="filetype_standard_error" required="required" + class="form-check-input" /> + <label for="filetype_standard_error" class="form-check-label"> + standard error + </label> + </div> + </div> + + <div class="form-group"> + <span id="no-file-error" class="alert-danger" style="display: none;"> + No file selected + </span> + <label for="file_upload" class="form-label">select file</label> + <input type="file" name="qc_text_file" id="file_upload" + accept="text/plain, text/tab-separated-values, application/zip" + class="form-control"/> + </div> + + <button type="submit" + class="btn btn-primary" + data-toggle="modal" + data-target="#upload-progress-indicator">upload file</button> + </form> +</div> +{%endblock%} + + +{%block javascript%} +<script type="text/javascript" src="static/js/upload_progress.js"></script> +<script type="text/javascript"> + function setup_formdata(form) { + var formdata = new FormData(); + formdata.append( + "speciesid", + form.querySelector("#select_species01").value) + formdata.append( + "qc_text_file", + form.querySelector("input[type='file']").files[0]); + formdata.append( + "filetype", + selected_filetype( + Array.from(form.querySelectorAll("input[type='radio']")))); + return formdata; + } + + setup_upload_handlers( + "frm-upload-expression-data", make_data_uploader(setup_formdata)); +</script> +{%endblock%} diff --git a/uploader/templates/select_study.html b/uploader/templates/select_study.html new file mode 100644 index 0000000..648ad4c --- /dev/null +++ b/uploader/templates/select_study.html @@ -0,0 +1,108 @@ +{%extends "base.html"%} +{%from "dbupdate_hidden_fields.html" import hidden_fields%} + +{%block title%}Select Dataset{%endblock%} + +{%block css%} +<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> +{%endblock%} + +{%block contents%} +<h2 class="heading">{{filename}}: select study</h2> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.select_dataset')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col1"> + <legend class="heading">Select from existing study</legend> + {{hidden_fields(filename, filetype, species=species, genechipid=genechipid, + totallines=totallines)}} + + <div class="form-group"> + <label class="form-label" for="study">study:</label> + <select id="study" name="studyid" class="form-control"> + {%for study in studies:%} + <option value="{{study['Id']}}">{{study["Name"]}}</option> + {%endfor%} + </select> + </div> + + <button type="submit" + class="btn btn-primary" + {%if studies | length == 0:%} + disabled="disabled" + {%endif%} />submit selected study</button> +</form> +</div> + +<div class="row"> + <p class="two-col-sep-separator">OR</p> +</div> + +<div class="row"> + <form method="POST" action="{{url_for('dbinsert.create_study')}}" + id="select-platform-form" data-genechips="{{genechips_data}}" + class="two-col-sep-col2"> + {%with messages = get_flashed_messages(with_categories=true)%} + {%if messages:%} + <ul> + {%for category, message in messages:%} + <li class="{{category}}">{{message}}</li> + {%endfor%} + </ul> + {%endif%} + {%endwith%} + <legend class="heading">Create new study</legend> + {{hidden_fields(filename, filetype, species=species, genechipid=genechipid, + totallines=totallines)}} + + <div class="form-group"> + <label class="form-label" for="studyname">name:</label> + <input type="text" id="studyname" name="studyname" class="form-control" + required="required" + {%if studyname:%} + value="{{studyname}}" + {%endif%} /> + </div> + + <div class="form-group"> + <label class="form-label" for="group">group:</label> + <select id="group" name="inbredsetid" class="form-control" + required="required"> + <option value="">Select group</option> + {%for family in groups:%} + <optgroup label="{{family}}"> + {%for group in groups[family]:%} + <option value="{{group['InbredSetId']}}" + {%if group["InbredSetId"] == selected_group:%} + selected="selected" + {%endif%}> + {{group["FullName"]}} + </option> + {%endfor%} + </optgroup> + {%endfor%} + </select> + </div> + + <div class="form-group"> + <label class="form-label" for="tissue">tissue:</label> + <select id="tissue" name="tissueid" class="form-control" + required="required"> + <option value="">Select type</option> + {%for tissue in tissues:%} + <option value="{{tissue['TissueId']}}" + {%if tissue["TissueId"] == selected_tissue:%} + selected="selected" + {%endif%}> + {{tissue["Name"]}} + </option> + {%endfor%} + </select> + </div> + + <button type="submit" class="btn btn-primary">create study</button> + </form> +</div> + +{%endblock%} diff --git a/uploader/templates/stdout_output.html b/uploader/templates/stdout_output.html new file mode 100644 index 0000000..85345a9 --- /dev/null +++ b/uploader/templates/stdout_output.html @@ -0,0 +1,8 @@ +{%macro stdout_output(job)%} + +<h4>STDOUT Output</h4> +<div class="cli-output"> + <pre>{{job.get("stdout", "")}}</pre> +</div> + +{%endmacro%} diff --git a/uploader/templates/unhandled_exception.html b/uploader/templates/unhandled_exception.html new file mode 100644 index 0000000..6e6a051 --- /dev/null +++ b/uploader/templates/unhandled_exception.html @@ -0,0 +1,21 @@ +{%extends "base.html"%} + +{%block title%}System Error{%endblock%} + +{%block css%} +<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> +{%endblock%} + +{%block contents%} +<p> + An error has occured, and your request has been aborted. Please notify the + administrator to try and get this sorted. +</p> +<p> + Provide the following information to help the administrator figure out and fix + the issue:<br /> + <hr /><br /> + {{trace}} + <hr /><br /> +</p> +{%endblock%} diff --git a/uploader/templates/upload_progress_indicator.html b/uploader/templates/upload_progress_indicator.html new file mode 100644 index 0000000..e274e83 --- /dev/null +++ b/uploader/templates/upload_progress_indicator.html @@ -0,0 +1,35 @@ +{%macro upload_progress_indicator()%} +<div id="upload-progress-indicator" class="modal fade" tabindex="-1" role="dialog"> + <div class="modal-dialog" role="document"> + <div class="modal-content"> + <div class="modal-header"> + <h3 class="modal-title">Uploading file</h3> + </div> + + <div class="modal-body"> + <form id="frm-cancel-upload" style="border-style: none;"> + <div class="form-group"> + <span id="progress-filename" class="form-text">No file selected!</span> + <progress id="progress-bar" value="0" max="100" class="form-control"> + 0</progress> + </div> + + <div class="form-group"> + <span class="form-text text-muted" id="progress-text"> + Uploading 0%</span> + <span class="form-text text-muted" id="progress-extra-text"> + Processing</span> + </div> + </form> + </div> + + <div class="modal-footer"> + <button id="btn-cancel-upload" + type="button" + class="btn btn-danger" + data-dismiss="modal">Cancel</button> + </div> + </div> + </div> +</div> +{%endmacro%} diff --git a/uploader/templates/worker_failure.html b/uploader/templates/worker_failure.html new file mode 100644 index 0000000..b65b140 --- /dev/null +++ b/uploader/templates/worker_failure.html @@ -0,0 +1,24 @@ +{%extends "base.html"%} + +{%block title%}Worker Failure{%endblock%} + +{%block contents%} +<h1 class="heading">Worker Failure</h1> + +<p> + There was a critical failure launching the job to parse your file. + This is our fault and (probably) has nothing to do with the file you uploaded. +</p> + +<p> + Please notify the developers of this issue when you encounter it, + providing the link to this page, or the information below. +</p> + +<h4>Debugging Information</h4> + +<ul> + <li><strong>job id</strong>: {{job_id}}</li> +</ul> + +{%endblock%} diff --git a/uploader/upload/__init__.py b/uploader/upload/__init__.py new file mode 100644 index 0000000..5f120d4 --- /dev/null +++ b/uploader/upload/__init__.py @@ -0,0 +1,7 @@ +"""Package handling upload of files.""" +from flask import Blueprint + +from .rqtl2 import rqtl2 + +upload = Blueprint("upload", __name__) +upload.register_blueprint(rqtl2, url_prefix="/rqtl2") diff --git a/uploader/upload/rqtl2.py b/uploader/upload/rqtl2.py new file mode 100644 index 0000000..6aed1f7 --- /dev/null +++ b/uploader/upload/rqtl2.py @@ -0,0 +1,1157 @@ +"""Module to handle uploading of R/qtl2 bundles."""#pylint: disable=[too-many-lines] +import sys +import json +import traceback +from pathlib import Path +from datetime import date +from uuid import UUID, uuid4 +from functools import partial +from zipfile import ZipFile, is_zipfile +from typing import Union, Callable, Optional + +import MySQLdb as mdb +from redis import Redis +from MySQLdb.cursors import DictCursor +from werkzeug.utils import secure_filename +from flask import ( + flash, + escape, + request, + jsonify, + url_for, + redirect, + Response, + Blueprint, + render_template, + current_app as app) + +from r_qtl import r_qtl2 + +from uploader import jobs +from uploader.files import save_file, fullpath +from uploader.dbinsert import species as all_species +from uploader.db_utils import with_db_connection, database_connection + +from uploader.db.platforms import platform_by_id, platforms_by_species +from uploader.db.averaging import averaging_methods, averaging_method_by_id +from uploader.db.tissues import all_tissues, tissue_by_id, create_new_tissue +from uploader.db import ( + species_by_id, + save_population, + populations_by_species, + population_by_species_and_id,) +from uploader.db.datasets import ( + geno_dataset_by_id, + geno_datasets_by_species_and_population, + + probeset_study_by_id, + probeset_create_study, + probeset_dataset_by_id, + probeset_create_dataset, + probeset_datasets_by_study, + probeset_studies_by_species_and_population) + +rqtl2 = Blueprint("rqtl2", __name__) + +@rqtl2.route("/", methods=["GET", "POST"]) +@rqtl2.route("/select-species", methods=["GET", "POST"]) +def select_species(): + """Select the species.""" + if request.method == "GET": + return render_template("rqtl2/index.html", species=with_db_connection(all_species)) + + species_id = request.form.get("species_id") + species = with_db_connection( + lambda conn: species_by_id(conn, species_id)) + if bool(species): + return redirect(url_for( + "upload.rqtl2.select_population", species_id=species_id)) + flash("Invalid species or no species selected!", "alert-error error-rqtl2") + return redirect(url_for("upload.rqtl2.select_species")) + + +@rqtl2.route("/upload/species/<int:species_id>/select-population", + methods=["GET", "POST"]) +def select_population(species_id: int): + """Select/Create the population to organise data under.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("Invalid species selected!", "alert-error error-rqtl2") + return redirect(url_for("upload.rqtl2.select_species")) + + if request.method == "GET": + return render_template( + "rqtl2/select-population.html", + species=species, + populations=populations_by_species(conn, species_id)) + + population = population_by_species_and_id( + conn, species["SpeciesId"], request.form.get("inbredset_id")) + if not bool(population): + flash("Invalid Population!", "alert-error error-rqtl2") + return redirect( + url_for("upload.rqtl2.select_population", pgsrc="error"), + code=307) + + return redirect(url_for("upload.rqtl2.upload_rqtl2_bundle", + species_id=species["SpeciesId"], + population_id=population["InbredSetId"])) + + +@rqtl2.route("/upload/species/<int:species_id>/create-population", + methods=["POST"]) +def create_population(species_id: int): + """Create a new population for the given species.""" + population_page = redirect(url_for("upload.rqtl2.select_population", + species_id=species_id)) + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + population_name = request.form.get("inbredset_name", "").strip() + population_fullname = request.form.get("inbredset_fullname", "").strip() + if not bool(species): + flash("Invalid species!", "alert-error error-rqtl2") + return redirect(url_for("upload.rqtl2.select_species")) + if not bool(population_name): + flash("Invalid Population Name!", "alert-error error-rqtl2") + return population_page + if not bool(population_fullname): + flash("Invalid Population Full Name!", "alert-error error-rqtl2") + return population_page + new_population = save_population(conn, { + "SpeciesId": species["SpeciesId"], + "Name": population_name, + "InbredSetName": population_fullname, + "FullName": population_fullname, + "Family": request.form.get("inbredset_family") or None, + "Description": request.form.get("description") or None + }) + + flash("Population created successfully.", "alert-success") + return redirect( + url_for("upload.rqtl2.upload_rqtl2_bundle", + species_id=species_id, + population_id=new_population["population_id"], + pgsrc="create-population"), + code=307) + + +class __RequestError__(Exception): #pylint: disable=[invalid-name] + """Internal class to avoid pylint's `too-many-return-statements` error.""" + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle"), + methods=["GET", "POST"]) +def upload_rqtl2_bundle(species_id: int, population_id: int): + """Allow upload of R/qtl2 bundle.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + population = population_by_species_and_id( + conn, species["SpeciesId"], population_id) + if not bool(species): + flash("Invalid species!", "alert-error error-rqtl2") + return redirect(url_for("upload.rqtl2.select_species")) + if not bool(population): + flash("Invalid Population!", "alert-error error-rqtl2") + return redirect( + url_for("upload.rqtl2.select_population", pgsrc="error"), + code=307) + if request.method == "GET" or ( + request.method == "POST" + and bool(request.args.get("pgsrc"))): + return render_template("rqtl2/upload-rqtl2-bundle-step-01.html", + species=species, + population=population) + + try: + app.logger.debug("Files in the form: %s", request.files) + the_file = save_file(request.files["rqtl2_bundle_file"], + Path(app.config["UPLOAD_FOLDER"])) + except AssertionError: + app.logger.debug(traceback.format_exc()) + flash("Please provide a valid R/qtl2 zip bundle.", + "alert-error error-rqtl2") + return redirect(url_for("upload.rqtl2.upload_rqtl2_bundle", + species_id=species_id, + population_id=population_id)) + + if not is_zipfile(str(the_file)): + app.logger.debug("The file is not a zip file.") + raise __RequestError__("Invalid file! Expected a zip file.") + + jobid = trigger_rqtl2_bundle_qc( + species_id, + population_id, + the_file, + request.files["rqtl2_bundle_file"].filename)#type: ignore[arg-type] + return redirect(url_for( + "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid)) + + +def trigger_rqtl2_bundle_qc( + species_id: int, + population_id: int, + rqtl2bundle: Path, + originalfilename: str +) -> UUID: + """Trigger QC on the R/qtl2 bundle.""" + redisuri = app.config["REDIS_URL"] + with Redis.from_url(redisuri, decode_responses=True) as rconn: + jobid = uuid4() + redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"] + jobs.launch_job( + jobs.initialise_job( + rconn, + jobs.jobsnamespace(), + str(jobid), + [sys.executable, "-m", "scripts.qc_on_rqtl2_bundle", + app.config["SQL_URI"], app.config["REDIS_URL"], + jobs.jobsnamespace(), str(jobid), str(species_id), + str(population_id), "--redisexpiry", + str(redis_ttl_seconds)], + "rqtl2-bundle-qc-job", + redis_ttl_seconds, + {"job-metadata": json.dumps({ + "speciesid": species_id, + "populationid": population_id, + "rqtl2-bundle-file": str(rqtl2bundle.absolute()), + "original-filename": originalfilename})}), + redisuri, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + return jobid + + +def chunk_name(uploadfilename: str, chunkno: int) -> str: + """Generate chunk name from original filename and chunk number""" + if uploadfilename == "": + raise ValueError("Name cannot be empty!") + if chunkno < 1: + raise ValueError("Chunk number must be greater than zero") + return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}" + + +def chunks_directory(uniqueidentifier: str) -> Path: + """Compute the directory where chunks are temporarily stored.""" + if uniqueidentifier == "": + raise ValueError("Unique identifier cannot be empty!") + return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}") + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle-chunked"), + methods=["GET"]) +def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"] + species_id: int, + population_id: int +): + """ + Extension to the `upload_rqtl2_bundle` endpoint above that provides a way + for testing whether all the chunks have been uploaded and to assist with + resuming a failed upload. + """ + fileid = request.args.get("resumableIdentifier", type=str) or "" + filename = request.args.get("resumableFilename", type=str) or "" + chunk = request.args.get("resumableChunkNumber", type=int) or 0 + if not(fileid or filename or chunk): + return jsonify({ + "message": "At least one required query parameter is missing.", + "error": "BadRequest", + "statuscode": 400 + }), 400 + + if Path(chunks_directory(fileid), + chunk_name(filename, chunk)).exists(): + return "OK" + + return jsonify({ + "message": f"Chunk {chunk} was not found.", + "error": "NotFound", + "statuscode": 404 + }), 404 + + +def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path: + """Merge the chunks into a single file.""" + with open(targetfile, "ab") as _target: + for chunkfile in chunkpaths: + with open(chunkfile, "rb") as _chunkdata: + _target.write(_chunkdata.read()) + + chunkfile.unlink() + return targetfile + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle-chunked"), + methods=["POST"]) +def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int): + """ + Extension to the `upload_rqtl2_bundle` endpoint above that allows large + files to be uploaded in chunks. + + This should hopefully speed up uploads, and if done right, even enable + resumable uploads + """ + _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0 + _chunk = request.form.get("resumableChunkNumber", default=1, type=int) + _uploadfilename = request.form.get( + "resumableFilename", default="", type=str) or "" + _fileid = request.form.get( + "resumableIdentifier", default="", type=str) or "" + _targetfile = Path(app.config["UPLOAD_FOLDER"], _fileid) + + if _targetfile.exists(): + return jsonify({ + "message": ( + "A file with a similar unique identifier has previously been " + "uploaded and possibly is/has being/been processed."), + "error": "BadRequest", + "statuscode": 400 + }), 400 + + try: + # save chunk data + chunks_directory(_fileid).mkdir(exist_ok=True, parents=True) + request.files["file"].save(Path(chunks_directory(_fileid), + chunk_name(_uploadfilename, _chunk))) + + # Check whether upload is complete + chunkpaths = tuple( + Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk)) + for _achunk in range(1, _totalchunks+1)) + if all(_file.exists() for _file in chunkpaths): + # merge_files and clean up chunks + __merge_chunks__(_targetfile, chunkpaths) + chunks_directory(_fileid).rmdir() + jobid = trigger_rqtl2_bundle_qc( + species_id, population_id, _targetfile, _uploadfilename) + return url_for( + "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid) + except Exception as exc:# pylint: disable=[broad-except] + msg = "Error processing uploaded file chunks." + app.logger.error(msg, exc_info=True, stack_info=True) + return jsonify({ + "message": msg, + "error": type(exc).__name__, + "error-description": " ".join(str(arg) for arg in exc.args), + "error-trace": traceback.format_exception(exc) + }), 500 + + return "OK" + + +@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>", + methods=["GET", "POST"]) +def rqtl2_bundle_qc_status(jobid: UUID): + """Check the status of the QC jobs.""" + with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, + database_connection(app.config["SQL_URI"]) as dbconn): + try: + thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid) + messagelistname = thejob.get("log-messagelist") + logmessages = (rconn.lrange(messagelistname, 0, -1) + if bool(messagelistname) else []) + jobstatus = thejob["status"] + if jobstatus == "error": + return render_template("rqtl2/rqtl2-qc-job-error.html", + job=thejob, + errorsgeneric=json.loads( + thejob.get("errors-generic", "[]")), + errorsgeno=json.loads( + thejob.get("errors-geno", "[]")), + errorspheno=json.loads( + thejob.get("errors-pheno", "[]")), + errorsphenose=json.loads( + thejob.get("errors-phenose", "[]")), + errorsphenocovar=json.loads( + thejob.get("errors-phenocovar", "[]")), + messages=logmessages) + if jobstatus == "success": + jobmeta = json.loads(thejob["job-metadata"]) + species = species_by_id(dbconn, jobmeta["speciesid"]) + return render_template( + "rqtl2/rqtl2-qc-job-results.html", + species=species, + population=population_by_species_and_id( + dbconn, species["SpeciesId"], jobmeta["populationid"]), + rqtl2bundle=Path(jobmeta["rqtl2-bundle-file"]).name, + rqtl2bundleorig=jobmeta["original-filename"]) + + def compute_percentage(thejob, filetype) -> Union[str, None]: + if f"{filetype}-linecount" in thejob: + return "100" + if f"{filetype}-filesize" in thejob: + percent = ((int(thejob.get(f"{filetype}-checked", 0)) + / + int(thejob.get(f"{filetype}-filesize", 1))) + * 100) + return f"{percent:.2f}" + return None + + return render_template( + "rqtl2/rqtl2-qc-job-status.html", + job=thejob, + geno_percent=compute_percentage(thejob, "geno"), + pheno_percent=compute_percentage(thejob, "pheno"), + phenose_percent=compute_percentage(thejob, "phenose"), + messages=logmessages) + except jobs.JobNotFound: + return render_template("rqtl2/no-such-job.html", jobid=jobid) + + +def redirect_on_error(flaskroute, **kwargs): + """Utility to redirect on error""" + return redirect(url_for(flaskroute, **kwargs, pgsrc="error"), + code=(307 if request.method == "POST" else 302)) + + +def check_species(conn: mdb.Connection, formargs: dict) -> Optional[ + tuple[str, Response]]: + """ + Check whether the 'species_id' value is provided, and whether a + corresponding species exists in the database. + + Maybe give the function a better name...""" + speciespage = redirect_on_error("upload.rqtl2.select_species") + if "species_id" not in formargs: + return "You MUST provide the Species identifier.", speciespage + + if not bool(species_by_id(conn, formargs["species_id"])): + return "No species with the provided identifier exists.", speciespage + + return None + + +def check_population(conn: mdb.Connection, + formargs: dict, + species_id) -> Optional[tuple[str, Response]]: + """ + Check whether the 'population_id' value is provided, and whether a + corresponding population exists in the database. + + Maybe give the function a better name...""" + poppage = redirect_on_error( + "upload.rqtl2.select_species", species_id=species_id) + if "population_id" not in formargs: + return "You MUST provide the Population identifier.", poppage + + if not bool(population_by_species_and_id( + conn, species_id, formargs["population_id"])): + return "No population with the provided identifier exists.", poppage + + return None + + +def check_r_qtl2_bundle(formargs: dict, + species_id, + population_id) -> Optional[tuple[str, Response]]: + """Check for the existence of the R/qtl2 bundle.""" + fileuploadpage = redirect_on_error("upload.rqtl2.upload_rqtl2_bundle", + species_id=species_id, + population_id=population_id) + if not "rqtl2_bundle_file" in formargs: + return ( + "You MUST provide a R/qtl2 zip bundle for upload.", fileuploadpage) + + if not Path(fullpath(formargs["rqtl2_bundle_file"])).exists(): + return "No R/qtl2 bundle with the given name exists.", fileuploadpage + + return None + + +def check_geno_dataset(conn: mdb.Connection, + formargs: dict, + species_id, + population_id) -> Optional[tuple[str, Response]]: + """Check for the Genotype dataset.""" + genodsetpg = redirect_on_error("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id) + if not bool(formargs.get("geno-dataset-id")): + return ( + "You MUST provide a valid Genotype dataset identifier", genodsetpg) + + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s", + (formargs["geno-dataset-id"],)) + results = cursor.fetchall() + if not bool(results): + return ("No genotype dataset with the provided identifier exists.", + genodsetpg) + if len(results) > 1: + return ( + "Data corruption: More than one genotype dataset with the same " + "identifier.", + genodsetpg) + + return None + +def check_tissue( + conn: mdb.Connection,formargs: dict) -> Optional[tuple[str, Response]]: + """Check for tissue/organ/biological material.""" + selectdsetpg = redirect_on_error("upload.rqtl2.select_dataset_info", + species_id=formargs["species_id"], + population_id=formargs["population_id"]) + if not bool(formargs.get("tissueid", "").strip()): + return ("No tissue/organ/biological material provided.", selectdsetpg) + + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Tissue WHERE Id=%s", + (formargs["tissueid"],)) + results = cursor.fetchall() + if not bool(results): + return ("No tissue/organ with the provided identifier exists.", + selectdsetpg) + + if len(results) > 1: + return ( + "Data corruption: More than one tissue/organ with the same " + "identifier.", + selectdsetpg) + + return None + + +def check_probe_study(conn: mdb.Connection, + formargs: dict, + species_id, + population_id) -> Optional[tuple[str, Response]]: + """Check for the ProbeSet study.""" + dsetinfopg = redirect_on_error("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id) + if not bool(formargs.get("probe-study-id")): + return "No probeset study was selected!", dsetinfopg + + if not bool(probeset_study_by_id(conn, formargs["probe-study-id"])): + return ("No probeset study with the provided identifier exists", + dsetinfopg) + + return None + + +def check_probe_dataset(conn: mdb.Connection, + formargs: dict, + species_id, + population_id) -> Optional[tuple[str, Response]]: + """Check for the ProbeSet dataset.""" + dsetinfopg = redirect_on_error("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id) + if not bool(formargs.get("probe-dataset-id")): + return "No probeset dataset was selected!", dsetinfopg + + if not bool(probeset_dataset_by_id(conn, formargs["probe-dataset-id"])): + return ("No probeset dataset with the provided identifier exists", + dsetinfopg) + + return None + + +def with_errors(endpointthunk: Callable, *checkfns): + """Run 'endpointthunk' with error checking.""" + formargs = {**dict(request.args), **dict(request.form)} + errors = tuple(item for item in (_fn(formargs=formargs) for _fn in checkfns) + if item is not None) + if len(errors) > 0: + flash(errors[0][0], "alert-error error-rqtl2") + return errors[0][1] + + return endpointthunk() + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/select-geno-dataset"), + methods=["POST"]) +def select_geno_dataset(species_id: int, population_id: int): + """Select from existing geno datasets.""" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + geno_dset = geno_datasets_by_species_and_population( + conn, species_id, population_id) + if not bool(geno_dset): + flash("No genotype dataset was provided!", + "alert-error error-rqtl2") + return redirect(url_for("upload.rqtl2.select_geno_dataset", + species_id=species_id, + population_id=population_id, + pgsrc="error"), + code=307) + + flash("Genotype accepted", "alert-success error-rqtl2") + return redirect(url_for("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id, + pgsrc="upload.rqtl2.select_geno_dataset"), + code=307) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/create-geno-dataset"), + methods=["POST"]) +def create_geno_dataset(species_id: int, population_id: int): + """Create a new geno dataset.""" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + sgeno_page = redirect(url_for("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id, + pgsrc="error"), + code=307) + errorclasses = "alert-error error-rqtl2 error-rqtl2-create-geno-dataset" + if not bool(request.form.get("dataset-name")): + flash("You must provide the dataset name", errorclasses) + return sgeno_page + if not bool(request.form.get("dataset-fullname")): + flash("You must provide the dataset full name", errorclasses) + return sgeno_page + public = 2 if request.form.get("dataset-public") == "on" else 0 + + with conn.cursor(cursorclass=DictCursor) as cursor: + datasetname = request.form["dataset-name"] + new_dataset = { + "name": datasetname, + "fname": request.form.get("dataset-fullname"), + "sname": request.form.get("dataset-shortname") or datasetname, + "today": date.today().isoformat(), + "pub": public, + "isetid": population_id + } + cursor.execute("SELECT * FROM GenoFreeze WHERE Name=%s", + (datasetname,)) + results = cursor.fetchall() + if bool(results): + flash( + f"A genotype dataset with name '{escape(datasetname)}' " + "already exists.", + errorclasses) + return redirect(url_for("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id, + pgsrc="error"), + code=307) + cursor.execute( + "INSERT INTO GenoFreeze(" + "Name, FullName, ShortName, CreateTime, public, InbredSetId" + ") " + "VALUES(" + "%(name)s, %(fname)s, %(sname)s, %(today)s, %(pub)s, %(isetid)s" + ")", + new_dataset) + flash("Created dataset successfully.", "alert-success") + return render_template( + "rqtl2/create-geno-dataset-success.html", + species=species_by_id(conn, species_id), + population=population_by_species_and_id( + conn, species_id, population_id), + rqtl2_bundle_file=request.form["rqtl2_bundle_file"], + geno_dataset={**new_dataset, "id": cursor.lastrowid}) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, conn=conn, species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/select-tissue"), + methods=["POST"]) +def select_tissue(species_id: int, population_id: int): + """Select from existing tissues.""" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + if not bool(request.form.get("tissueid", "").strip()): + flash("Invalid tissue selection!", + "alert-error error-select-tissue error-rqtl2") + + return redirect(url_for("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id, + pgsrc="upload.rqtl2.select_geno_dataset"), + code=307) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id)) + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/create-tissue"), + methods=["POST"]) +def create_tissue(species_id: int, population_id: int): + """Add new tissue, organ or biological material to the system.""" + form = request.form + datasetinfopage = redirect( + url_for("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id, + pgsrc="upload.rqtl2.select_geno_dataset"), + code=307) + with database_connection(app.config["SQL_URI"]) as conn: + tissuename = form.get("tissuename", "").strip() + tissueshortname = form.get("tissueshortname", "").strip() + if not bool(tissuename): + flash("Organ/Tissue name MUST be provided.", + "alert-error error-create-tissue error-rqtl2") + return datasetinfopage + + if not bool(tissueshortname): + flash("Organ/Tissue short name MUST be provided.", + "alert-error error-create-tissue error-rqtl2") + return datasetinfopage + + try: + tissue = create_new_tissue(conn, tissuename, tissueshortname) + flash("Tissue created successfully!", "alert-success") + return render_template( + "rqtl2/create-tissue-success.html", + species=species_by_id(conn, species_id), + population=population_by_species_and_id( + conn, species_id, population_id), + rqtl2_bundle_file=request.form["rqtl2_bundle_file"], + geno_dataset=geno_dataset_by_id( + conn, + int(request.form["geno-dataset-id"])), + tissue=tissue) + except mdb.IntegrityError as _ierr: + flash("Tissue/Organ with that short name already exists!", + "alert-error error-create-tissue error-rqtl2") + return datasetinfopage + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/select-probeset-study"), + methods=["POST"]) +def select_probeset_study(species_id: int, population_id: int): + """Select or create a probeset study.""" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + summary_page = redirect(url_for("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id), + code=307) + if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))): + flash("Invalid study selected!", "alert-error error-rqtl2") + return summary_page + + return summary_page + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_tissue, conn=conn), + partial(check_probe_study, + conn=conn, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/select-probeset-dataset"), + methods=["POST"]) +def select_probeset_dataset(species_id: int, population_id: int): + """Select or create a probeset dataset.""" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + summary_page = redirect(url_for("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id), + code=307) + if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))): + flash("Invalid study selected!", "alert-error error-rqtl2") + return summary_page + + return summary_page + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_tissue, conn=conn), + partial(check_probe_study, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_probe_dataset, + conn=conn, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/create-probeset-study"), + methods=["POST"]) +def create_probeset_study(species_id: int, population_id: int): + """Create a new probeset study.""" + errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-study" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + form = request.form + dataset_info_page = redirect( + url_for("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id), + code=307) + + if not (bool(form.get("platformid")) and + bool(platform_by_id(conn, int(form["platformid"])))): + flash("Invalid platform selected.", errorclasses) + return dataset_info_page + + if not (bool(form.get("tissueid")) and + bool(tissue_by_id(conn, int(form["tissueid"])))): + flash("Invalid tissue selected.", errorclasses) + return dataset_info_page + + studyname = form["studyname"] + try: + study = probeset_create_study( + conn, population_id, int(form["platformid"]), int(form["tissueid"]), + studyname, form.get("studyfullname") or "", + form.get("studyshortname") or "") + except mdb.IntegrityError as _ierr: + flash(f"ProbeSet study with name '{escape(studyname)}' already " + "exists.", + errorclasses) + return dataset_info_page + return render_template( + "rqtl2/create-probe-study-success.html", + species=species_by_id(conn, species_id), + population=population_by_species_and_id( + conn, species_id, population_id), + rqtl2_bundle_file=request.form["rqtl2_bundle_file"], + geno_dataset=geno_dataset_by_id( + conn, + int(request.form["geno-dataset-id"])), + study=study) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_tissue, conn=conn)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/create-probeset-dataset"), + methods=["POST"]) +def create_probeset_dataset(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements] + """Create a new probeset dataset.""" + errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-dataset" + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__():#pylint: disable=[too-many-return-statements] + form = request.form + summary_page = redirect(url_for("upload.rqtl2.select_dataset_info", + species_id=species_id, + population_id=population_id), + code=307) + if not bool(form.get("averageid")): + flash("Averaging method not selected!", errorclasses) + return summary_page + if not bool(form.get("datasetname")): + flash("Dataset name not provided!", errorclasses) + return summary_page + if not bool(form.get("datasetfullname")): + flash("Dataset full name not provided!", errorclasses) + return summary_page + + tissue = tissue_by_id(conn, form.get("tissueid", "").strip()) + + study = probeset_study_by_id(conn, int(form["probe-study-id"])) + if not bool(study): + flash("Invalid ProbeSet study provided!", errorclasses) + return summary_page + + avgmethod = averaging_method_by_id(conn, int(form["averageid"])) + if not bool(avgmethod): + flash("Invalid averaging method provided!", errorclasses) + return summary_page + + try: + dset = probeset_create_dataset(conn, + int(form["probe-study-id"]), + int(form["averageid"]), + form["datasetname"], + form["datasetfullname"], + form["datasetshortname"], + form["datasetpublic"] == "on", + form.get( + "datasetdatascale", "log2")) + except mdb.IntegrityError as _ierr: + app.logger.debug("Possible integrity error: %s", traceback.format_exc()) + flash(("IntegrityError: The data you provided has some errors: " + f"{_ierr.args}"), + errorclasses) + return summary_page + except Exception as _exc:# pylint: disable=[broad-except] + app.logger.debug("Error creating ProbeSet dataset: %s", + traceback.format_exc()) + flash(("There was a problem creating your dataset. Please try " + "again."), + errorclasses) + return summary_page + return render_template( + "rqtl2/create-probe-dataset-success.html", + species=species_by_id(conn, species_id), + population=population_by_species_and_id( + conn, species_id, population_id), + rqtl2_bundle_file=request.form["rqtl2_bundle_file"], + geno_dataset=geno_dataset_by_id( + conn, + int(request.form["geno-dataset-id"])), + tissue=tissue, + study=study, + avgmethod=avgmethod, + dataset=dset) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_tissue, conn=conn), + partial(check_probe_study, + conn=conn, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/dataset-info"), + methods=["POST"]) +def select_dataset_info(species_id: int, population_id: int): + """ + If `geno` files exist in the R/qtl2 bundle, prompt user to provide the + dataset the genotypes belong to. + """ + form = request.form + with database_connection(app.config["SQL_URI"]) as conn: + def __thunk__(): + species = species_by_id(conn, species_id) + population = population_by_species_and_id( + conn, species_id, population_id) + thefile = fullpath(form["rqtl2_bundle_file"]) + with ZipFile(str(thefile), "r") as zfile: + cdata = r_qtl2.control_data(zfile) + + geno_dataset = geno_dataset_by_id( + conn,form.get("geno-dataset-id", "").strip()) + if "geno" in cdata and not bool(form.get("geno-dataset-id")): + return render_template( + "rqtl2/select-geno-dataset.html", + species=species, + population=population, + rqtl2_bundle_file=thefile.name, + datasets=geno_datasets_by_species_and_population( + conn, species_id, population_id)) + + tissue = tissue_by_id(conn, form.get("tissueid", "").strip()) + if "pheno" in cdata and not bool(tissue): + return render_template( + "rqtl2/select-tissue.html", + species=species, + population=population, + rqtl2_bundle_file=thefile.name, + geno_dataset=geno_dataset, + studies=probeset_studies_by_species_and_population( + conn, species_id, population_id), + platforms=platforms_by_species(conn, species_id), + tissues=all_tissues(conn)) + + probeset_study = probeset_study_by_id( + conn, form.get("probe-study-id", "").strip()) + if "pheno" in cdata and not bool(probeset_study): + return render_template( + "rqtl2/select-probeset-study-id.html", + species=species, + population=population, + rqtl2_bundle_file=thefile.name, + geno_dataset=geno_dataset, + studies=probeset_studies_by_species_and_population( + conn, species_id, population_id), + platforms=platforms_by_species(conn, species_id), + tissue=tissue) + probeset_study = probeset_study_by_id( + conn, int(form["probe-study-id"])) + + probeset_dataset = probeset_dataset_by_id( + conn, form.get("probe-dataset-id", "").strip()) + if "pheno" in cdata and not bool(probeset_dataset): + return render_template( + "rqtl2/select-probeset-dataset.html", + species=species, + population=population, + rqtl2_bundle_file=thefile.name, + geno_dataset=geno_dataset, + probe_study=probeset_study, + tissue=tissue, + datasets=probeset_datasets_by_study( + conn, int(form["probe-study-id"])), + avgmethods=averaging_methods(conn)) + + return render_template("rqtl2/summary-info.html", + species=species, + population=population, + rqtl2_bundle_file=thefile.name, + geno_dataset=geno_dataset, + tissue=tissue, + probe_study=probeset_study, + probe_dataset=probeset_dataset) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" + "/rqtl2-bundle/confirm-bundle-details"), + methods=["POST"]) +def confirm_bundle_details(species_id: int, population_id: int): + """Confirm the details and trigger R/qtl2 bundle processing...""" + redisuri = app.config["REDIS_URL"] + with (database_connection(app.config["SQL_URI"]) as conn, + Redis.from_url(redisuri, decode_responses=True) as rconn): + def __thunk__(): + redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"] + jobid = str(uuid4()) + _job = jobs.launch_job( + jobs.initialise_job( + rconn, + jobs.jobsnamespace(), + jobid, + [ + sys.executable, "-m", "scripts.process_rqtl2_bundle", + app.config["SQL_URI"], app.config["REDIS_URL"], + jobs.jobsnamespace(), jobid, "--redisexpiry", + str(redis_ttl_seconds)], + "R/qtl2 Bundle Upload", + redis_ttl_seconds, + { + "bundle-metadata": json.dumps({ + "speciesid": species_id, + "populationid": population_id, + "rqtl2-bundle-file": str(fullpath( + request.form["rqtl2_bundle_file"])), + "geno-dataset-id": request.form.get( + "geno-dataset-id", ""), + "probe-study-id": request.form.get( + "probe-study-id", ""), + "probe-dataset-id": request.form.get( + "probe-dataset-id", ""), + **({ + "platformid": probeset_study_by_id( + conn, + int(request.form["probe-study-id"]))["ChipId"] + } if bool(request.form.get("probe-study-id")) else {}) + }) + }), + redisuri, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + + return redirect(url_for("upload.rqtl2.rqtl2_processing_status", + jobid=jobid)) + + return with_errors(__thunk__, + partial(check_species, conn=conn), + partial(check_population, + conn=conn, + species_id=species_id), + partial(check_r_qtl2_bundle, + species_id=species_id, + population_id=population_id), + partial(check_geno_dataset, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_probe_study, + conn=conn, + species_id=species_id, + population_id=population_id), + partial(check_probe_dataset, + conn=conn, + species_id=species_id, + population_id=population_id)) + + +@rqtl2.route("/status/<uuid:jobid>") +def rqtl2_processing_status(jobid: UUID): + """Retrieve the status of the job processing the uploaded R/qtl2 bundle.""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid) + + messagelistname = thejob.get("log-messagelist") + logmessages = (rconn.lrange(messagelistname, 0, -1) + if bool(messagelistname) else []) + + if thejob["status"] == "error": + return render_template( + "rqtl2/rqtl2-job-error.html", job=thejob, messages=logmessages) + if thejob["status"] == "success": + return render_template("rqtl2/rqtl2-job-results.html", + job=thejob, + messages=logmessages) + + return render_template( + "rqtl2/rqtl2-job-status.html", job=thejob, messages=logmessages) + except jobs.JobNotFound as _exc: + return render_template("rqtl2/no-such-job.html", jobid=jobid) |
