diff options
Diffstat (limited to 'qc_app')
83 files changed, 0 insertions, 6590 deletions
diff --git a/qc_app/__init__.py b/qc_app/__init__.py deleted file mode 100644 index 9907695..0000000 --- a/qc_app/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -"""The Quality-Control Web Application entry point""" -import os -import logging -from pathlib import Path - -from flask import Flask - -from .entry import entrybp -from .upload import upload -from .parse import parsebp -from .samples import samples -from .base_routes import base -from .dbinsert import dbinsertbp -from .errors import register_error_handlers - -def override_settings_with_envvars( - app: Flask, ignore: tuple[str, ...]=tuple()) -> None: - """Override settings in `app` with those in ENVVARS""" - for setting in (key for key in app.config if key not in ignore): - app.config[setting] = os.environ.get(setting) or app.config[setting] - - -def create_app(): - """The application factory""" - app = Flask(__name__) - app.config.from_pyfile( - Path(__file__).parent.joinpath("default_settings.py")) - if "QCAPP_CONF" in os.environ: - app.config.from_envvar("QCAPP_CONF") # Override defaults with instance path - - override_settings_with_envvars(app, ignore=tuple()) - - if "QCAPP_SECRETS" in os.environ: - app.config.from_envvar("QCAPP_SECRETS") - - # setup blueprints - app.register_blueprint(base, url_prefix="/") - app.register_blueprint(entrybp, url_prefix="/") - app.register_blueprint(parsebp, url_prefix="/parse") - app.register_blueprint(upload, url_prefix="/upload") - app.register_blueprint(dbinsertbp, url_prefix="/dbinsert") - app.register_blueprint(samples, url_prefix="/samples") - - register_error_handlers(app) - return app diff --git a/qc_app/base_routes.py b/qc_app/base_routes.py deleted file mode 100644 index 9daf439..0000000 --- a/qc_app/base_routes.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Basic routes required for all pages""" -import os -from flask import Blueprint, send_from_directory - -base = Blueprint("base", __name__) - -def appenv(): - """Get app's guix environment path.""" - return os.environ.get("GN_UPLOADER_ENVIRONMENT") - -@base.route("/bootstrap/<path:filename>") -def bootstrap(filename): - """Fetch bootstrap files.""" - return send_from_directory( - appenv(), f"share/genenetwork2/javascript/bootstrap/{filename}") - - -@base.route("/jquery/<path:filename>") -def jquery(filename): - """Fetch jquery files.""" - return send_from_directory( - appenv(), f"share/genenetwork2/javascript/jquery/{filename}") - - -@base.route("/node-modules/<path:filename>") -def node_modules(filename): - """Fetch node-js modules.""" - return send_from_directory( - appenv(), f"lib/node_modules/{filename}") diff --git a/qc_app/check_connections.py b/qc_app/check_connections.py deleted file mode 100644 index ceccc32..0000000 --- a/qc_app/check_connections.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Check the various connection used in the application""" -import sys -import traceback - -import redis -import MySQLdb - -from qc_app.db_utils import database_connection - -def check_redis(uri: str): - "Check the redis connection" - try: - with redis.Redis.from_url(uri) as rconn: - rconn.ping() - except redis.exceptions.ConnectionError as conn_err: - print(conn_err, file=sys.stderr) - print(traceback.format_exc(), file=sys.stderr) - sys.exit(1) - -def check_db(uri: str): - "Check the mysql connection" - try: - with database_connection(uri) as dbconn: # pylint: disable=[unused-variable] - pass - except MySQLdb.OperationalError as op_err: - print(op_err, file=sys.stderr) - print(traceback.format_exc(), file=sys.stderr) - sys.exit(1) diff --git a/qc_app/db/__init__.py b/qc_app/db/__init__.py deleted file mode 100644 index 36e93e8..0000000 --- a/qc_app/db/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Database functions""" -from .species import species, species_by_id -from .populations import ( - save_population, - population_by_id, - populations_by_species, - population_by_species_and_id) -from .datasets import geno_datasets_by_species_and_population diff --git a/qc_app/db/averaging.py b/qc_app/db/averaging.py deleted file mode 100644 index 62bbe67..0000000 --- a/qc_app/db/averaging.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Functions for db interactions for averaging methods""" -from typing import Optional - -import MySQLdb as mdb -from MySQLdb.cursors import DictCursor - -def averaging_methods(conn: mdb.Connection) -> tuple[dict, ...]: - """Fetch all available averaging methods""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM AvgMethod") - return tuple(dict(row) for row in cursor.fetchall()) - -def averaging_method_by_id( - conn: mdb.Connection, averageid: int) -> Optional[dict]: - """Fetch the averaging method by its ID""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM AvgMethod WHERE Id=%s", - (averageid,)) - result = cursor.fetchone() - if bool(result): - return dict(result) - - return None diff --git a/qc_app/db/datasets.py b/qc_app/db/datasets.py deleted file mode 100644 index 767ec41..0000000 --- a/qc_app/db/datasets.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Functions for accessing the database relating to datasets.""" -from datetime import date -from typing import Optional - -import MySQLdb as mdb -from MySQLdb.cursors import DictCursor - -def geno_datasets_by_species_and_population( - conn: mdb.Connection, - speciesid: int, - populationid: int) -> tuple[dict, ...]: - """Retrieve all genotypes datasets by species and population""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - "SELECT gf.* FROM InbredSet AS iset INNER JOIN GenoFreeze AS gf " - "ON iset.InbredSetId=gf.InbredSetId " - "WHERE iset.SpeciesId=%(sid)s AND iset.InbredSetId=%(pid)s", - {"sid": speciesid, "pid": populationid}) - return tuple(dict(row) for row in cursor.fetchall()) - -def geno_dataset_by_id(conn: mdb.Connection, dataset_id) -> Optional[dict]: - """Retrieve genotype dataset by ID""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s", (dataset_id,)) - _dataset = cursor.fetchone() - return dict(_dataset) if bool(_dataset) else None - -def probeset_studies_by_species_and_population( - conn: mdb.Connection, - speciesid: int, - populationid: int) -> tuple[dict, ...]: - """Retrieve all probesets""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - "SELECT pf.* FROM InbredSet AS iset INNER JOIN ProbeFreeze AS pf " - "ON iset.InbredSetId=pf.InbredSetId " - "WHERE iset.SpeciesId=%(sid)s AND iset.InbredSetId=%(pid)s", - {"sid": speciesid, "pid": populationid}) - return tuple(dict(row) for row in cursor.fetchall()) - -def probeset_datasets_by_study(conn: mdb.Connection, - studyid: int) -> tuple[dict, ...]: - """Retrieve all probeset databases by study.""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s", - (studyid,)) - return tuple(dict(row) for row in cursor.fetchall()) - -def probeset_study_by_id(conn: mdb.Connection, studyid) -> Optional[dict]: - """Retrieve ProbeSet study by ID""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM ProbeFreeze WHERE Id=%s", (studyid,)) - _study = cursor.fetchone() - return dict(_study) if bool(_study) else None - -def probeset_create_study(conn: mdb.Connection,#pylint: disable=[too-many-arguments] - populationid: int, - platformid: int, - tissueid: int, - studyname: str, - studyfullname: str = "", - studyshortname: str = ""): - """Create a new ProbeSet study.""" - with conn.cursor(cursorclass=DictCursor) as cursor: - studydata = { - "platid": platformid, - "tissueid": tissueid, - "name": studyname, - "fname": studyfullname or studyname, - "sname": studyshortname, - "today": date.today().isoformat(), - "popid": populationid - } - cursor.execute( - """ - INSERT INTO ProbeFreeze( - ChipId, TissueId, Name, FullName, ShortName, CreateTime, - InbredSetId - ) VALUES ( - %(platid)s, %(tissueid)s, %(name)s, %(fname)s, %(sname)s, - %(today)s, %(popid)s - ) - """, - studydata) - studyid = cursor.lastrowid - cursor.execute("UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s", - (studyid, studyid)) - return {**studydata, "studyid": studyid} - -def probeset_create_dataset(conn: mdb.Connection,#pylint: disable=[too-many-arguments] - studyid: int, - averageid: int, - datasetname: str, - datasetfullname: str, - datasetshortname: str="", - public: bool = True, - datascale="log2") -> dict: - """Create a new ProbeSet dataset.""" - with conn.cursor(cursorclass=DictCursor) as cursor: - dataset = { - "studyid": studyid, - "averageid": averageid, - "name2": datasetname, - "fname": datasetfullname, - "name": datasetshortname, - "sname": datasetshortname, - "today": date.today().isoformat(), - "public": 2 if public else 0, - "authorisedusers": "williamslab", - "datascale": datascale - } - cursor.execute( - """ - INSERT INTO ProbeSetFreeze( - ProbeFreezeId, AvgId, Name, Name2, FullName, ShortName, - CreateTime, public, AuthorisedUsers, DataScale) - VALUES( - %(studyid)s, %(averageid)s, %(name)s, %(name2)s, %(fname)s, - %(sname)s, %(today)s, %(public)s, %(authorisedusers)s, - %(datascale)s) - """, - dataset) - return {**dataset, "datasetid": cursor.lastrowid} - -def probeset_dataset_by_id(conn: mdb.Connection, datasetid) -> Optional[dict]: - """Fetch a ProbeSet dataset by its ID""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Id=%s", (datasetid,)) - result = cursor.fetchone() - if bool(result): - return dict(result) - - return None diff --git a/qc_app/db/platforms.py b/qc_app/db/platforms.py deleted file mode 100644 index cb527a7..0000000 --- a/qc_app/db/platforms.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Handle db interactions for platforms.""" -from typing import Optional - -import MySQLdb as mdb -from MySQLdb.cursors import DictCursor - -def platforms_by_species( - conn: mdb.Connection, speciesid: int) -> tuple[dict, ...]: - """Retrieve platforms by the species""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM GeneChip WHERE SpeciesId=%s " - "ORDER BY GeneChipName ASC", - (speciesid,)) - return tuple(dict(row) for row in cursor.fetchall()) - -def platform_by_id(conn: mdb.Connection, platformid: int) -> Optional[dict]: - """Retrieve a platform by its ID""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM GeneChip WHERE Id=%s", - (platformid,)) - result = cursor.fetchone() - if bool(result): - return dict(result) - - return None diff --git a/qc_app/db/populations.py b/qc_app/db/populations.py deleted file mode 100644 index 4485e52..0000000 --- a/qc_app/db/populations.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Functions for accessing the database relating to species populations.""" -import MySQLdb as mdb -from MySQLdb.cursors import DictCursor - -def population_by_id(conn: mdb.Connection, population_id) -> dict: - """Get the grouping/population by id.""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM InbredSet WHERE InbredSetId=%s", - (population_id,)) - return cursor.fetchone() - -def population_by_species_and_id( - conn: mdb.Connection, species_id, population_id) -> dict: - """Retrieve a population by its identifier and species.""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM InbredSet WHERE SpeciesId=%s AND Id=%s", - (species_id, population_id)) - return cursor.fetchone() - -def populations_by_species(conn: mdb.Connection, speciesid) -> tuple: - "Retrieve group (InbredSet) information from the database." - with conn.cursor(cursorclass=DictCursor) as cursor: - query = "SELECT * FROM InbredSet WHERE SpeciesId=%s" - cursor.execute(query, (speciesid,)) - return tuple(cursor.fetchall()) - - return tuple() - -def save_population(conn: mdb.Connection, population_details: dict) -> dict: - """Save the population details to the db.""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - "INSERT INTO InbredSet(" - "InbredSetId, InbredSetName, Name, SpeciesId, FullName, " - "MenuOrderId, Description" - ") " - "VALUES (" - "%(InbredSetId)s, %(InbredSetName)s, %(Name)s, %(SpeciesId)s, " - "%(FullName)s, %(MenuOrderId)s, %(Description)s" - ")", - { - "MenuOrderId": 0, - "InbredSetId": 0, - **population_details - }) - new_id = cursor.lastrowid - cursor.execute("UPDATE InbredSet SET InbredSetId=%s WHERE Id=%s", - (new_id, new_id)) - return { - **population_details, - "Id": new_id, - "InbredSetId": new_id, - "population_id": new_id - } diff --git a/qc_app/db/species.py b/qc_app/db/species.py deleted file mode 100644 index 653e59b..0000000 --- a/qc_app/db/species.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Database functions for species.""" -import MySQLdb as mdb -from MySQLdb.cursors import DictCursor - -def species(conn: mdb.Connection) -> tuple: - "Retrieve the species from the database." - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - "SELECT SpeciesId, SpeciesName, LOWER(Name) AS Name, MenuName, " - "FullName FROM Species") - return tuple(cursor.fetchall()) - - return tuple() - -def species_by_id(conn: mdb.Connection, speciesid) -> dict: - "Retrieve the species from the database by id." - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - "SELECT SpeciesId, SpeciesName, LOWER(Name) AS Name, MenuName, " - "FullName FROM Species WHERE SpeciesId=%s", - (speciesid,)) - return cursor.fetchone() diff --git a/qc_app/db/tissues.py b/qc_app/db/tissues.py deleted file mode 100644 index 9fe7bab..0000000 --- a/qc_app/db/tissues.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Handle db interactions for tissue.""" -from typing import Union, Optional - -import MySQLdb as mdb -from MySQLdb.cursors import DictCursor - -def all_tissues(conn: mdb.Connection) -> tuple[dict, ...]: - """All available tissue.""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM Tissue ORDER BY TissueName") - return tuple(dict(row) for row in cursor.fetchall()) - - -def tissue_by_id(conn: mdb.Connection, tissueid) -> Optional[dict]: - """Retrieve a tissue by its ID""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM Tissue WHERE Id=%s", (tissueid,)) - result = cursor.fetchone() - if bool(result): - return dict(result) - - return None - - -def create_new_tissue( - conn: mdb.Connection, - name: str, - shortname: str, - birnlexid: Optional[str] = None, - birnlexname: Optional[str] = None -) -> dict[str, Union[int, str, None]]: - """Add a new tissue, organ or biological material to the database.""" - with conn.cursor() as cursor: - cursor.execute( - "INSERT INTO " - "Tissue(TissueName, Name, Short_Name, BIRN_lex_ID, BIRN_lex_Name) " - "VALUES (%s, %s, %s, %s, %s)", - (name, name, shortname, birnlexid, birnlexname)) - tissueid = cursor.lastrowid - cursor.execute("UPDATE Tissue SET TissueId=%s WHERE Id=%s", - (tissueid, tissueid)) - return { - "Id": tissueid, - "TissueId": tissueid, - "TissueName": name, - "Name": name, - "Short_Name": shortname, - "BIRN_lex_ID": birnlexid, - "BIRN_lex_Name": birnlexname - } diff --git a/qc_app/db_utils.py b/qc_app/db_utils.py deleted file mode 100644 index ef26398..0000000 --- a/qc_app/db_utils.py +++ /dev/null @@ -1,46 +0,0 @@ -"""module contains all db related stuff""" -import logging -import traceback -import contextlib -from urllib.parse import urlparse -from typing import Any, Tuple, Optional, Iterator, Callable - -import MySQLdb as mdb -from redis import Redis -from flask import current_app as app - -def parse_db_url(db_url) -> Tuple: - """ - Parse SQL_URI configuration variable. - """ - parsed_db = urlparse(db_url) - return (parsed_db.hostname, parsed_db.username, - parsed_db.password, parsed_db.path[1:], parsed_db.port) - - -@contextlib.contextmanager -def database_connection(db_url: Optional[str] = None) -> Iterator[mdb.Connection]: - """function to create db connector""" - host, user, passwd, db_name, db_port = parse_db_url( - db_url or app.config["SQL_URI"]) - connection = mdb.connect( - host, user, passwd, db_name, port=(db_port or 3306)) - try: - yield connection - connection.commit() - except mdb.Error as _mdb_err: - logging.error(traceback.format_exc()) - connection.rollback() - finally: - connection.close() - -def with_db_connection(func: Callable[[mdb.Connection], Any]) -> Any: - """Call `func` with a MySQDdb database connection.""" - with database_connection(app.config["SQL_URI"]) as conn: - return func(conn) - -def with_redis_connection(func: Callable[[Redis], Any]) -> Any: - """Call `func` with a redis connection.""" - redisuri = app.config["REDIS_URL"] - with Redis.from_url(redisuri, decode_responses=True) as rconn: - return func(rconn) diff --git a/qc_app/dbinsert.py b/qc_app/dbinsert.py deleted file mode 100644 index ef08423..0000000 --- a/qc_app/dbinsert.py +++ /dev/null @@ -1,397 +0,0 @@ -"Handle inserting data into the database" -import os -import json -from typing import Union -from functools import reduce -from datetime import datetime - -from redis import Redis -from MySQLdb.cursors import DictCursor -from flask import ( - flash, request, url_for, Blueprint, redirect, render_template, - current_app as app) - -from qc_app.db_utils import with_db_connection, database_connection -from qc_app.db import species, species_by_id, populations_by_species - -from . import jobs - -dbinsertbp = Blueprint("dbinsert", __name__) - -def render_error(error_msg): - "Render the generic error page" - return render_template("dbupdate_error.html", error_message=error_msg), 400 - -def make_menu_items_grouper(grouping_fn=lambda item: item): - "Build function to be used to group menu items." - def __grouper__(acc, row): - grouping = grouping_fn(row[2]) - row_values = (row[0].strip(), row[1].strip()) - if acc.get(grouping) is None: - return {**acc, grouping: (row_values,)} - return {**acc, grouping: (acc[grouping] + (row_values,))} - return __grouper__ - -def genechips(): - "Retrieve the genechip information from the database" - def __organise_by_species__(acc, chip): - speciesid = chip["SpeciesId"] - if acc.get(speciesid) is None: - return {**acc, speciesid: (chip,)} - return {**acc, speciesid: acc[speciesid] + (chip,)} - - with database_connection() as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM GeneChip ORDER BY GeneChipName ASC") - return reduce(__organise_by_species__, cursor.fetchall(), {}) - - return {} - -def platform_by_id(genechipid:int) -> Union[dict, None]: - "Retrieve the gene platform by id" - with database_connection() as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - "SELECT * FROM GeneChip WHERE GeneChipId=%s", - (genechipid,)) - return cursor.fetchone() - -def studies_by_species_and_platform(speciesid:int, genechipid:int) -> tuple: - "Retrieve the studies by the related species and gene platform" - with database_connection() as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - query = ( - "SELECT Species.SpeciesId, ProbeFreeze.* " - "FROM Species INNER JOIN InbredSet " - "ON Species.SpeciesId=InbredSet.SpeciesId " - "INNER JOIN ProbeFreeze " - "ON InbredSet.InbredSetId=ProbeFreeze.InbredSetId " - "WHERE Species.SpeciesId = %s " - "AND ProbeFreeze.ChipId = %s") - cursor.execute(query, (speciesid, genechipid)) - return tuple(cursor.fetchall()) - - return tuple() - -def organise_groups_by_family(acc:dict, group:dict) -> dict: - "Organise the group (InbredSet) information by the group field" - family = group["Family"] - if acc.get(family): - return {**acc, family: acc[family] + (group,)} - return {**acc, family: (group,)} - -def tissues() -> tuple: - "Retrieve type (Tissue) information from the database." - with database_connection() as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM Tissue ORDER BY Name") - return tuple(cursor.fetchall()) - - return tuple() - -@dbinsertbp.route("/platform", methods=["POST"]) -def select_platform(): - "Select the platform (GeneChipId) used for the data." - job_id = request.form["job_id"] - with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, - database_connection(app.config["SQL_URI"]) as conn): - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - if job: - filename = job["filename"] - filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" - if os.path.exists(filepath): - default_species = 1 - gchips = genechips() - return render_template( - "select_platform.html", filename=filename, - filetype=job["filetype"], totallines=int(job["currentline"]), - default_species=default_species, species=species(conn), - genechips=gchips[default_species], - genechips_data=json.dumps(gchips)) - return render_error(f"File '{filename}' no longer exists.") - return render_error(f"Job '{job_id}' no longer exists.") - return render_error("Unknown error") - -@dbinsertbp.route("/study", methods=["POST"]) -def select_study(): - "View to select/create the study (ProbeFreeze) associated with the data." - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - - speciesid = form["species"] - genechipid = form["genechipid"] - - the_studies = studies_by_species_and_platform(speciesid, genechipid) - the_groups = reduce( - organise_groups_by_family, - with_db_connection( - lambda conn: populations_by_species(conn, speciesid)), - {}) - return render_template( - "select_study.html", filename=form["filename"], - filetype=form["filetype"], totallines=form["totallines"], - species=speciesid, genechipid=genechipid, studies=the_studies, - groups=the_groups, tissues = tissues(), - selected_group=int(form.get("inbredsetid", -13)), - selected_tissue=int(form.get("tissueid", -13))) - except AssertionError as aserr: - return render_error(f"Missing data: {aserr.args[0]}") - -@dbinsertbp.route("/create-study", methods=["POST"]) -def create_study(): - "Create a new study (ProbeFreeze)." - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - assert form.get("studyname"), "study name" - assert form.get("inbredsetid"), "group" - assert form.get("tissueid"), "type/tissue" - - with database_connection() as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - values = ( - form["genechipid"], - form["tissueid"], - form["studyname"], - form.get("studyfullname", ""), - form.get("studyshortname", ""), - datetime.now().date().strftime("%Y-%m-%d"), - form["inbredsetid"]) - query = ( - "INSERT INTO ProbeFreeze(" - "ChipId, TissueId, Name, FullName, ShortName, CreateTime, " - "InbredSetId" - ") VALUES (%s, %s, %s, %s, %s, %s, %s)") - cursor.execute(query, values) - new_studyid = cursor.lastrowid - cursor.execute( - "UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s", - (new_studyid, new_studyid)) - flash("Study created successfully", "alert-success") - return render_template( - "continue_from_create_study.html", - filename=form["filename"], filetype=form["filetype"], - totallines=form["totallines"], species=form["species"], - genechipid=form["genechipid"], studyid=new_studyid) - except AssertionError as aserr: - flash(f"Missing data: {aserr.args[0]}", "alert-error") - return redirect(url_for("dbinsert.select_study"), code=307) - -def datasets_by_study(studyid:int) -> tuple: - "Retrieve datasets associated with a study with the ID `studyid`." - with database_connection() as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - query = "SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s" - cursor.execute(query, (studyid,)) - return tuple(cursor.fetchall()) - - return tuple() - -def averaging_methods() -> tuple: - "Retrieve averaging methods from database" - with database_connection() as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM AvgMethod") - return tuple(cursor.fetchall()) - - return tuple() - -def dataset_datascales() -> tuple: - "Retrieve datascales from database" - with database_connection() as conn: - with conn.cursor() as cursor: - cursor.execute( - 'SELECT DISTINCT DataScale FROM ProbeSetFreeze ' - 'WHERE DataScale IS NOT NULL AND DataScale != ""') - return tuple( - item for item in - (res[0].strip() for res in cursor.fetchall()) - if (item is not None and item != "")) - - return tuple() - -@dbinsertbp.route("/dataset", methods=["POST"]) -def select_dataset(): - "Select the dataset to add the file contents against" - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - assert form.get("studyid"), "study" - - studyid = form["studyid"] - datasets = datasets_by_study(studyid) - return render_template( - "select_dataset.html", **{**form, "studyid": studyid}, - datasets=datasets, avgmethods=averaging_methods(), - datascales=dataset_datascales()) - except AssertionError as aserr: - return render_error(f"Missing data: {aserr.args[0]}") - -@dbinsertbp.route("/create-dataset", methods=["POST"]) -def create_dataset(): - "Select the dataset to add the file contents against" - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - assert form.get("studyid"), "study" - assert form.get("avgid"), "averaging method" - assert form.get("datasetname2"), "Dataset Name 2" - assert form.get("datasetfullname"), "Dataset Full Name" - assert form.get("datasetshortname"), "Dataset Short Name" - assert form.get("datasetpublic"), "Dataset public specification" - assert form.get("datasetconfidentiality"), "Dataset confidentiality" - assert form.get("datasetdatascale"), "Dataset Datascale" - - with database_connection() as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - datasetname = form["datasetname"] - cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Name=%s", - (datasetname,)) - results = cursor.fetchall() - if bool(results): - flash("A dataset with that name already exists.", - "alert-error") - return redirect(url_for("dbinsert.select_dataset"), code=307) - values = ( - form["studyid"], form["avgid"], - datasetname, form["datasetname2"], - form["datasetfullname"], form["datasetshortname"], - datetime.now().date().strftime("%Y-%m-%d"), - form["datasetpublic"], form["datasetconfidentiality"], - "williamslab", form["datasetdatascale"]) - query = ( - "INSERT INTO ProbeSetFreeze(" - "ProbeFreezeId, AvgID, Name, Name2, FullName, " - "ShortName, CreateTime, OrderList, public, " - "confidentiality, AuthorisedUsers, DataScale) " - "VALUES" - "(%s, %s, %s, %s, %s, %s, %s, NULL, %s, %s, %s, %s)") - cursor.execute(query, values) - new_datasetid = cursor.lastrowid - return render_template( - "continue_from_create_dataset.html", - filename=form["filename"], filetype=form["filetype"], - species=form["species"], genechipid=form["genechipid"], - studyid=form["studyid"], datasetid=new_datasetid, - totallines=form["totallines"]) - except AssertionError as aserr: - flash(f"Missing data {aserr.args[0]}", "alert-error") - return redirect(url_for("dbinsert.select_dataset"), code=307) - -def study_by_id(studyid:int) -> Union[dict, None]: - "Get a study by its Id" - with database_connection() as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - "SELECT * FROM ProbeFreeze WHERE Id=%s", - (studyid,)) - return cursor.fetchone() - -def dataset_by_id(datasetid:int) -> Union[dict, None]: - "Retrieve a dataset by its id" - with database_connection() as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - ("SELECT AvgMethod.Name AS AvgMethodName, ProbeSetFreeze.* " - "FROM ProbeSetFreeze INNER JOIN AvgMethod " - "ON ProbeSetFreeze.AvgId=AvgMethod.AvgMethodId " - "WHERE ProbeSetFreeze.Id=%s"), - (datasetid,)) - return cursor.fetchone() - -def selected_keys(original: dict, keys: tuple) -> dict: - "Return a new dict from the `original` dict with only `keys` present." - return {key: value for key,value in original.items() if key in keys} - -@dbinsertbp.route("/final-confirmation", methods=["POST"]) -def final_confirmation(): - "Preview the data before triggering entry into the database" - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - assert form.get("studyid"), "study" - assert form.get("datasetid"), "dataset" - - speciesid = form["species"] - genechipid = form["genechipid"] - studyid = form["studyid"] - datasetid=form["datasetid"] - return render_template( - "final_confirmation.html", filename=form["filename"], - filetype=form["filetype"], totallines=form["totallines"], - species=speciesid, genechipid=genechipid, studyid=studyid, - datasetid=datasetid, the_species=selected_keys( - with_db_connection(lambda conn: species_by_id(conn, speciesid)), - ("SpeciesName", "Name", "MenuName")), - platform=selected_keys( - platform_by_id(genechipid), - ("GeneChipName", "Name", "GeoPlatform", "Title", "GO_tree_value")), - study=selected_keys( - study_by_id(studyid), ("Name", "FullName", "ShortName")), - dataset=selected_keys( - dataset_by_id(datasetid), - ("AvgMethodName", "Name", "Name2", "FullName", "ShortName", - "DataScale"))) - except AssertionError as aserr: - return render_error(f"Missing data: {aserr.args[0]}") - -@dbinsertbp.route("/insert-data", methods=["POST"]) -def insert_data(): - "Trigger data insertion" - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - assert form.get("studyid"), "study" - assert form.get("datasetid"), "dataset" - - filename = form["filename"] - filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" - redisurl = app.config["REDIS_URL"] - if os.path.exists(filepath): - with Redis.from_url(redisurl, decode_responses=True) as rconn: - job = jobs.launch_job( - jobs.data_insertion_job( - rconn, filepath, form["filetype"], form["totallines"], - form["species"], form["genechipid"], form["datasetid"], - app.config["SQL_URI"], redisurl, - app.config["JOBS_TTL_SECONDS"]), - redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors") - - return redirect(url_for("dbinsert.insert_status", job_id=job["jobid"])) - return render_error(f"File '{filename}' no longer exists.") - except AssertionError as aserr: - return render_error(f"Missing data: {aserr.args[0]}") - -@dbinsertbp.route("/status/<job_id>", methods=["GET"]) -def insert_status(job_id: str): - "Retrieve status of data insertion." - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - - if job: - job_status = job["status"] - if job_status == "success": - return render_template("insert_success.html", job=job) - if job["status"] == "error": - return render_template("insert_error.html", job=job) - return render_template("insert_progress.html", job=job) - return render_template("no_such_job.html", job_id=job_id), 400 diff --git a/qc_app/entry.py b/qc_app/entry.py deleted file mode 100644 index f2db878..0000000 --- a/qc_app/entry.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Entry-point module""" -import os -import mimetypes -from typing import Tuple -from zipfile import ZipFile, is_zipfile - -from werkzeug.utils import secure_filename -from flask import ( - flash, - request, - url_for, - redirect, - Blueprint, - render_template, - current_app as app, - send_from_directory) - -from qc_app.db import species -from qc_app.db_utils import with_db_connection - -entrybp = Blueprint("entry", __name__) - -@entrybp.route("/favicon.ico", methods=["GET"]) -def favicon(): - """Return the favicon.""" - return send_from_directory(os.path.join(app.root_path, "static"), - "images/CITGLogo.png", - mimetype="image/png") - - -def errors(rqst) -> Tuple[str, ...]: - """Return a tuple of the errors found in the request `rqst`. If no error is - found, then an empty tuple is returned.""" - def __filetype_error__(): - return ( - ("Invalid file type provided.",) - if rqst.form.get("filetype") not in ("average", "standard-error") - else tuple()) - - def __file_missing_error__(): - return ( - ("No file was uploaded.",) - if ("qc_text_file" not in rqst.files or - rqst.files["qc_text_file"].filename == "") - else tuple()) - - def __file_mimetype_error__(): - text_file = rqst.files["qc_text_file"] - return ( - ( - ("Invalid file! Expected a tab-separated-values file, or a zip " - "file of the a tab-separated-values file."),) - if text_file.mimetype not in ( - "text/plain", "text/tab-separated-values", - "application/zip") - else tuple()) - - return ( - __filetype_error__() + - (__file_missing_error__() or __file_mimetype_error__())) - -def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]: - """Check the uploaded zip file for errors.""" - zfile_errors: Tuple[str, ...] = tuple() - if is_zipfile(filepath): - with ZipFile(filepath, "r") as zfile: - infolist = zfile.infolist() - if len(infolist) != 1: - zfile_errors = zfile_errors + ( - ("Expected exactly one (1) member file within the uploaded zip " - f"file. Got {len(infolist)} member files."),) - if len(infolist) == 1 and infolist[0].is_dir(): - zfile_errors = zfile_errors + ( - ("Expected a member text file in the uploaded zip file. Got a " - "directory/folder."),) - - if len(infolist) == 1 and not infolist[0].is_dir(): - zfile.extract(infolist[0], path=upload_dir) - mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}") - if mime[0] != "text/tab-separated-values": - zfile_errors = zfile_errors + ( - ("Expected the member text file in the uploaded zip file to" - " be a tab-separated file."),) - - return zfile_errors - -@entrybp.route("/", methods=["GET"]) -def index(): - """Load the landing page""" - return render_template("index.html") - -@entrybp.route("/upload", methods=["GET", "POST"]) -def upload_file(): - """Enables uploading the files""" - if request.method == "GET": - return render_template( - "select_species.html", species=with_db_connection(species)) - - upload_dir = app.config["UPLOAD_FOLDER"] - request_errors = errors(request) - if request_errors: - for error in request_errors: - flash(error, "alert-danger error-expr-data") - return redirect(url_for("entry.upload_file")) - - filename = secure_filename(request.files["qc_text_file"].filename) - if not os.path.exists(upload_dir): - os.mkdir(upload_dir) - - filepath = os.path.join(upload_dir, filename) - request.files["qc_text_file"].save(os.path.join(upload_dir, filename)) - - zip_errors = zip_file_errors(filepath, upload_dir) - if zip_errors: - for error in zip_errors: - flash(error, "alert-danger error-expr-data") - return redirect(url_for("entry.upload_file")) - - return redirect(url_for("parse.parse", - speciesid=request.form["speciesid"], - filename=filename, - filetype=request.form["filetype"])) - -@entrybp.route("/data-review", methods=["GET"]) -def data_review(): - """Provide some help on data expectations to the user.""" - return render_template("data_review.html") diff --git a/qc_app/errors.py b/qc_app/errors.py deleted file mode 100644 index 3e7c893..0000000 --- a/qc_app/errors.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Application error handling.""" -import traceback -from werkzeug.exceptions import HTTPException - -import MySQLdb as mdb -from flask import Flask, request, render_template, current_app as app - -def handle_general_exception(exc: Exception): - """Handle generic exceptions.""" - trace = traceback.format_exc() - app.logger.error( - "Error (%s.%s): Generic unhandled exception!! (URI: %s)\n%s", - exc.__class__.__module__, exc.__class__.__name__, request.url, trace) - return render_template("unhandled_exception.html", trace=trace), 500 - -def handle_http_exception(exc: HTTPException): - """Handle HTTP exceptions.""" - app.logger.error( - "HTTP Error %s: %s", exc.code, exc.description, exc_info=True) - return render_template("http-error.html", - request_url=request.url, - exc=exc, - trace=traceback.format_exception(exc)), exc.code - -def register_error_handlers(appl: Flask): - """Register top-level error/exception handlers.""" - appl.register_error_handler(Exception, handle_general_exception) - appl.register_error_handler(HTTPException, handle_http_exception) - appl.register_error_handler(mdb.MySQLError, handle_general_exception) diff --git a/qc_app/files.py b/qc_app/files.py deleted file mode 100644 index b163612..0000000 --- a/qc_app/files.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Utilities to deal with uploaded files.""" -import hashlib -from pathlib import Path -from datetime import datetime -from flask import current_app - -from werkzeug.utils import secure_filename -from werkzeug.datastructures import FileStorage - -def save_file(fileobj: FileStorage, upload_dir: Path) -> Path: - """Save the uploaded file and return the path.""" - assert bool(fileobj), "Invalid file object!" - hashed_name = hashlib.sha512( - f"{fileobj.filename}::{datetime.now().isoformat()}".encode("utf8") - ).hexdigest() - filename = Path(secure_filename(hashed_name)) # type: ignore[arg-type] - if not upload_dir.exists(): - upload_dir.mkdir() - - filepath = Path(upload_dir, filename) - fileobj.save(filepath) - return filepath - -def fullpath(filename: str): - """Get a file's full path. This makes use of `flask.current_app`.""" - return Path(current_app.config["UPLOAD_FOLDER"], filename).absolute() diff --git a/qc_app/input_validation.py b/qc_app/input_validation.py deleted file mode 100644 index 9abe742..0000000 --- a/qc_app/input_validation.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Input validation utilities""" -from typing import Any - -def is_empty_string(value: str) -> bool: - """Check whether as string is empty""" - return (isinstance(value, str) and value.strip() == "") - -def is_empty_input(value: Any) -> bool: - """Check whether user provided an empty value.""" - return (value is None or is_empty_string(value)) - -def is_integer_input(value: Any) -> bool: - """ - Check whether user provided a value that can be parsed into an integer. - """ - def __is_int__(val, base): - try: - int(val, base=base) - except ValueError: - return False - return True - return isinstance(value, int) or ( - (not is_empty_input(value)) and ( - isinstance(value, str) and ( - __is_int__(value, 10) - or __is_int__(value, 8) - or __is_int__(value, 16)))) diff --git a/qc_app/jobs.py b/qc_app/jobs.py deleted file mode 100644 index 21889da..0000000 --- a/qc_app/jobs.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Handle jobs""" -import os -import sys -import shlex -import subprocess -from uuid import UUID, uuid4 -from datetime import timedelta -from typing import Union, Optional - -from redis import Redis -from flask import current_app as app - -JOBS_PREFIX = "JOBS" - -class JobNotFound(Exception): - """Raised if we try to retrieve a non-existent job.""" - -def jobsnamespace(): - """ - Return the jobs namespace prefix. It depends on app configuration. - - Calling this function outside of an application context will cause an - exception to be raised. It is mostly a convenience utility to use within the - application. - """ - return f"{app.config['GNQC_REDIS_PREFIX']}:{JOBS_PREFIX}" - -def job_key(namespaceprefix: str, jobid: Union[str, UUID]) -> str: - """Build the key by appending it to the namespace prefix.""" - return f"{namespaceprefix}:{jobid}" - -def raise_jobnotfound(rprefix:str, jobid: Union[str,UUID]): - """Utility to raise a `NoSuchJobError`""" - raise JobNotFound(f"Could not retrieve job '{jobid}' from '{rprefix}.") - -def error_filename(jobid, error_dir): - "Compute the path of the file where errors will be dumped." - return f"{error_dir}/job_{jobid}.error" - -def initialise_job(# pylint: disable=[too-many-arguments] - rconn: Redis, rprefix: str, jobid: str, command: list, job_type: str, - ttl_seconds: int = 86400, extra_meta: Optional[dict] = None) -> dict: - "Initialise a job 'object' and put in on redis" - the_job = { - "jobid": jobid, "command": shlex.join(command), "status": "pending", - "percent": 0, "job-type": job_type, **(extra_meta or {}) - } - rconn.hset(job_key(rprefix, jobid), mapping=the_job) - rconn.expire( - name=job_key(rprefix, jobid), time=timedelta(seconds=ttl_seconds)) - return the_job - -def build_file_verification_job(#pylint: disable=[too-many-arguments] - redis_conn: Redis, - dburi: str, - redisuri: str, - speciesid: int, - filepath: str, - filetype: str, - ttl_seconds: int): - "Build a file verification job" - jobid = str(uuid4()) - command = [ - sys.executable, "-m", "scripts.validate_file", - dburi, redisuri, jobsnamespace(), jobid, - "--redisexpiry", str(ttl_seconds), - str(speciesid), filetype, filepath, - ] - return initialise_job( - redis_conn, jobsnamespace(), jobid, command, "file-verification", - ttl_seconds, { - "filetype": filetype, - "filename": os.path.basename(filepath), "percent": 0 - }) - -def data_insertion_job(# pylint: disable=[too-many-arguments] - redis_conn: Redis, filepath: str, filetype: str, totallines: int, - speciesid: int, platformid: int, datasetid: int, databaseuri: str, - redisuri: str, ttl_seconds: int) -> dict: - "Build a data insertion job" - jobid = str(uuid4()) - command = [ - sys.executable, "-m", "scripts.insert_data", filetype, filepath, - speciesid, platformid, datasetid, databaseuri, redisuri - ] - return initialise_job( - redis_conn, jobsnamespace(), jobid, command, "data-insertion", - ttl_seconds, { - "filename": os.path.basename(filepath), "filetype": filetype, - "totallines": totallines - }) - -def launch_job(the_job: dict, redisurl: str, error_dir): - """Launch a job in the background""" - if not os.path.exists(error_dir): - os.mkdir(error_dir) - - jobid = the_job["jobid"] - with open(error_filename(jobid, error_dir), - "w", - encoding="utf-8") as errorfile: - subprocess.Popen( # pylint: disable=[consider-using-with] - [sys.executable, "-m", "scripts.worker", redisurl, jobsnamespace(), - jobid], - stderr=errorfile, - env={"PYTHONPATH": ":".join(sys.path)}) - - return the_job - -def job(rconn: Redis, rprefix: str, jobid: Union[str,UUID]): - "Retrieve the job" - thejob = (rconn.hgetall(job_key(rprefix, jobid)) or - raise_jobnotfound(rprefix, jobid)) - return thejob - -def update_status( - rconn: Redis, rprefix: str, jobid: Union[str, UUID], status: str): - """Update status of job in redis.""" - rconn.hset(name=job_key(rprefix, jobid), key="status", value=status) - -def update_stdout_stderr(rconn: Redis, - rprefix: str, - jobid: Union[str, UUID], - bytes_read: bytes, - stream: str): - "Update the stdout/stderr keys according to the value of `stream`." - thejob = job(rconn, rprefix, jobid) - contents = thejob.get(stream, '') - new_contents = contents + bytes_read.decode("utf-8") - rconn.hset(name=job_key(rprefix, jobid), key=stream, value=new_contents) diff --git a/qc_app/parse.py b/qc_app/parse.py deleted file mode 100644 index d20f6f0..0000000 --- a/qc_app/parse.py +++ /dev/null @@ -1,175 +0,0 @@ -"""File parsing module""" -import os - -import jsonpickle -from redis import Redis -from flask import flash, request, url_for, redirect, Blueprint, render_template -from flask import current_app as app - -from quality_control.errors import InvalidValue, DuplicateHeading - -from qc_app import jobs -from qc_app.dbinsert import species_by_id -from qc_app.db_utils import with_db_connection - -parsebp = Blueprint("parse", __name__) - -def isinvalidvalue(item): - """Check whether item is of type InvalidValue""" - return isinstance(item, InvalidValue) - -def isduplicateheading(item): - """Check whether item is of type DuplicateHeading""" - return isinstance(item, DuplicateHeading) - -@parsebp.route("/parse", methods=["GET"]) -def parse(): - """Trigger file parsing""" - errors = False - speciesid = request.args.get("speciesid") - filename = request.args.get("filename") - filetype = request.args.get("filetype") - if speciesid is None: - flash("No species selected", "alert-error error-expr-data") - errors = True - else: - try: - speciesid = int(speciesid) - species = with_db_connection( - lambda con: species_by_id(con, speciesid)) - if not bool(species): - flash("No such species.", "alert-error error-expr-data") - errors = True - except ValueError: - flash("Invalid speciesid provided. Expected an integer.", - "alert-error error-expr-data") - errors = True - - if filename is None: - flash("No file provided", "alert-error error-expr-data") - errors = True - - if filetype is None: - flash("No filetype provided", "alert-error error-expr-data") - errors = True - - if filetype not in ("average", "standard-error"): - flash("Invalid filetype provided", "alert-error error-expr-data") - errors = True - - if filename: - filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename) - if not os.path.exists(filepath): - flash("Selected file does not exist (any longer)", - "alert-error error-expr-data") - errors = True - - if errors: - return redirect(url_for("entry.upload_file")) - - redisurl = app.config["REDIS_URL"] - with Redis.from_url(redisurl, decode_responses=True) as rconn: - job = jobs.launch_job( - jobs.build_file_verification_job( - rconn, app.config["SQL_URI"], redisurl, - speciesid, filepath, filetype, - app.config["JOBS_TTL_SECONDS"]), - redisurl, - f"{app.config['UPLOAD_FOLDER']}/job_errors") - - return redirect(url_for("parse.parse_status", job_id=job["jobid"])) - -@parsebp.route("/status/<job_id>", methods=["GET"]) -def parse_status(job_id: str): - "Retrieve the status of the job" - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: - try: - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - except jobs.JobNotFound as _exc: - return render_template("no_such_job.html", job_id=job_id), 400 - - error_filename = jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") - if os.path.exists(error_filename): - stat = os.stat(error_filename) - if stat.st_size > 0: - return redirect(url_for("parse.fail", job_id=job_id)) - - job_id = job["jobid"] - progress = float(job["percent"]) - status = job["status"] - filename = job.get("filename", "uploaded file") - errors = jsonpickle.decode( - job.get("errors", jsonpickle.encode(tuple()))) - if status in ("success", "aborted"): - return redirect(url_for("parse.results", job_id=job_id)) - - if status == "parse-error": - return redirect(url_for("parse.fail", job_id=job_id)) - - app.jinja_env.globals.update( - isinvalidvalue=isinvalidvalue, - isduplicateheading=isduplicateheading) - return render_template( - "job_progress.html", - job_id = job_id, - job_status = status, - progress = progress, - message = job.get("message", ""), - job_name = f"Parsing '{filename}'", - errors=errors) - -@parsebp.route("/results/<job_id>", methods=["GET"]) -def results(job_id: str): - """Show results of parsing...""" - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - - if job: - filename = job["filename"] - errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple()))) - app.jinja_env.globals.update( - isinvalidvalue=isinvalidvalue, - isduplicateheading=isduplicateheading) - return render_template( - "parse_results.html", - errors=errors, - job_name = f"Parsing '{filename}'", - user_aborted = job.get("user_aborted"), - job_id=job["jobid"]) - - return render_template("no_such_job.html", job_id=job_id) - -@parsebp.route("/fail/<job_id>", methods=["GET"]) -def fail(job_id: str): - """Handle parsing failure""" - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - - if job: - error_filename = jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") - if os.path.exists(error_filename): - stat = os.stat(error_filename) - if stat.st_size > 0: - return render_template( - "worker_failure.html", job_id=job_id) - - return render_template("parse_failure.html", job=job) - - return render_template("no_such_job.html", job_id=job_id) - -@parsebp.route("/abort", methods=["POST"]) -def abort(): - """Handle user request to abort file processing""" - job_id = request.form["job_id"] - - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - - if job: - rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id), - key="user_aborted", - value=int(True)) - - return redirect(url_for("parse.parse_status", job_id=job_id)) diff --git a/qc_app/samples.py b/qc_app/samples.py deleted file mode 100644 index 804f262..0000000 --- a/qc_app/samples.py +++ /dev/null @@ -1,354 +0,0 @@ -"""Code regarding samples""" -import os -import sys -import csv -import uuid -from pathlib import Path -from typing import Iterator - -import MySQLdb as mdb -from redis import Redis -from MySQLdb.cursors import DictCursor -from flask import ( - flash, - request, - url_for, - redirect, - Blueprint, - render_template, - current_app as app) - -from functional_tools import take - -from qc_app import jobs -from qc_app.files import save_file -from qc_app.input_validation import is_integer_input -from qc_app.db_utils import ( - with_db_connection, - database_connection, - with_redis_connection) -from qc_app.db import ( - species_by_id, - save_population, - population_by_id, - populations_by_species, - species as fetch_species) - -samples = Blueprint("samples", __name__) - -@samples.route("/upload/species", methods=["GET", "POST"]) -def select_species(): - """Select the species.""" - if request.method == "GET": - return render_template("samples/select-species.html", - species=with_db_connection(fetch_species)) - - index_page = redirect(url_for("entry.upload_file")) - species_id = request.form.get("species_id") - if bool(species_id): - species_id = int(species_id) - species = with_db_connection( - lambda conn: species_by_id(conn, species_id)) - if bool(species): - return redirect(url_for( - "samples.select_population", species_id=species_id)) - flash("Invalid species selected!", "alert-error") - flash("You need to select a species", "alert-error") - return index_page - -@samples.route("/upload/species/<int:species_id>/create-population", - methods=["POST"]) -def create_population(species_id: int): - """Create new grouping/population.""" - if not is_integer_input(species_id): - flash("You did not provide a valid species. Please select one to " - "continue.", - "alert-danger") - return redirect(url_for("samples.select_species")) - species = with_db_connection(lambda conn: species_by_id(conn, species_id)) - if not bool(species): - flash("Species with given ID was not found.", "alert-danger") - return redirect(url_for("samples.select_species")) - - species_page = redirect(url_for("samples.select_species"), code=307) - with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - pop_name = request.form.get("inbredset_name", "").strip() - pop_fullname = request.form.get("inbredset_fullname", "").strip() - - if not bool(species): - flash("Invalid species!", "alert-error error-create-population") - return species_page - if (not bool(pop_name)) or (not bool(pop_fullname)): - flash("You *MUST* provide a grouping/population name", - "alert-error error-create-population") - return species_page - - pop = save_population(conn, { - "SpeciesId": species["SpeciesId"], - "Name": pop_name, - "InbredSetName": pop_fullname, - "FullName": pop_fullname, - "Family": request.form.get("inbredset_family") or None, - "Description": request.form.get("description") or None - }) - - flash("Grouping/Population created successfully.", "alert-success") - return redirect(url_for("samples.upload_samples", - species_id=species_id, - population_id=pop["population_id"])) - -@samples.route("/upload/species/<int:species_id>/population", - methods=["GET", "POST"]) -def select_population(species_id: int): - """Select from existing groupings/populations.""" - if not is_integer_input(species_id): - flash("You did not provide a valid species. Please select one to " - "continue.", - "alert-danger") - return redirect(url_for("samples.select_species")) - species = with_db_connection(lambda conn: species_by_id(conn, species_id)) - if not bool(species): - flash("Species with given ID was not found.", "alert-danger") - return redirect(url_for("samples.select_species")) - - if request.method == "GET": - return render_template( - "samples/select-population.html", - species=species, - populations=with_db_connection( - lambda conn: populations_by_species(conn, species_id))) - - population_page = redirect(url_for( - "samples.select_population", species_id=species_id), code=307) - _population_id = request.form.get("inbredset_id") - if not is_integer_input(_population_id): - flash("You did not provide a valid population. Please select one to " - "continue.", - "alert-danger") - return population_page - population = with_db_connection( - lambda conn: population_by_id(conn, _population_id)) - if not bool(population): - flash("Invalid grouping/population!", - "alert-error error-select-population") - return population_page - - return redirect(url_for("samples.upload_samples", - species_id=species_id, - population_id=_population_id), - code=307) - -def read_samples_file(filepath, separator: str, firstlineheading: bool, **kwargs) -> Iterator[dict]: - """Read the samples file.""" - with open(filepath, "r", encoding="utf-8") as inputfile: - reader = csv.DictReader( - inputfile, - fieldnames=( - None if firstlineheading - else ("Name", "Name2", "Symbol", "Alias")), - delimiter=separator, - quotechar=kwargs.get("quotechar", '"')) - for row in reader: - yield row - -def save_samples_data(conn: mdb.Connection, - speciesid: int, - file_data: Iterator[dict]): - """Save the samples to DB.""" - data = ({**row, "SpeciesId": speciesid} for row in file_data) - total = 0 - with conn.cursor() as cursor: - while True: - batch = take(data, 5000) - if len(batch) == 0: - break - cursor.executemany( - "INSERT INTO Strain(Name, Name2, SpeciesId, Symbol, Alias) " - "VALUES(" - " %(Name)s, %(Name2)s, %(SpeciesId)s, %(Symbol)s, %(Alias)s" - ") ON DUPLICATE KEY UPDATE Name=Name", - batch) - total += len(batch) - print(f"\tSaved {total} samples total so far.") - -def cross_reference_samples(conn: mdb.Connection, - species_id: int, - population_id: int, - strain_names: Iterator[str]): - """Link samples to their population.""" - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - "SELECT MAX(OrderId) AS loid FROM StrainXRef WHERE InbredSetId=%s", - (population_id,)) - last_order_id = (cursor.fetchone()["loid"] or 10) - total = 0 - while True: - batch = take(strain_names, 5000) - if len(batch) == 0: - break - params_str = ", ".join(["%s"] * len(batch)) - ## This query is slow -- investigate. - cursor.execute( - "SELECT s.Id FROM Strain AS s LEFT JOIN StrainXRef AS sx " - "ON s.Id = sx.StrainId WHERE s.SpeciesId=%s AND s.Name IN " - f"({params_str}) AND sx.StrainId IS NULL", - (species_id,) + tuple(batch)) - strain_ids = (sid["Id"] for sid in cursor.fetchall()) - params = tuple({ - "pop_id": population_id, - "strain_id": strain_id, - "order_id": last_order_id + (order_id * 10), - "mapping": "N", - "pedigree": None - } for order_id, strain_id in enumerate(strain_ids, start=1)) - cursor.executemany( - "INSERT INTO StrainXRef( " - " InbredSetId, StrainId, OrderId, Used_for_mapping, PedigreeStatus" - ")" - "VALUES (" - " %(pop_id)s, %(strain_id)s, %(order_id)s, %(mapping)s, " - " %(pedigree)s" - ")", - params) - last_order_id += (len(params) * 10) - total += len(batch) - print(f"\t{total} total samples cross-referenced to the population " - "so far.") - -def build_sample_upload_job(# pylint: disable=[too-many-arguments] - speciesid: int, - populationid: int, - samplesfile: Path, - separator: str, - firstlineheading: bool, - quotechar: str): - """Define the async command to run the actual samples data upload.""" - return [ - sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"], - str(speciesid), str(populationid), str(samplesfile.absolute()), - separator, f"--redisuri={app.config['REDIS_URL']}", - f"--quotechar={quotechar}" - ] + (["--firstlineheading"] if firstlineheading else []) - -@samples.route("/upload/species/<int:species_id>/populations/<int:population_id>/samples", - methods=["GET", "POST"]) -def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements] - """Upload the samples.""" - samples_uploads_page = redirect(url_for("samples.upload_samples", - species_id=species_id, - population_id=population_id)) - if not is_integer_input(species_id): - flash("You did not provide a valid species. Please select one to " - "continue.", - "alert-danger") - return redirect(url_for("samples.select_species")) - species = with_db_connection(lambda conn: species_by_id(conn, species_id)) - if not bool(species): - flash("Species with given ID was not found.", "alert-danger") - return redirect(url_for("samples.select_species")) - - if not is_integer_input(population_id): - flash("You did not provide a valid population. Please select one " - "to continue.", - "alert-danger") - return redirect(url_for("samples.select_population", - species_id=species_id), - code=307) - population = with_db_connection( - lambda conn: population_by_id(conn, int(population_id))) - if not bool(population): - flash("Invalid grouping/population!", "alert-error") - return redirect(url_for("samples.select_population", - species_id=species_id), - code=307) - - if request.method == "GET" or request.files.get("samples_file") is None: - return render_template("samples/upload-samples.html", - species=species, - population=population) - - try: - samples_file = save_file(request.files["samples_file"], - Path(app.config["UPLOAD_FOLDER"])) - except AssertionError: - flash("You need to provide a file with the samples data.", - "alert-error") - return samples_uploads_page - - firstlineheading = (request.form.get("first_line_heading") == "on") - - separator = request.form.get("separator", ",") - if separator == "other": - separator = request.form.get("other_separator", ",") - if not bool(separator): - flash("You need to provide a separator character.", "alert-error") - return samples_uploads_page - - quotechar = (request.form.get("field_delimiter", '"') or '"') - - redisuri = app.config["REDIS_URL"] - with Redis.from_url(redisuri, decode_responses=True) as rconn: - the_job = jobs.launch_job( - jobs.initialise_job( - rconn, - jobs.jobsnamespace(), - str(uuid.uuid4()), - build_sample_upload_job( - species["SpeciesId"], - population["InbredSetId"], - samples_file, - separator, - firstlineheading, - quotechar), - "samples_upload", - app.config["JOBS_TTL_SECONDS"], - {"job_name": f"Samples Upload: {samples_file.name}"}), - redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") - return redirect(url_for( - "samples.upload_status", job_id=the_job["jobid"])) - -@samples.route("/upload/status/<uuid:job_id>", methods=["GET"]) -def upload_status(job_id: uuid.UUID): - """Check on the status of a samples upload job.""" - job = with_redis_connection(lambda rconn: jobs.job( - rconn, jobs.jobsnamespace(), job_id)) - if job: - status = job["status"] - if status == "success": - return render_template("samples/upload-success.html", job=job) - - if status == "error": - return redirect(url_for("samples.upload_failure", job_id=job_id)) - - error_filename = Path(jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) - if error_filename.exists(): - stat = os.stat(error_filename) - if stat.st_size > 0: - return redirect(url_for( - "samples.upload_failure", job_id=job_id)) - - return render_template( - "samples/upload-progress.html", - job=job) # maybe also handle this? - - return render_template("no_such_job.html", job_id=job_id), 400 - -@samples.route("/upload/failure/<uuid:job_id>", methods=["GET"]) -def upload_failure(job_id: uuid.UUID): - """Display the errors of the samples upload failure.""" - job = with_redis_connection(lambda rconn: jobs.job( - rconn, jobs.jobsnamespace(), job_id)) - if not bool(job): - return render_template("no_such_job.html", job_id=job_id), 400 - - error_filename = Path(jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) - if error_filename.exists(): - stat = os.stat(error_filename) - if stat.st_size > 0: - return render_template("worker_failure.html", job_id=job_id) - - return render_template("samples/upload-failure.html", job=job) diff --git a/qc_app/static/css/custom-bootstrap.css b/qc_app/static/css/custom-bootstrap.css deleted file mode 100644 index 67f1199..0000000 --- a/qc_app/static/css/custom-bootstrap.css +++ /dev/null @@ -1,23 +0,0 @@ -/** Customize some bootstrap selectors **/ -.btn { - text-transform: capitalize; -} - -.navbar-inverse { - background-color: #336699; - border-color: #080808; - color: #FFFFFF; - background-image: none; -} - -.navbar-inverse .navbar-nav>li>a { - color: #FFFFFF; -} - -.navbar-nav > li > a { - padding: 5px; -} - -.navbar { - min-height: 30px; -} diff --git a/qc_app/static/css/styles.css b/qc_app/static/css/styles.css deleted file mode 100644 index a88c229..0000000 --- a/qc_app/static/css/styles.css +++ /dev/null @@ -1,7 +0,0 @@ -.heading { - text-transform: capitalize; -} - -label { - text-transform: capitalize; -} diff --git a/qc_app/static/css/two-column-with-separator.css b/qc_app/static/css/two-column-with-separator.css deleted file mode 100644 index b6efd46..0000000 --- a/qc_app/static/css/two-column-with-separator.css +++ /dev/null @@ -1,27 +0,0 @@ -.two-column-with-separator { - display: grid; - grid-template-columns: 9fr 1fr 9fr; -} - -.two-col-sep-col1 { - grid-column: 1 / 2; -} - -.two-col-sep-separator { - grid-column: 2 / 3; - text-align: center; - color: #FE3535; - font-weight: bolder; -} - -.two-col-sep-col2 { - grid-column: 3 / 4; -} - -.two-col-sep-col1, .two-col-sep-col2 { - border-style: solid; - border-color: #FE3535; - border-width: 1px; - border-radius: 2em; - padding: 2em 3em 2em 3em; -} diff --git a/qc_app/static/images/CITGLogo.png b/qc_app/static/images/CITGLogo.png Binary files differdeleted file mode 100644 index ae99fed..0000000 --- a/qc_app/static/images/CITGLogo.png +++ /dev/null diff --git a/qc_app/static/js/select_platform.js b/qc_app/static/js/select_platform.js deleted file mode 100644 index 4fdd865..0000000 --- a/qc_app/static/js/select_platform.js +++ /dev/null @@ -1,70 +0,0 @@ -function radio_column(chip) { - col = document.createElement("td"); - radio = document.createElement("input"); - radio.setAttribute("type", "radio"); - radio.setAttribute("name", "genechipid"); - radio.setAttribute("value", chip["GeneChipId"]); - radio.setAttribute("required", "required"); - col.appendChild(radio); - return col; -} - -function setup_genechips(genechip_data) { - columns = ["GeneChipId", "GeneChipName"] - submit_button = document.querySelector( - "#select-platform-form button[type='submit']"); - elt = document.getElementById( - "genechips-table").getElementsByTagName("tbody")[0]; - remove_children(elt); - if((genechip_data === undefined) || genechip_data.length === 0) { - row = document.createElement("tr"); - col = document.createElement("td"); - col.setAttribute("colspan", "3"); - text = document.createTextNode("No chips found for selected species"); - col.appendChild(text); - row.appendChild(col); - elt.appendChild(row); - submit_button.setAttribute("disabled", true); - return false; - } - - submit_button.removeAttribute("disabled") - genechip_data.forEach(chip => { - row = document.createElement("tr"); - row.appendChild(radio_column(chip)); - columns.forEach(column => { - col = document.createElement("td"); - content = document.createTextNode(chip[column]); - col.appendChild(content); - row.appendChild(col); - }); - elt.appendChild(row); - }); -} - -function genechips() { - return JSON.parse( - document.getElementById("select-platform-form").getAttribute( - "data-genechips")); -} - -function update_genechips(event) { - genec = genechips(); - - species_elt = document.getElementById("species"); - - if(event.target == species_elt) { - setup_genechips(genec[species_elt.value.toLowerCase()]); - } -} - -function select_row_radio(row) { - radio = row.getElementsByTagName( - "td")[0].getElementsByTagName( - "input")[0]; - if(radio === undefined) { - return false; - } - radio.setAttribute("checked", "checked"); - return true; -} diff --git a/qc_app/static/js/upload_progress.js b/qc_app/static/js/upload_progress.js deleted file mode 100644 index 9638b36..0000000 --- a/qc_app/static/js/upload_progress.js +++ /dev/null @@ -1,97 +0,0 @@ -function make_processing_indicator(elt) { - var count = 0; - return function() { - var message = "Finalising upload and saving file: " - if(count > 5) { - count = 1; - } - for(i = 0; i < count; i++) { - message = message + "."; - } - elt.innerHTML = message - count = count + 1 - }; -} - -function make_progress_updater(file, indicator_elt) { - var progress_bar = indicator_elt.querySelector("#progress-bar"); - var progress_text = indicator_elt.querySelector("#progress-text"); - var extra_text = indicator_elt.querySelector("#progress-extra-text"); - return function(event) { - if(event.loaded <= file.size) { - var percent = Math.round((event.loaded / file.size) * 100); - progress_bar.value = percent - progress_text.innerHTML = "Uploading: " + percent + "%"; - extra_text.setAttribute("class", "hidden") - } - - if(event.loaded == event.total) { - progress_bar.value = 100; - progress_text.innerHTML = "Uploaded: 100%"; - extra_text.setAttribute("class", null); - intv = setInterval(make_processing_indicator(extra_text), 400); - setTimeout(function() {clearTimeout(intv);}, 20000); - } - }; -} - -function setup_cancel_upload(request, indicator_elt) { - document.getElementById("btn-cancel-upload").addEventListener( - "click", function(event) { - event.preventDefault(); - request.abort(); - }); -} - -function setup_request(file, progress_indicator_elt) { - var request = new XMLHttpRequest(); - var updater = make_progress_updater(file, progress_indicator_elt); - request.upload.addEventListener("progress", updater); - request.onload = function(event) { - document.location.assign(request.responseURL); - }; - setup_cancel_upload(request, progress_indicator_elt) - return request; -} - -function selected_filetype(radios) { - for(idx = 0; idx < radios.length; idx++) { - if(radios[idx].checked) { - return radios[idx].value; - } - } -} - -function make_data_uploader(setup_formdata) { - return function(event) { - event.preventDefault(); - - var pindicator = document.getElementById("upload-progress-indicator"); - - var form = event.target; - var the_file = form.querySelector("input[type='file']").files[0]; - if(the_file === undefined) { - form.querySelector("input[type='file']").parentElement.setAttribute( - "class", "invalid-input"); - error_elt = form.querySelector("#no-file-error"); - if(error_elt !== undefined) { - error_elt.setAttribute("style", "display: block;"); - } - return false; - } - var formdata = setup_formdata(form); - - document.getElementById("progress-filename").innerHTML = the_file.name; - var request = setup_request(the_file, pindicator); - request.open(form.getAttribute("method"), form.getAttribute("action")); - request.send(formdata); - return false; - } -} - - -function setup_upload_handlers(formid, datauploader) { - console.info("Setting up the upload handlers.") - upload_form = document.getElementById(formid); - upload_form.addEventListener("submit", datauploader); -} diff --git a/qc_app/static/js/upload_samples.js b/qc_app/static/js/upload_samples.js deleted file mode 100644 index aed536f..0000000 --- a/qc_app/static/js/upload_samples.js +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Read the file content and set the `data-preview-content` attribute on the - * file element - */ -function read_first_n_lines(event, - fileelement, - numlines, - firstlineheading = true) { - var thefile = fileelement.files[0]; - var reader = new FileReader(); - reader.addEventListener("load", (event) => { - var filecontent = event.target.result.split( - "\n").slice( - 0, (numlines + (firstlineheading ? 1 : 0))).map( - (line) => {return line.trim("\r");}); - fileelement.setAttribute( - "data-preview-content", JSON.stringify(filecontent)); - display_preview(event); - }) - reader.readAsText(thefile); -} - -function remove_rows(preview_table) { - var table_body = preview_table.getElementsByTagName("tbody")[0]; - while(table_body.children.length > 0) { - table_body.removeChild(table_body.children.item(0)); - } -} - -/* - * Display error row - */ -function display_error_row(preview_table, error_message) { - remove_rows(preview_table); - row = document.createElement("tr"); - cell = document.createElement("td"); - cell.setAttribute("colspan", 4); - cell.innerHTML = error_message; - row.appendChild(cell); - preview_table.getElementsByTagName("tbody")[0].appendChild(row); -} - -function strip(str, chars) { - var end = str.length; - var start = 0 - for(var j = str.length; j > 0; j--) { - if(!chars.includes(str[j - 1])) { - break; - } - end = end - 1; - } - for(var i = 0; i < end; i++) { - if(!chars.includes(str[i])) { - break; - } - start = start + 1; - } - return str.slice(start, end); -} - -function process_preview_data(preview_data, separator, delimiter) { - return preview_data.map((line) => { - return line.split(separator).map((field) => { - return strip(field, delimiter); - }); - }); -} - -function render_preview(preview_table, preview_data) { - remove_rows(preview_table); - var table_body = preview_table.getElementsByTagName("tbody")[0]; - preview_data.forEach((line) => { - var row = document.createElement("tr"); - line.forEach((field) => { - var cell = document.createElement("td"); - cell.innerHTML = field; - row.appendChild(cell); - }); - table_body.appendChild(row); - }); -} - -/* - * Display a preview of the data, relying on the user's selection. - */ -function display_preview(event) { - var data_preview_table = document.getElementById("tbl:samples-preview"); - remove_rows(data_preview_table); - - var separator = document.getElementById("select:separator").value; - if(separator === "other") { - separator = document.getElementById("txt:separator").value; - } - if(separator == "") { - display_error_row(data_preview_table, "Please provide a separator."); - return false; - } - - var delimiter = document.getElementById("txt:delimiter").value; - - var firstlineheading = document.getElementById("chk:heading").checked; - - var fileelement = document.getElementById("file:samples"); - var preview_data = JSON.parse( - fileelement.getAttribute("data-preview-content") || "[]"); - if(preview_data.length == 0) { - display_error_row( - data_preview_table, - "No file data to preview. Check that file is provided."); - } - - render_preview(data_preview_table, process_preview_data( - preview_data.slice(0 + (firstlineheading ? 1 : 0)), - separator, - delimiter)); -} - -document.getElementById("chk:heading").addEventListener( - "change", display_preview); -document.getElementById("select:separator").addEventListener( - "change", display_preview); -document.getElementById("txt:separator").addEventListener( - "keyup", display_preview); -document.getElementById("txt:delimiter").addEventListener( - "keyup", display_preview); -document.getElementById("file:samples").addEventListener( - "change", (event) => { - read_first_n_lines(event, - document.getElementById("file:samples"), - 30, - document.getElementById("chk:heading").checked); - }); diff --git a/qc_app/static/js/utils.js b/qc_app/static/js/utils.js deleted file mode 100644 index 045dd47..0000000 --- a/qc_app/static/js/utils.js +++ /dev/null @@ -1,10 +0,0 @@ -function remove_children(element) { - Array.from(element.children).forEach(child => { - element.removeChild(child); - }); -} - -function trigger_change_event(element) { - evt = new Event("change"); - element.dispatchEvent(evt); -} diff --git a/qc_app/templates/base.html b/qc_app/templates/base.html deleted file mode 100644 index eb5e6b7..0000000 --- a/qc_app/templates/base.html +++ /dev/null @@ -1,51 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> - <head> - <meta charset="UTF-8" /> - <meta application-name="GeneNetwork Quality-Control Application" /> - <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - {%block extrameta%}{%endblock%} - - <title>GN Uploader: {%block title%}{%endblock%}</title> - - <link rel="stylesheet" type="text/css" - href="{{url_for('base.bootstrap', - filename='css/bootstrap.min.css')}}" /> - <link rel="stylesheet" type="text/css" - href="{{url_for('base.bootstrap', - filename='css/bootstrap-theme.min.css')}}" /> - - - <link rel="shortcut icon" type="image/png" sizes="64x64" - href="{{url_for('static', filename='images/CITGLogo.png')}}" /> - - <link rel="stylesheet" type="text/css" href="/static/css/custom-bootstrap.css" /> - <link rel="stylesheet" type="text/css" href="/static/css/styles.css" /> - - {%block css%}{%endblock%} - </head> - - <body> - <div class="navbar navbar-inverse navbar-static-top pull-left" - role="navigation" - style="width: 100%;min-width: 850px;white-space: nowrap;"> - <div class="container-fluid" style="width: 100%"> - <ul class="nav navbar-nav"> - <li><a href="/" style="font-weight: bold">GN Uploader</a></li> - <li> - <a href="{{gnuri or 'https://genenetwork.org'}}">GeneNetwork</a> - </li> - </ul> - </div> - </div> - <div class="container"> - {%block contents%}{%endblock%} - </div> - - <script src="{{url_for('base.jquery', - filename='jquery.min.js')}}"></script> - <script src="{{url_for('base.bootstrap', - filename='js/bootstrap.min.js')}}"></script> - {%block javascript%}{%endblock%} - </body> -</html> diff --git a/qc_app/templates/cli-output.html b/qc_app/templates/cli-output.html deleted file mode 100644 index 33fb73b..0000000 --- a/qc_app/templates/cli-output.html +++ /dev/null @@ -1,8 +0,0 @@ -{%macro cli_output(job, stream)%} - -<h4>{{stream | upper}} Output</h4> -<div class="cli-output"> - <pre>{{job.get(stream, "")}}</pre> -</div> - -{%endmacro%} diff --git a/qc_app/templates/continue_from_create_dataset.html b/qc_app/templates/continue_from_create_dataset.html deleted file mode 100644 index 03bb49c..0000000 --- a/qc_app/templates/continue_from_create_dataset.html +++ /dev/null @@ -1,52 +0,0 @@ -{%extends "base.html"%} -{%from "dbupdate_hidden_fields.html" import hidden_fields%} - -{%block title%}Create Study{%endblock%} - -{%block css%} -<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> -{%endblock%} - -{%block contents%} -<h2 class="heading">{{filename}}: create study</h2> - -{%with messages = get_flashed_messages(with_categories=true)%} -{%if messages:%} -<ul> - {%for category, message in messages:%} - <li class="{{category}}">{{message}}</li> - {%endfor%} -</ul> -{%endif%} -{%endwith%} - -<div class="row"> - <form method="POST" action="{{url_for('dbinsert.final_confirmation')}}" - id="select-platform-form" data-genechips="{{genechips_data}}" - class="two-col-sep-col1"> - <legend>continue with new dataset</legend> - {{hidden_fields( - filename, filetype, species=species, genechipid=genechipid, - studyid=studyid, datasetid=datasetid, totallines=totallines)}} - - <button type="submit" class="btn btn-primary">continue</button> - </form> -</div> - -<div class="row"> - <p class="two-col-sep-separator">OR</p> -</div> - -<div class="row"> - <form method="POST" action="{{url_for('dbinsert.select_dataset')}}" - id="select-platform-form" data-genechips="{{genechips_data}}" - class="two-col-sep-col2"> - <legend>Select from existing dataset</legend> - {{hidden_fields( - filename, filetype, species=species, genechipid=genechipid, - studyid=studyid, datasetid=datasetid, totallines=totallines)}} - - <button type="submit" class="btn btn-primary">go back</button> - </form> -</div> -{%endblock%} diff --git a/qc_app/templates/continue_from_create_study.html b/qc_app/templates/continue_from_create_study.html deleted file mode 100644 index 34e6e5e..0000000 --- a/qc_app/templates/continue_from_create_study.html +++ /dev/null @@ -1,52 +0,0 @@ -{%extends "base.html"%} -{%from "dbupdate_hidden_fields.html" import hidden_fields%} - -{%block title%}Create Study{%endblock%} - -{%block css%} -<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> -{%endblock%} - -{%block contents%} -<h2 class="heading">{{filename}}: create study</h2> - -{%with messages = get_flashed_messages(with_categories=true)%} -{%if messages:%} -<ul> - {%for category, message in messages:%} - <li class="{{category}}">{{message}}</li> - {%endfor%} -</ul> -{%endif%} -{%endwith%} - -<div class="row"> - <form method="POST" action="{{url_for('dbinsert.select_dataset')}}" - id="select-platform-form" data-genechips="{{genechips_data}}" - class="two-col-sep-col1"> - <legend>continue with new study</legend> - {{hidden_fields( - filename, filetype, species=species, genechipid=genechipid, - studyid=studyid, totallines=totallines)}} - - <button type="submit" class="btn btn-primary">continue</button> - </form> -</div> - -<div class="row"> - <p class="two-col-sep-separator">OR</p> -</div> - -<div class="row"> - <form method="POST" action="{{url_for('dbinsert.select_study')}}" - id="select-platform-form" data-genechips="{{genechips_data}}" - class="two-col-sep-col2"> - <legend>Select from existing study</legend> - {{hidden_fields( - filename, filetype, species=species, genechipid=genechipid, - studyid=studyid, totallines=totallines)}} - - <button type="submit" class="btn btn-primary">go back</button> - </form> -</div> -{%endblock%} diff --git a/qc_app/templates/data_review.html b/qc_app/templates/data_review.html deleted file mode 100644 index b7528fd..0000000 --- a/qc_app/templates/data_review.html +++ /dev/null @@ -1,85 +0,0 @@ -{%extends "base.html"%} - -{%block title%}Data Review{%endblock%} - -{%block contents%} -<h1 class="heading">data review</h1> - -<div class="row"> - <h2 id="data-concerns">Data Concerns</h2> - <p>The following are some of the requirements that the data in your file - <strong>MUST</strong> fulfil before it is considered valid for this system: - </p> - - <ol> - <li>File headings - <ul> - <li>The first row in the file should contains the headings. The number of - headings in this first row determines the number of columns expected for - all other lines in the file.</li> - <li>Each heading value in the first row MUST appear in the first row - <strong>ONE AND ONLY ONE</strong> time</li> - <li>The sample/cases (previously 'strains') headers in your first row will be - against those in the <a href="https://genenetwork.org" - title="Link to the GeneNetwork service"> - GeneNetwork</a> database.<br /> - <small class="text-muted"> - If you encounter an error saying your sample(s)/case(s) do not exist - in the GeneNetwork database, then you will have to use the - <a href="{{url_for('samples.select_species')}}" - title="Upload samples/cases feature">Upload Samples/Cases</a> - option on this system to upload them. - </small> - </ul> - </li> - - <li>Data - <ol> - <li><strong>NONE</strong> of the data cells/fields is allowed to be empty. - All fields/cells <strong>MUST</strong> contain a value.</li> - <li>The first column of the data rows will be considered a textual field, - holding the "identifier" for that row<li> - <li>Except for the first column/field for each data row, - <strong>NONE</strong> of the data columns/cells/fields should contain - spurious characters like `eeeee`, `5.555iloveguix`, etc...<br /> - All of them should be decimal values</li> - <li>decimal numbers must conform to the following criteria: - <ul> - <li>when checking an average file decimal numbers must have exactly three - decimal places to the right of the decimal point.</li> - <li>when checking a standard error file decimal numbers must have six or - greater decimal places to the right of the decimal point.</li> - <li>there must be a number to the left side of the decimal place - (e.g. 0.55555 is allowed but .55555 is not).</li> - </ul> - </li> - </ol> - </li> - </ol> -</div> - - -<div class="row"> - <h2 id="file-types">Supported File Types</h2> - We support the following file types: - - <ul> - <li>Tab-Separated value files (.tsv) - <ul> - <li>The <strong>TAB</strong> character is used to separate the fields of each - column</li> - <li>The values of each field <strong>ARE NOT</strong> quoted.</li> - <li>Here is an - <a href="https://gitlab.com/fredmanglis/gnqc_py/-/blob/main/tests/test_data/no_data_errors.tsv"> - example file</a> with a single data row.</li> - </ul> - </li> - <li>.txt files: Content has the same format as .tsv file above</li> - <li>.zip files: each zip file should contain - <strong>ONE AND ONLY ONE</strong> file of the .tsv or .txt type above. - <br />Any zip file with more than one file is invalid, and so is an empty - zip file.</li> - </ul> - -</div> -{%endblock%} diff --git a/qc_app/templates/dbupdate_error.html b/qc_app/templates/dbupdate_error.html deleted file mode 100644 index e1359d2..0000000 --- a/qc_app/templates/dbupdate_error.html +++ /dev/null @@ -1,12 +0,0 @@ -{%extends "base.html"%} - -{%block title%}DB Update Error{%endblock%} - -{%block contents%} -<h1 class="heading">database update error</h2> - -<p class="alert-danger"> - <strong>Database Update Error</strong>: {{error_message}} -</p> - -{%endblock%} diff --git a/qc_app/templates/dbupdate_hidden_fields.html b/qc_app/templates/dbupdate_hidden_fields.html deleted file mode 100644 index ccbc299..0000000 --- a/qc_app/templates/dbupdate_hidden_fields.html +++ /dev/null @@ -1,29 +0,0 @@ -{%macro hidden_fields(filename, filetype):%} - -<!-- {{kwargs}}: mostly for accessing the kwargs in macro --> - -<input type="hidden" name="filename" value="{{filename}}" /> -<input type="hidden" name="filetype" value="{{filetype}}" /> -{%if kwargs.get("totallines")%} -<input type="hidden" name="totallines" value="{{kwargs['totallines']}}" /> -{%endif%} -{%if kwargs.get("species"):%} -<input type="hidden" name="species" value="{{kwargs['species']}}" /> -{%endif%} -{%if kwargs.get("genechipid"):%} -<input type="hidden" name="genechipid" value="{{kwargs['genechipid']}}" /> -{%endif%} -{%if kwargs.get("inbredsetid"):%} -<input type="hidden" name="inbredsetid" value="{{kwargs['inbredsetid']}}" /> -{%endif%} -{%if kwargs.get("tissueid"):%} -<input type="hidden" name="tissueid" value="{{kwargs['tissueid']}}" /> -{%endif%} -{%if kwargs.get("studyid"):%} -<input type="hidden" name="studyid" value="{{kwargs['studyid']}}" /> -{%endif%} -{%if kwargs.get("datasetid"):%} -<input type="hidden" name="datasetid" value="{{kwargs['datasetid']}}" /> -{%endif%} - -{%endmacro%} diff --git a/qc_app/templates/errors_display.html b/qc_app/templates/errors_display.html deleted file mode 100644 index 715cfcf..0000000 --- a/qc_app/templates/errors_display.html +++ /dev/null @@ -1,47 +0,0 @@ -{%macro errors_display(errors, no_error_msg, error_message, complete)%} - -{%if errors | length == 0 %} -<span {%if complete%}class="alert-success"{%endif%}>{{no_error_msg}}</span> -{%else %} -<p class="alert-danger">{{error_message}}</p> - -<table class="table reports-table"> - <thead> - <tr> - <th>line number</th> - <th>column(s)</th> - <th>error</th> - <th>error message</th> - </tr> - </thead> - - <tbody> - {%for error in errors%} - <tr> - <td>{{error["line"]}}</td> - <td> - {%if isinvalidvalue(error):%} - {{error.column}} - {%elif isduplicateheading(error): %} - {{error.columns}} - {%else: %} - - - {%endif %} - </td> - <td> - {%if isinvalidvalue(error):%} - Invalid Value - {%elif isduplicateheading(error): %} - Duplicate Header - {%else%} - Inconsistent Columns - {%endif %} - </td> - <td>{{error["message"]}}</td> - </tr> - {%endfor%} - </tbody> -</table> -{%endif%} - -{%endmacro%} diff --git a/qc_app/templates/final_confirmation.html b/qc_app/templates/final_confirmation.html deleted file mode 100644 index 0727fc8..0000000 --- a/qc_app/templates/final_confirmation.html +++ /dev/null @@ -1,47 +0,0 @@ -{%extends "base.html"%} -{%from "dbupdate_hidden_fields.html" import hidden_fields%} - -{%block title%}Confirmation{%endblock%} - -{%macro display_item(item_name, item_data):%} -<li> - <strong>{{item_name}}</strong> - {%if item_data%} - <ul> - {%for term,value in item_data.items():%} - <li><strong>{{term}}:</strong> {{value}}</li> - {%endfor%} - </ul> - {%endif%} -</li> -{%endmacro%} - -{%block contents%} -<h2 class="heading">Final Confirmation</h2> - -<div class="two-col-sep-col1"> - <p><strong>Selected Data</strong></p> - <ul> - <li><strong>File</strong> - <ul> - <li><strong>Filename</strong>: {{filename}}</li> - <li><strong>File Type</strong>: {{filetype}}</li> - </ul> - </li> - {{display_item("Species", the_species)}} - {{display_item("Platform", platform)}} - {{display_item("Study", study)}} - {{display_item("Dataset", dataset)}} - </ul> -</div> - -<form method="POST" action="{{url_for('dbinsert.insert_data')}}"> - {{hidden_fields( - filename, filetype, species=species, genechipid=genechipid, - studyid=studyid,datasetid=datasetid, totallines=totallines)}} - <fieldset> - <input type="submit" class="btn btn-primary" value="confirm" /> - </fieldset> -</form> -</div> -{%endblock%} diff --git a/qc_app/templates/flash_messages.html b/qc_app/templates/flash_messages.html deleted file mode 100644 index b7af178..0000000 --- a/qc_app/templates/flash_messages.html +++ /dev/null @@ -1,25 +0,0 @@ -{%macro flash_all_messages()%} -{%with messages = get_flashed_messages(with_categories=true)%} -{%if messages:%} -<ul> - {%for category, message in messages:%} - <li class="{{category}}">{{message}}</li> - {%endfor%} -</ul> -{%endif%} -{%endwith%} -{%endmacro%} - -{%macro flash_messages(filter_class)%} -{%with messages = get_flashed_messages(with_categories=true)%} -{%if messages:%} -<ul> - {%for category, message in messages:%} - {%if filter_class in category%} - <li class="{{category}}">{{message}}</li> - {%endif%} - {%endfor%} -</ul> -{%endif%} -{%endwith%} -{%endmacro%} diff --git a/qc_app/templates/http-error.html b/qc_app/templates/http-error.html deleted file mode 100644 index 374fb86..0000000 --- a/qc_app/templates/http-error.html +++ /dev/null @@ -1,18 +0,0 @@ -{%extends "base.html"%} - -{%block title%}HTTP Error: {{exc.code}}{%endblock%} - -{%block contents%} -<h1>{{exc.code}}: {{exc.description}}</h1> - -<div class="row"> - <p> - You attempted to access {{request_url}} which failed with the following - error: - </p> -</div> - -<div class="row"> - <pre>{{"\n".join(trace)}}</pre> -</div> -{%endblock%} diff --git a/qc_app/templates/index.html b/qc_app/templates/index.html deleted file mode 100644 index 89d2ae9..0000000 --- a/qc_app/templates/index.html +++ /dev/null @@ -1,81 +0,0 @@ -{%extends "base.html"%} - -{%block title%}Data Upload{%endblock%} - -{%block contents%} -<div class="row"> - <h1 class="heading">data upload</h1> - - <div class="explainer"> - <p>Each of the sections below gives you a different option for data upload. - Please read the documentation for each section carefully to understand what - each section is about.</p> - </div> -</div> - -<div class="row"> - <h2 class="heading">R/qtl2 Bundles</h2> - - <div class="explainer"> - <p>This feature combines and extends the two upload methods below. Instead of - uploading one item at a time, the R/qtl2 bundle you upload can contain both - the genotypes data (samples/individuals/cases and their data) and the - expression data.</p> - <p>The R/qtl2 bundle, additionally, can contain extra metadata, that neither - of the methods below can handle.</p> - - <a href="{{url_for('upload.rqtl2.select_species')}}" - title="Upload a zip bundle of R/qtl2 files"> - <button class="btn btn-primary">upload R/qtl2 bundle</button></a> - </div> -</div> - - -<div class="row"> - <h2 class="heading">Expression Data</h2> - - <div class="explainer"> - <p>This feature enables you to upload expression data. It expects the data to - be in <strong>tab-separated values (TSV)</strong> files. The data should be - a simple matrix of <em>phenotype × sample</em>, i.e. The first column is a - list of the <em>phenotypes</em> and the first row is a list of - <em>samples/cases</em>.</p> - - <p>If you haven't done so please go to this page to learn the requirements for - file formats and helpful suggestions to enter your data in a fast and easy - way.</p> - - <ol> - <li><strong>PLEASE REVIEW YOUR DATA.</strong>Make sure your data complies - with our system requirements. ( - <a href="{{url_for('entry.data_review')}}#data-concerns" - title="Details for the data expectations.">Help</a> - )</li> - <li><strong>UPLOAD YOUR DATA FOR DATA VERIFICATION.</strong> We accept - <strong>.csv</strong>, <strong>.txt</strong> and <strong>.zip</strong> - files (<a href="{{url_for('entry.data_review')}}#file-types" - title="Details for the data expectations.">Help</a>)</li> - </ol> - </div> - - <a href="{{url_for('entry.upload_file')}}" - title="Upload your expression data" - class="btn btn-primary">upload expression data</a> -</div> - -<div class="row"> - <h2 class="heading">samples/cases</h2> - - <div class="explainer"> - <p>For the expression data above, you need the samples/cases in your file to - already exist in the GeneNetwork database. If there are any samples that do - not already exist the upload of the expression data will fail.</p> - <p>This section gives you the opportunity to upload any missing samples</p> - </div> - - <a href="{{url_for('samples.select_species')}}" - title="Upload samples/cases/individuals for your data" - class="btn btn-primary">upload Samples/Cases</a> -</div> - -{%endblock%} diff --git a/qc_app/templates/insert_error.html b/qc_app/templates/insert_error.html deleted file mode 100644 index 5301288..0000000 --- a/qc_app/templates/insert_error.html +++ /dev/null @@ -1,32 +0,0 @@ -{%extends "base.html"%} - -{%block title%}Data Insertion Failure{%endblock%} - -{%block contents%} -<h1 class="heading">Insertion Failure</h1> - -<div class="row"> - <p> - There was an error inserting data into the database - </p> - - <p> - Please notify the developers of this issue when you encounter it, - providing the information below. - </p> - - <h4>Debugging Information</h4> - - <ul> - <li><strong>job id</strong>: {{job["jobid"]}}</li> - </ul> -</div> - -<div class="row"> - <h4>STDERR Output</h4> - <pre class="cli-output"> - {{job["stderr"]}} - </pre> -</div> - -{%endblock%} diff --git a/qc_app/templates/insert_progress.html b/qc_app/templates/insert_progress.html deleted file mode 100644 index 52177d6..0000000 --- a/qc_app/templates/insert_progress.html +++ /dev/null @@ -1,46 +0,0 @@ -{%extends "base.html"%} -{%from "stdout_output.html" import stdout_output%} - -{%block extrameta%} -<meta http-equiv="refresh" content="5"> -{%endblock%} - -{%block title%}Job Status{%endblock%} - -{%block contents%} -<h1 class="heading">{{job_name}}</h1> - -<div class="row"> - <form> - <div class="form-group"> - <label for="job_status" class="form-label">status:</label> - <span class="form-text">{{job_status}}: {{message}}</span> - </div> - -{%if job.get("stdout", "").split("\n\n") | length < 3 %} -{%set lines = 0%} -{%else%} -{%set lines = (job.get("stdout", "").split("\n\n") | length / 3) %} -{%endif%} -{%set totallines = job.get("totallines", lines+3) | int %} -{%if totallines > 1000 %} -{%set fraction = ((lines*1000)/totallines) %} -{%else%} -{%set fraction = (lines/totallines)%} -{%endif%} - - <div class="form-group"> - <label for="job_{{job_id}}" class="form-label">inserting: </label> - <progress id="jobs_{{job_id}}" - value="{{(fraction)}}" - class="form-control">{{fraction*100}}</progress> - <span class="form-text text-muted"> - {{"%.2f" | format(fraction * 100 | float)}}%</span> - </div> - </form> -</div> - - -{{stdout_output(job)}} - -{%endblock%} diff --git a/qc_app/templates/insert_success.html b/qc_app/templates/insert_success.html deleted file mode 100644 index 7e1fa8d..0000000 --- a/qc_app/templates/insert_success.html +++ /dev/null @@ -1,19 +0,0 @@ -{%extends "base.html"%} -{%from "stdout_output.html" import stdout_output%} - -{%block title%}Insertion Success{%endblock%} - -{%block contents%} -<h1 class="heading">Insertion Success</h1> - -<div class="row"> -<p>Data inserted successfully!</p> - -<p>The following queries were run:</p> -</div> - -<div class="row"> - {{stdout_output(job)}} -</div> - -{%endblock%} diff --git a/qc_app/templates/job_progress.html b/qc_app/templates/job_progress.html deleted file mode 100644 index 1af0763..0000000 --- a/qc_app/templates/job_progress.html +++ /dev/null @@ -1,40 +0,0 @@ -{%extends "base.html"%} -{%from "errors_display.html" import errors_display%} - -{%block extrameta%} -<meta http-equiv="refresh" content="5"> -{%endblock%} - -{%block title%}Job Status{%endblock%} - -{%block contents%} -<h1 class="heading">{{job_name}}</h2> - -<div class="row"> - <form action="{{url_for('parse.abort')}}" method="POST"> - <legend class="heading">Status</legend> - <div class="form-group"> - <label for="job_status" class="form-label">status:</label> - <span class="form-text">{{job_status}}: {{message}}</span><br /> - </div> - - <div class="form-group"> - <label for="job_{{job_id}}" class="form-label">parsing: </label> - <progress id="job_{{job_id}}" - value="{{progress/100}}" - class="form-control"> - {{progress}}</progress> - <span class="form-text text-muted">{{"%.2f" | format(progress)}}%</span> - </div> - - <input type="hidden" name="job_id" value="{{job_id}}" /> - - <button type="submit" class="btn btn-danger">Abort</button> - </form> -</div> - -<div class="row"> - {{errors_display(errors, "No errors found so far", "We have found the following errors so far", False)}} -</div> - -{%endblock%} diff --git a/qc_app/templates/no_such_job.html b/qc_app/templates/no_such_job.html deleted file mode 100644 index 42a2d48..0000000 --- a/qc_app/templates/no_such_job.html +++ /dev/null @@ -1,14 +0,0 @@ -{%extends "base.html"%} - -{%block extrameta%} -<meta http-equiv="refresh" content="5;url={{url_for('entry.upload_file')}}"> -{%endblock%} - -{%block title%}No Such Job{%endblock%} - -{%block contents%} -<h1 class="heading">No Such Job: {{job_id}}</h2> - -<p>No job, with the id '<em>{{job_id}}</em>' was found!</p> - -{%endblock%} diff --git a/qc_app/templates/parse_failure.html b/qc_app/templates/parse_failure.html deleted file mode 100644 index 31f6be8..0000000 --- a/qc_app/templates/parse_failure.html +++ /dev/null @@ -1,26 +0,0 @@ -{%extends "base.html"%} - -{%block title%}Worker Failure{%endblock%} - -{%block contents%} -<h1 class="heading">Worker Failure</h1> - -<p> - There was an error while parsing your file. -</p> - -<p> - Please notify the developers of this issue when you encounter it, - providing the information below. -</p> - -<h4>Debugging Information</h4> - -<ul> - <li><strong>job id</strong>: {{job["job_id"]}}</li> - <li><strong>filename</strong>: {{job["filename"]}}</li> - <li><strong>line number</strong>: {{job["line_number"]}}</li> - <li><strong>Progress</strong>: {{job["percent"]}} %</li> -</ul> - -{%endblock%} diff --git a/qc_app/templates/parse_results.html b/qc_app/templates/parse_results.html deleted file mode 100644 index e2bf7f0..0000000 --- a/qc_app/templates/parse_results.html +++ /dev/null @@ -1,30 +0,0 @@ -{%extends "base.html"%} -{%from "errors_display.html" import errors_display%} - -{%block title%}Parse Results{%endblock%} - -{%block contents%} -<h1 class="heading">{{job_name}}: parse results</h2> - -{%if user_aborted%} -<span class="alert-warning">Job aborted by the user</span> -{%endif%} - -{{errors_display(errors, "No errors found in the file", "We found the following errors", True)}} - -{%if errors | length == 0 and not user_aborted %} -<form method="post" action="{{url_for('dbinsert.select_platform')}}"> - <input type="hidden" name="job_id" value="{{job_id}}" /> - <input type="submit" value="update database" class="btn btn-primary" /> -</form> -{%endif%} - -{%if errors | length > 0 or user_aborted %} -<br /> -<a href="{{url_for('entry.upload_file')}}" title="Back to index page." - class="btn btn-primary"> - Go back -</a> -{%endif%} - -{%endblock%} diff --git a/qc_app/templates/rqtl2/create-geno-dataset-success.html b/qc_app/templates/rqtl2/create-geno-dataset-success.html deleted file mode 100644 index 1b50221..0000000 --- a/qc_app/templates/rqtl2/create-geno-dataset-success.html +++ /dev/null @@ -1,55 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Upload R/qtl2 Bundle{%endblock%} - -{%block contents%} -<h2 class="heading">Select Genotypes Dataset</h2> - -<div class="explainer"> - <p>You successfully created the genotype dataset with the following - information. - <dl> - <dt>ID</dt> - <dd>{{geno_dataset.id}}</dd> - - <dt>Name</dt> - <dd>{{geno_dataset.name}}</dd> - - <dt>Full Name</dt> - <dd>{{geno_dataset.fname}}</dd> - - <dt>Short Name</dt> - <dd>{{geno_dataset.sname}}</dd> - - <dt>Created On</dt> - <dd>{{geno_dataset.today}}</dd> - - <dt>Public?</dt> - <dd>{%if geno_dataset.public == 0%}No{%else%}Yes{%endif%}</dd> - </dl> - </p> -</div> - -<div class="row"> - <form id="frm-upload-rqtl2-bundle" - action="{{url_for('upload.rqtl2.select_dataset_info', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - method="POST" - enctype="multipart/form-data"> - <legend class="heading">select from existing genotype datasets</legend> - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" - value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" - value="{{geno_dataset.id}}" /> - - <button type="submit" class="btn btn-primary">continue</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/create-probe-dataset-success.html b/qc_app/templates/rqtl2/create-probe-dataset-success.html deleted file mode 100644 index 790d174..0000000 --- a/qc_app/templates/rqtl2/create-probe-dataset-success.html +++ /dev/null @@ -1,59 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Upload R/qtl2 Bundle{%endblock%} - -{%block contents%} -<h2 class="heading">Create ProbeSet Dataset</h2> - -<div class="row"> - <p>You successfully created the ProbeSet dataset with the following - information. - <dl> - <dt>Averaging Method</dt> - <dd>{{avgmethod.Name}}</dd> - - <dt>ID</dt> - <dd>{{dataset.datasetid}}</dd> - - <dt>Name</dt> - <dd>{{dataset.name2}}</dd> - - <dt>Full Name</dt> - <dd>{{dataset.fname}}</dd> - - <dt>Short Name</dt> - <dd>{{dataset.sname}}</dd> - - <dt>Created On</dt> - <dd>{{dataset.today}}</dd> - - <dt>DataScale</dt> - <dd>{{dataset.datascale}}</dd> - </dl> - </p> -</div> - -<div class="row"> - <form id="frm-upload-rqtl2-bundle" - action="{{url_for('upload.rqtl2.select_dataset_info', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - method="POST" - enctype="multipart/form-data"> - <legend class="heading">Create ProbeSet dataset</legend> - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> - <input type="hidden" name="probe-study-id" value="{{study.Id}}" /> - <input type="hidden" name="probe-dataset-id" value="{{dataset.datasetid}}" /> - - <button type="submit" class="btn btn-primary">continue</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/create-probe-study-success.html b/qc_app/templates/rqtl2/create-probe-study-success.html deleted file mode 100644 index d0ee508..0000000 --- a/qc_app/templates/rqtl2/create-probe-study-success.html +++ /dev/null @@ -1,49 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Upload R/qtl2 Bundle{%endblock%} - -{%block contents%} -<h2 class="heading">Create ProbeSet Study</h2> - -<div class="row"> - <p>You successfully created the ProbeSet study with the following - information. - <dl> - <dt>ID</dt> - <dd>{{study.id}}</dd> - - <dt>Name</dt> - <dd>{{study.name}}</dd> - - <dt>Full Name</dt> - <dd>{{study.fname}}</dd> - - <dt>Short Name</dt> - <dd>{{study.sname}}</dd> - - <dt>Created On</dt> - <dd>{{study.today}}</dd> - </dl> - </p> - - <form id="frm-upload-rqtl2-bundle" - action="{{url_for('upload.rqtl2.select_dataset_info', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - method="POST" - enctype="multipart/form-data"> - <legend class="heading">Create ProbeSet study</legend> - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - <input type="hidden" name="probe-study-id" value="{{study.studyid}}" /> - - <button type="submit" class="btn btn-primary">continue</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/create-tissue-success.html b/qc_app/templates/rqtl2/create-tissue-success.html deleted file mode 100644 index 5f2c5a0..0000000 --- a/qc_app/templates/rqtl2/create-tissue-success.html +++ /dev/null @@ -1,106 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_all_messages%} - -{%block title%}Upload R/qtl2 Bundle{%endblock%} - -{%block contents%} -<h2 class="heading">Select Tissue</h2> - -<div class="row"> - <p>You have successfully added a new tissue, organ or biological material with - the following details:</p> -</div> - -<div class="row"> - {{flash_all_messages()}} - - <form id="frm-create-tissue-display" - method="POST" - action="#"> - <legend class="heading">Create Tissue</legend> - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> - - <div class="form-group"> - <label>Name</label> - <label>{{tissue.TissueName}}</label> - </div> - - <div class="form-group"> - <label>Short Name</label> - <label>{{tissue.Short_Name}}</label> - </div> - - {%if tissue.BIRN_lex_ID%} - <div class="form-group"> - <label>BIRN Lex ID</label> - <label>{{tissue.BIRN_lex_ID}}</label> - </div> - {%endif%} - - {%if tissue.BIRN_lex_Name%} - <div class="form-group"> - <label>BIRN Lex Name</label> - <label>{{tissue.BIRN_lex_Name}}</label> - </div> - {%endif%} - </form> - - <div id="action-buttons" - style="width:65ch;display:inline-grid;column-gap:5px;"> - - <form id="frm-create-tissue-success-continue" - method="POST" - action="{{url_for('upload.rqtl2.select_dataset_info', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - style="display: inline; width: 100%; grid-column: 1 / 2; - padding-top: 0.5em; text-align: center; border: none; - background-color: inherit;"> - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> - - <button type="submit" class="btn btn-primary">continue</button> - </form> - </div> -</div> - -<div class="row"> - <p style="display:inline;width:100%;grid-column:2/3;text-align:center; - color:#336699;font-weight:bold;"> - OR - </p> -</div> - -<div class="row"> - <form id="frm-create-tissue-success-select-existing" - method="POST" - action="{{url_for('upload.rqtl2.select_tissue', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - style="display: inline; width: 100%; grid-column: 3 / 4; - padding-top: 0.5em; text-align: center; border: none; - background-color: inherit;"> - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - - <button type="submit" class="btn btn-primary"> - select from existing tissues</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/index.html b/qc_app/templates/rqtl2/index.html deleted file mode 100644 index f3329c2..0000000 --- a/qc_app/templates/rqtl2/index.html +++ /dev/null @@ -1,36 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Data Upload{%endblock%} - -{%block contents%} -<h1 class="heading">R/qtl2 data upload</h1> - -<h2>R/qtl2 Upload</h2> - -<form method="POST" action="{{url_for('upload.rqtl2.select_species')}}" - id="frm-rqtl2-upload"> - <legend class="heading">upload R/qtl2 bundle</legend> - {{flash_messages("error-rqtl2")}} - - <div class="form-group"> - <label for="select:species" class="form-label">Species</label> - <select id="select:species" - name="species_id" - required="required" - class="form-control"> - <option value="">Select species</option> - {%for spec in species%} - <option value="{{spec.SpeciesId}}">{{spec.MenuName}}</option> - {%endfor%} - </select> - <small class="form-text text-muted"> - Data that you upload to the system should belong to a know species. - Here you can select the species that you wish to upload data for. - </small> - </div> - - <button type="submit" class="btn btn-primary" />submit</button> -</form> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/no-such-job.html b/qc_app/templates/rqtl2/no-such-job.html deleted file mode 100644 index b17004f..0000000 --- a/qc_app/templates/rqtl2/no-such-job.html +++ /dev/null @@ -1,13 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Job Status{%endblock%} - -{%block contents%} -<h1 class="heading">R/qtl2 job status</h1> - -<h2>R/qtl2 Upload: No Such Job</h2> - -<p class="alert-danger">No job with ID {{jobid}} was found.</p> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/rqtl2-job-error.html b/qc_app/templates/rqtl2/rqtl2-job-error.html deleted file mode 100644 index 72a334b..0000000 --- a/qc_app/templates/rqtl2/rqtl2-job-error.html +++ /dev/null @@ -1,33 +0,0 @@ -{%extends "base.html"%} -{%from "cli-output.html" import cli_output%} - -{%block title%}Job Status{%endblock%} - -{%block contents%} -<h1 class="heading">R/qtl2 job status</h1> - -<h2>R/qtl2 Upload: Job Status</h2> - -<div class="explainer"> - <p>The processing of the R/qtl2 bundle you uploaded has failed. We have - provided some information below to help you figure out what the problem - could be.</p> - <p>If you find that you cannot figure out what the problem is on your own, - please contact the team running the system for assistance, providing the - R/qtl2 bundle you uploaded, and a screenshot of this page.</p> -</div> - -<h4>stdout</h4> -{{cli_output(job, "stdout")}} - -<h4>stderr</h4> -{{cli_output(job, "stderr")}} - -<h4>Log</h4> -<div class="cli-output"> - {%for msg in messages%} - {{msg}}<br /> - {%endfor%} -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/rqtl2-job-results.html b/qc_app/templates/rqtl2/rqtl2-job-results.html deleted file mode 100644 index 4ecd415..0000000 --- a/qc_app/templates/rqtl2/rqtl2-job-results.html +++ /dev/null @@ -1,24 +0,0 @@ -{%extends "base.html"%} -{%from "cli-output.html" import cli_output%} - -{%block title%}Job Status{%endblock%} - -{%block contents%} -<h1 class="heading">R/qtl2 job status</h1> - -<h2>R/qtl2 Upload: Job Status</h2> - -<div class="explainer"> - <p>The processing of the R/qtl2 bundle you uploaded has completed - successfully.</p> - <p>You should now be able to use GeneNetwork to run analyses on your data.</p> -</div> - -<h4>Log</h4> -<div class="cli-output"> - {%for msg in messages%} - {{msg}}<br /> - {%endfor%} -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/rqtl2-job-status.html b/qc_app/templates/rqtl2/rqtl2-job-status.html deleted file mode 100644 index e896f88..0000000 --- a/qc_app/templates/rqtl2/rqtl2-job-status.html +++ /dev/null @@ -1,20 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Job Status{%endblock%} - -{%block extrameta%} -<meta http-equiv="refresh" content="3"> -{%endblock%} - -{%block contents%} -<h1 class="heading">R/qtl2 job status</h1> - -<h2>R/qtl2 Upload: Job Status</h2> - -<h4>Log</h4> -<div class="cli-output"> - <pre>{{"\n".join(messages)}}</pre> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html deleted file mode 100644 index 90e8887..0000000 --- a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html +++ /dev/null @@ -1,120 +0,0 @@ -{%extends "base.html"%} -{%from "cli-output.html" import cli_output%} - -{%block title%}R/qtl2 bundle: QC Job Error{%endblock%} - -{%macro errors_table(tableid, errors)%} -<table id="{{tableid}}" class="table error-table"> - <caption>{{caption}}</caption> - <thead> - <tr> - <th>Line</th> - <th>Column</th> - <th>Value</th> - <th>Message</th> - </tr> - </thead> - <tbody> - {%for error in errors%} - <tr> - <td>{{error.line}}</td> - <td>{{error.column}}</td> - <td>{{error.value}}</td> - <td>{{error.message}}</td> - </tr> - {%else%} - <tr> - <td colspan="4">No errors to display here.</td> - </tr> - {%endfor%} - </tbody> -</table> -{%endmacro%} - -{%block contents%} -<h1 class="heading">R/qtl2 bundle: QC job Error</h1> - -<div class="explainer"> - <p>The R/qtl2 bundle has failed some <emph>Quality Control</emph> checks.</p> - <p>We list below some of the errors that need to be fixed before the data can - be uploaded onto GeneNetwork.</p> -</div> - -{%if errorsgeneric | length > 0%} -<h2 class="heading">Generic Errors ({{errorsgeneric | length}})</h3> -<div class="explainer"> - We found the following generic errors in your R/qtl2 bundle: -</div> - -<h3>Missing Files</h3> -<div class="explainer"> - <p>These files are listed in the bundle's control file, but do not actually - exist in the bundle</p> -</div> -<table id="tbl-errors-missing-files" class="table error-table"> - <thead> - <tr> - <th>Control File Key</th> - <th>Bundle File Name</th> - <th>Message</th> - </tr> - </thead> - <tbody> - {%for error in (errorsgeneric | selectattr("type", "equalto", "MissingFile"))%} - <tr> - <td>{{error.controlfilekey}}</td> - <td>{{error.filename}}</td> - <td>{{error.message}}</td> - </tr> - {%endfor%} - </tbody> -</table> - -<h3>Other Generic Errors</h3> -{{errors_table("tbl-errors-generic", errorsgeneric| selectattr("type", "ne", "MissingFile"))}} -{%endif%} - -{%if errorsgeno | length > 0%} -<h2 class="heading">Geno Errors ({{errorsgeno | length}})</h3> -<div class="explainer"> - We found the following errors in the 'geno' file in your R/qtl2 bundle: -</div> -{{errors_table("tbl-errors-geno", errorsgeno[0:50])}} -{%endif%} - -{%if errorspheno | length > 0%} -<h2 class="heading">Pheno Errors ({{errorspheno | length}})</h3> -<div class="explainer"> - We found the following errors in the 'pheno' file in your R/qtl2 bundle: -</div> -{{errors_table("tbl-errors-pheno", errorspheno[0:50])}} -{%endif%} - -{%if errorsphenose | length > 0%} -<h2 class="heading">Phenose Errors ({{errorsphenose | length}})</h3> -<div class="explainer"> - We found the following errors in the 'phenose' file in your R/qtl2 bundle: -</div> -{{errors_table("tbl-errors-phenose", errorsphenose[0:50])}} -{%endif%} - -{%if errorsphenocovar | length > 0%} -<h2 class="heading">Phenocovar Errors ({{errorsphenocovar | length}})</h3> -<div class="explainer"> - We found the following errors in the 'phenocovar' file in your R/qtl2 bundle: -</div> -{{errorsphenocovar}} -{%endif%} - -<h4>stdout</h4> -{{cli_output(job, "stdout")}} - -<h4>stderr</h4> -{{cli_output(job, "stderr")}} - -<h4>Log</h4> -<div class="cli-output"> - <pre>{{"\n".join(messages)}}</pre> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-results.html b/qc_app/templates/rqtl2/rqtl2-qc-job-results.html deleted file mode 100644 index 59bc8cd..0000000 --- a/qc_app/templates/rqtl2/rqtl2-qc-job-results.html +++ /dev/null @@ -1,66 +0,0 @@ -{%extends "base.html"%} -{%from "cli-output.html" import cli_output%} - -{%block title%}R/qtl2 bundle: QC job results{%endblock%} - -{%block contents%} -<h1 class="heading">R/qtl2 bundle: QC job results</h1> - -<div class="row"> - <p>The R/qtl2 bundle you uploaded has passed all automated quality-control - checks successfully.</p> - <p>You may now continue to load the data into GeneNetwork for the bundle, with - the following details:</p> -</div> - -<div class="row"> - <form id="form-qc-job-results" - action="{{url_for('upload.rqtl2.select_dataset_info', - species_id=species.SpeciesId, - population_id=population.Id)}}" - method="POST"> - <div class="form-group"> - <legend>Species</legend> - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - - <span class="form-label">Name</span> - <span class="form-text">{{species.Name | capitalize}}</span> - - <span class="form-label">Scientific</span> - <span class="form-text">{{species.FullName | capitalize}}</span> - </div> - - <div class="form-group"> - <legend>population</legend> - <input type="hidden" name="population_id" value="{{population.Id}}" /> - - <span class="form-label">Name</span> - <span class="form-text">{{population.InbredSetName}}</span> - - <span class="form-label">Full Name</span> - <span class="form-text">{{population.FullName}}</span> - - <span class="form-label">Genetic Type</span> - <span class="form-text">{{population.GeneticType}}</span> - - <span class="form-label">Description</span> - <span class="form-text">{{population.Description or "-"}}</span> - </div> - - <div class="form-group"> - <legend>R/qtl2 Bundle File</legend> - <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2bundle}}" /> - <input type="hidden" name="original-filename" value="{{rqtl2bundleorig}}" /> - - <span class="form-label">Original Name</span> - <span class="form-text">{{rqtl2bundleorig}}</span> - - <span class="form-label">Internal Name</span> - <span class="form-text">{{rqtl2bundle[0:25]}}…</span> - </div> - - <button type="submit" class="btn btn-primary">continue</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-status.html b/qc_app/templates/rqtl2/rqtl2-qc-job-status.html deleted file mode 100644 index f4a6266..0000000 --- a/qc_app/templates/rqtl2/rqtl2-qc-job-status.html +++ /dev/null @@ -1,41 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Job Status{%endblock%} - -{%block extrameta%} -<meta http-equiv="refresh" content="3"> -{%endblock%} - -{%block contents%} -<h1 class="heading">R/qtl2 bundle: QC job status</h1> - -{%if geno_percent%} -<p> - <h2>Checking 'geno' file:</h2> - <progress id="prg-geno-checking" value="{{geno_percent}}" max="100"> - {{geno_percent}}%</progress> - {{geno_percent}}%</p> -{%endif%} - -{%if pheno_percent%} -<p> - <h2>Checking 'pheno' file:</h2> - <progress id="prg-pheno-checking" value="{{pheno_percent}}" max="100"> - {{pheno_percent}}%</progress> - {{pheno_percent}}%</p> -{%endif%} - -{%if phenose_percent%} -<p> - <h2>Checking 'phenose' file:</h2> - <progress id="prg-phenose-checking" value="{{phenose_percent}}" max="100"> - {{phenose_percent}}%</progress> - {{phenose_percent}}%</p> -{%endif%} - -<h4>Log</h4> -<div class="cli-output"> - <pre>{{"\n".join(messages)}}</pre> -</div> -{%endblock%} diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-success.html b/qc_app/templates/rqtl2/rqtl2-qc-job-success.html deleted file mode 100644 index 2861a04..0000000 --- a/qc_app/templates/rqtl2/rqtl2-qc-job-success.html +++ /dev/null @@ -1,37 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_all_messages%} - -{%block title%}R/qtl2 Bundle: Quality Control Successful{%endblock%} - -{%block contents%} -<h2 class="heading">R/qtl2 Bundle: Quality Control Successful</h2> - -<div class="row"> - <p>The R/qtl2 bundle you uploaded has passed <emph>all</emph> quality control - checks successfully, and is now ready for uploading into the database.</p> - <p>Click "Continue" below to proceed.</p> -</div> - -<!-- - The "action" on this form takes us to the next step, where we can - select all the other data necessary to enter the data into the database. - --> -<div class="row"> - <form id="frm-upload-rqtl2-bundle" - action="{{url_for('upload.rqtl2.select_dataset_info', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - method="POST" - enctype="multipart/form-data"> - {{flash_all_messages()}} - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" - value="{{rqtl2_bundle_file}}" /> - - <button type="submit" class="btn btn-primary">continue</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/select-geno-dataset.html b/qc_app/templates/rqtl2/select-geno-dataset.html deleted file mode 100644 index 873f9c3..0000000 --- a/qc_app/templates/rqtl2/select-geno-dataset.html +++ /dev/null @@ -1,144 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Upload R/qtl2 Bundle{%endblock%} - -{%block contents%} -<h2 class="heading">Select Genotypes Dataset</h2> - -<div class="row"> - <p>Your R/qtl2 files bundle contains a "geno" specification. You will - therefore need to select from one of the existing Genotype datasets or - create a new one.</p> - <p>This is the dataset where your data will be organised under.</p> -</div> - -<div class="row"> - <form id="frm-upload-rqtl2-bundle" - action="{{url_for('upload.rqtl2.select_geno_dataset', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - method="POST" - enctype="multipart/form-data"> - <legend class="heading">select from existing genotype datasets</legend> - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" - value="{{rqtl2_bundle_file}}" /> - - {{flash_messages("error-rqtl2-select-geno-dataset")}} - - <div class="form-group"> - <legend>Datasets</legend> - <label for="select:geno-datasets" class="form-label">Dataset</label> - <select id="select:geno-datasets" - name="geno-dataset-id" - required="required" - {%if datasets | length == 0%} - disabled="disabled" - {%endif%} - class="form-control" - aria-describedby="help-geno-dataset-select-dataset"> - <option value="">Select dataset</option> - {%for dset in datasets%} - <option value="{{dset['Id']}}">{{dset["Name"]}} ({{dset["FullName"]}})</option> - {%endfor%} - </select> - <span id="help-geno-dataset-select-dataset" class="form-text text-muted"> - Select from the existing genotype datasets for species - {{species.SpeciesName}} ({{species.FullName}}). - </span> - </div> - - <button type="submit" class="btn btn-primary">select dataset</button> - </form> -</div> - -<div class="row"> - <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> -</div> - -<div class="row"> - <form id="frm-upload-rqtl2-bundle" - action="{{url_for('upload.rqtl2.create_geno_dataset', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - method="POST" - enctype="multipart/form-data"> - <legend class="heading">create a new genotype dataset</legend> - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" - value="{{rqtl2_bundle_file}}" /> - - {{flash_messages("error-rqtl2-create-geno-dataset")}} - - <div class="form-group"> - <label for="txt:dataset-name" class="form-label">Name</label> - <input type="text" - id="txt:dataset-name" - name="dataset-name" - maxlength="100" - required="required" - class="form-control" - aria-describedby="help-geno-dataset-name" /> - <span id="help-geno-dataset-name" class="form-text text-muted"> - Provide the new name for the genotype dataset, e.g. "BXDGeno" - </span> - </div> - - <div class="form-group"> - <label for="txt:dataset-fullname" class="form-label">Full Name</label> - <input type="text" - id="txt:dataset-fullname" - name="dataset-fullname" - required="required" - maxlength="100" - class="form-control" - aria-describedby="help-geno-dataset-fullname" /> - - <span id="help-geno-dataset-fullname" class="form-text text-muted"> - Provide a longer name that better describes the genotype dataset, e.g. - "BXD Genotypes" - </span> - </div> - - <div class="form-group"> - <label for="txt:dataset-shortname" class="form-label">Short Name</label> - <input type="text" - id="txt:dataset-shortname" - name="dataset-shortname" - maxlength="100" - class="form-control" - aria-describedby="help-geno-dataset-shortname" /> - - <span id="help-geno-dataset-shortname" class="form-text text-muted"> - Provide a short name for the genotype dataset. This is optional. If not - provided, we'll default to the same value as the "Name" above. - </span> - </div> - - <div class="form-group"> - <input type="checkbox" - id="chk:dataset-public" - name="dataset-public" - checked="checked" - class="form-check" - aria-describedby="help-geno-datasent-public" /> - <label for="chk:dataset-public" class="form-check-label">Public?</label> - - <span id="help-geno-dataset-public" class="form-text text-muted"> - Specify whether the dataset will be available publicly. Check to make the - dataset publicly available and uncheck to limit who can access the dataset. - </span> - </div> - - <button type="submit" class="btn btn-primary">create new dataset</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/select-population.html b/qc_app/templates/rqtl2/select-population.html deleted file mode 100644 index 37731f0..0000000 --- a/qc_app/templates/rqtl2/select-population.html +++ /dev/null @@ -1,136 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Select Grouping/Population{%endblock%} - -{%block contents%} -<h1 class="heading">Select grouping/population</h1> - -<div class="explainer"> - <p>The data is organised in a hierarchical form, beginning with - <em>species</em> at the very top. Under <em>species</em> the data is - organised by <em>population</em>, sometimes referred to as <em>grouping</em>. - (In some really old documents/systems, you might see this referred to as - <em>InbredSet</em>.)</p> - <p>In this section, you get to define what population your data is to be - organised by.</p> -</div> - -<form method="POST" - action="{{url_for('upload.rqtl2.select_population', species_id=species.SpeciesId)}}"> - <legend class="heading">select grouping/population</legend> - {{flash_messages("error-select-population")}} - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - - <div class="form-group"> - <label for="select:inbredset" class="form-label">population</label> - <select id="select:inbredset" - name="inbredset_id" - required="required" - class="form-control"> - <option value="">Select a grouping/population</option> - {%for pop in populations%} - <option value="{{pop.InbredSetId}}"> - {{pop.InbredSetName}} ({{pop.FullName}})</option> - {%endfor%} - </select> - <span class="form-text text-muted">If you are adding data to an already existing - population, simply pick the population from this drop-down selector. If - you cannot find your population from this list, try the form below to - create a new one..</span> - </div> - - <button type="submit" class="btn btn-primary" />select population</button> -</form> - -<p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> - -<form method="POST" - action="{{url_for('upload.rqtl2.create_population', species_id=species.SpeciesId)}}"> - <legend class="heading">create new grouping/population</legend> - {{flash_messages("error-create-population")}} - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - - <div class="form-group"> - <legend class="heading">mandatory</legend> - - <div class="form-group"> - <label for="txt:inbredset-name" class="form-label">name</label> - <input id="txt:inbredset-name" - name="inbredset_name" - type="text" - required="required" - maxlength="30" - placeholder="Enter grouping/population name" - class="form-control" /> - <span class="form-text text-muted">This is a short name that identifies the - population. Useful for menus, and quick scanning.</span> - </div> - - <div class="form-group"> - <label for="txt:" class="form-label">full name</label> - <input id="txt:inbredset-fullname" - name="inbredset_fullname" - type="text" - required="required" - maxlength="100" - placeholder="Enter the grouping/population's full name" - class="form-control" /> - <span class="form-text text-muted">This can be the same as the name above, or can - be longer. Useful for documentation, and human communication.</span> - </div> - </div> - - <div class="form-group"> - <legend class="heading">optional</legend> - - <div class="form-group"> - <label for="num:public" class="form-label">public?</label> - <select id="num:public" - name="public" - class="form-control"> - <option value="0">0 - Only accessible to authorised users</option> - <option value="1">1 - Publicly accessible to all users</option> - <option value="2" selected> - 2 - Publicly accessible to all users</option> - </select> - <span class="form-text text-muted">This determines whether the - population/grouping will appear on the menus for users.</span> - </div> - - <div class="form-group"> - <label for="txt:inbredset-family" class="form-label">family</label> - <input id="txt:inbredset-family" - name="inbredset_family" - type="text" - placeholder="I am not sure what this is about." - class="form-control" /> - <span class="form-text text-muted">I do not currently know what this is about. - This is a failure on my part to figure out what this is and provide a - useful description. Please feel free to remind me.</span> - </div> - - <div class="form-group"> - <label for="txtarea:" class="form-label">Description</label> - <textarea id="txtarea:description" - name="description" - rows="5" - placeholder="Enter a description of this grouping/population" - class="form-control"></textarea> - <span class="form-text text-muted"> - A long-form description of what the population consists of. Useful for - humans.</span> - </div> - </div> - - <button type="submit" class="btn btn-primary" /> - create grouping/population</button> -</form> - -{%endblock%} - - -{%block javascript%} -{%endblock%} diff --git a/qc_app/templates/rqtl2/select-probeset-dataset.html b/qc_app/templates/rqtl2/select-probeset-dataset.html deleted file mode 100644 index 26f52ed..0000000 --- a/qc_app/templates/rqtl2/select-probeset-dataset.html +++ /dev/null @@ -1,191 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Upload R/qtl2 Bundle{%endblock%} - -{%block contents%} -<h2 class="heading">Phenotype(ProbeSet) Dataset</h2> - -<div class="row"> - <p>The R/qtl2 bundle you uploaded contains (a) "<strong>pheno</strong>" - file(s). This data needs to be organised under a dataset.</p> - <p>This page gives you the ability to do that.</p> -</div> - -{%if datasets | length > 0%} -<div class="row"> - <form method="POST" - action="{{url_for('upload.rqtl2.select_probeset_dataset', - species_id=species.SpeciesId, population_id=population.Id)}}" - id="frm:select-probeset-dataset"> - <legend class="heading">Select from existing ProbeSet datasets</legend> - {{flash_messages("error-rqtl2")}} - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" - value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> - <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" /> - - <div class="form-group"> - <label class="form-label" for="select:probe-dataset">Dataset</label> - <select id="select:probe-dataset" - name="probe-dataset-id" - required="required" - {%if datasets | length == 0%}disabled="disabled"{%endif%} - class="form-control" - aria-describedby="help-probe-dataset"> - <option value="">Select a dataset</option> - {%for dataset in datasets%} - <option value={{dataset.Id}}> - {{dataset.Name}} - {%if dataset.FullName%} - -- ({{dataset.FullName}}) - {%endif%} - </option> - {%endfor%} - </select> - - <span id="help-probe-dataset" class="form-text text-muted"> - Select from existing ProbeSet datasets.</span> - </div> - - <button type="submit" class="btn btn-primary" />select dataset</button> -</form> -</div> - -<div class="row"> - <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> -</div> -{%endif%} - -<div class="row"> - <p>Create an entirely new ProbeSet dataset for your data.</p> -</div> - -<div class="row"> - <form method="POST" - action="{{url_for('upload.rqtl2.create_probeset_dataset', - species_id=species.SpeciesId, population_id=population.Id)}}" - id="frm:create-probeset-dataset"> - <legend class="heading">Create a new ProbeSet dataset</legend> - {{flash_messages("error-rqtl2-create-probeset-dataset")}} - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" - value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> - <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" /> - - <div class="form-group"> - <label class="form-label" for="select:average">averaging method</label> - <select id="select:average" - name="averageid" - required="required" - class="form-control" - aria-describedby="help-average"> - <option value="">Select averaging method</option> - {%for avgmethod in avgmethods%} - <option value="{{avgmethod.Id}}"> - {{avgmethod.Name}} - {%if avgmethod.Normalization%} - ({{avgmethod.Normalization}}) - {%endif%} - </option> - {%endfor%} - </select> - - <span id="help-average" class="form-text text-muted"> - Select the averaging method used for your data. - </span> - </div> - - <div class="form-group"> - <label class="form-label" for="txt:datasetname">Name</label> - <input type="text" id="txt:datasetname" name="datasetname" - required="required" - maxlength="40" - title="Name of the dataset, e.g 'BXDMicroArray_ProbeSet_June03'. (Required)" - class="form-control" - aria-describedby="help-dataset-name" /> - - <span id="help-dataset-name" class="form-text text-muted"> - Provide a name for the dataset e.g. "BXDMicroArray_ProbeSet_June03". This - is mandatory <strong>MUST</strong> be provided. - </span> - </div> - - <div class="form-group"> - <label class="form-label" for="txt:datasetfullname">Full Name</label> - <input type="text" id="txt:datasetfullname" name="datasetfullname" - required="required" - maxlength="100" - title="A longer name for the dataset, e.g. 'UTHSC Brain mRNA U74Av2 (Jun03) MAS5'. (Required)" - class="form-control" - aria-describedby="help-dataset-fullname" /> - - <span id="help-dataset-fullname" class="form-text text-muted"> - Provide a longer, more descriptive name for the dataset e.g. - "UTHSC Brain mRNA U74Av2 (Jun03) MAS5". This is mandatory and - <strong>MUST</strong> be provided. - </span> - </div> - - <div class="form-group"> - <label class="form-label" for="txt:datasetshortname">Short Name</label> - <input type="text" id="txt:datasetshortname" name="datasetshortname" - maxlength="100" - title="An abbreviated name for the dataset, e.g 'Br_U_0603_M'. (Optional)" - class="form-control" - aria-describedby="help-dataset-shortname" /> - - <span id="help-dataset-shortname" class="form-text text-muted"> - Provide a longer, more descriptive name for the dataset e.g. "Br_U_0603_M". - This is optional. - </span> - </div> - - <div class="form-check"> - <input type="checkbox" id="chk:public" name="datasetpublic" - checked="checked" - title="Whether or not the dataset is accessible by the general public." - class="form-check-input" - aria-describedby="help-public" /> - <label class="form-check-label" for="chk:datasetpublic">Public?</label> - - <span id="help-public" class="form-text text-muted"> - Check to specify that the dataset will be publicly available. Uncheck to - limit access to the dataset. - </span> - </div> - - <div class="form-group"> - <label class="form-label" for="select:datasetdatascale">Data Scale</label> - <select id="select:datasetdatascale" - name="datasetdatascale" - required="required" - class="form-control" - aria-describedby="help-dataset-datascale"> - <option value="log2" selected="selected">log2</option> - <option value="z_score">z_score</option> - <option value="log2_ratio">log2_ratio</option> - <option value="linear">linear</option> - <option value="linear_positive">linear_positive</option> - </select> - - <span id="help-dataset-datascale" class="form-text text-muted"> - Select from a list of scaling methods. - </span> - </div> - - <button type="submit" class="btn btn-primary">create dataset</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/select-probeset-study-id.html b/qc_app/templates/rqtl2/select-probeset-study-id.html deleted file mode 100644 index b9bf52e..0000000 --- a/qc_app/templates/rqtl2/select-probeset-study-id.html +++ /dev/null @@ -1,143 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages %} - -{%block title%}Upload R/qtl2 Bundle{%endblock%} - -{%block contents%} -<h2 class="heading">Phenotype(ProbeSet) Study</h2> - -<div class="row"> - <p>The R/qtl2 bundle you uploaded contains (a) "<strong>pheno</strong>" - file(s). This data needs to be organised under a study.</p> - <p>In this page, you can either select from a existing dataset:</p> - - <form method="POST" - action="{{url_for('upload.rqtl2.select_probeset_study', - species_id=species.SpeciesId, population_id=population.Id)}}" - id="frm:select-probeset-study"> - <legend class="heading">Select from existing ProbeSet studies</legend> - {{flash_messages("error-rqtl2-select-probeset-study")}} - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" - value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> - - <div> - <label for="select:probe-study" class="form-label">Study</label> - <select id="select:probe-study" - name="probe-study-id" - required="required" - aria-describedby="help-select-probeset-study" - {%if studies | length == 0%}disabled="disabled"{%endif%} - class="form-control"> - <option value="">Select a study</option> - {%for study in studies%} - <option value={{study.Id}}> - {{study.Name}} - {%if study.FullName%} - -- ({{study.FullName}}) - {%endif%} - </option> - {%endfor%} - </select> - <small id="help-select-probeset-study" class="form-text text-muted"> - Select from existing ProbeSet studies. - </small> - </div> - - <button type="submit" class="btn btn-primary">select study</button> - </form> -</div> - -<div class="row"> - <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> -</div> - -<div class="row"> - - <p>Create a new ProbeSet dataset below:</p> - - <form method="POST" - action="{{url_for('upload.rqtl2.create_probeset_study', - species_id=species.SpeciesId, population_id=population.Id)}}" - id="frm:create-probeset-study"> - <legend class="heading">Create new ProbeSet study</legend> - - {{flash_messages("error-rqtl2-create-probeset-study")}} - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" - value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - <input type="hidden" name="tissueid" value="{{tissue.Id}}" /> - - <div> - <label for="select:platform" class="form-label">Platform</label> - <select id="select:platform" - name="platformid" - required="required" - aria-describedby="help-select-platform" - {%if platforms | length == 0%}disabled="disabled"{%endif%} - class="form-control"> - <option value="">Select a platform</option> - {%for platform in platforms%} - <option value="{{platform.GeneChipId}}"> - {{platform.GeneChipName}} ({{platform.Name}}) - </option> - {%endfor%} - </select> - <small id="help-select-platform" class="form-text text-muted"> - Select from a list of known genomics platforms. - </small> - </div> - - <div class="form-group"> - <label for="txt:studyname" class="form-label">Study name</label> - <input type="text" id="txt:studyname" name="studyname" - placeholder="Name of the study. (Required)" - required="required" - maxlength="100" - class="form-control" /> - <span class="form-text text-muted" id="help-study-name"> - Provide a name for the study.</span> - </div> - - <div class="form-group"> - <label for="txt:studyfullname" class="form-label">Full Study Name</label> - <input type="text" - id="txt:studyfullname" - name="studyfullname" - placeholder="Longer name of the study. (Optional)" - maxlength="100" - class="form-control" /> - <span class="form-text text-muted" id="help-study-full-name"> - Provide a longer, more descriptive name for the study. This is optional - and you can leave it blank. - </span> - </div> - - <div class="form-group"> - <label for="txt:studyshortname" class="form-label">Short Study Name</label> - <input type="text" - id="txt:studyshortname" - name="studyshortname" - placeholder="Shorter name of the study. (Optional)" - maxlength="100" - class="form-control" /> - <span class="form-text text-muted" id="help-study-short-name"> - Provide a shorter name for the study. This is optional and you can leave - it blank. - </span> - </div> - - <button type="submit" class="btn btn-primary">create study</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/select-tissue.html b/qc_app/templates/rqtl2/select-tissue.html deleted file mode 100644 index 34e1758..0000000 --- a/qc_app/templates/rqtl2/select-tissue.html +++ /dev/null @@ -1,115 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Upload R/qtl2 Bundle{%endblock%} - -{%block contents%} -<h2 class="heading">Tissue</h2> - -<div class="row"> - <p>The data you are uploading concerns a tissue, cell, organ, or other - biological material used in an experiment.</p> - <p>Select the appropriate biological material below</p> -</div> - -{%if tissues | length > 0%} -<div class="row"> - <form method="POST" - action="{{url_for('upload.rqtl2.select_tissue', - species_id=species.SpeciesId, population_id=population.Id)}}" - id="frm:select-probeset-dataset"> - <legend class="heading">Select from existing ProbeSet datasets</legend> - {{flash_messages("error-select-tissue")}} - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" - value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - - <div class="form-group"> - <label class="form-label" for="select-tissue">Tissue</label> - <select id="select-tissue" - name="tissueid" - required="required" - {%if tissues | length == 0%}disabled="disabled"{%endif%} - class="form-control" - aria-describedby="help-select-tissue"> - <option value="">Select a tissue</option> - {%for tissue in tissues%} - <option value={{tissue.Id}}> - {{tissue.Name}} - {%if tissue.Short_Name%} - -- ({{tissue.Short_Name}}) - {%endif%} - </option> - {%endfor%} - </select> - - <span id="help-select-tissue" class="form-text text-muted"> - Select from existing biological material.</span> - </div> - - <button type="submit" class="btn btn-primary">use selected</button> - </form> -</div> - -<div class="row"> - <p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> -</div> -{%endif%} - -<div class="row"> - <p>If you cannot find the biological material in the drop-down above, add it - to the system below.</p> - - <form method="POST" - action="{{url_for('upload.rqtl2.create_tissue', - species_id=species.SpeciesId, population_id=population.Id)}}" - id="frm:create-probeset-dataset"> - <legend class="heading">Add new tissue, organ or biological material</legend> - {{flash_messages("error-create-tissue")}} - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" - value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - - <div class="form-group"> - <label class="form-label" for="tissue-name">name</label> - <input type="text" - id="txt-tissuename" - name="tissuename" - required="required" - title = "A name to identify the tissue, organ or biological material." - class="form-control" - aria-describedby="help-tissue-name" /> - - <span class="form-text text-muted" id="help-tissue-name"> - A name to identify the tissue, organ or biological material. - </span> - </div> - - <div class="form-group"> - <label for="txt-shortname" class="form-label">short name</label> - <input type="text" id="txt-tissueshortname" name="tissueshortname" - required="required" - maxlength="7" - title="A short name (e.g. 'Mam') for the biological material." - class="form-control" - aria-describedby="help-tissue-short-name" /> - - <span class="form-text text-muted" id="help-tissue-short-name"> - Provide a short name for the tissue, organ or biological material used in - the experiment. - </span> - </div> - - <button type="submit" class="btn btn-primary" />add new material</button> -</form> -</div> - -{%endblock%} diff --git a/qc_app/templates/rqtl2/summary-info.html b/qc_app/templates/rqtl2/summary-info.html deleted file mode 100644 index 1be87fa..0000000 --- a/qc_app/templates/rqtl2/summary-info.html +++ /dev/null @@ -1,65 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Upload R/qtl2 Bundle{%endblock%} - -{%block contents%} -<h2 class="heading">Summary</h2> - -<div class="row"> - <p>This is the information you have provided to accompany the R/qtl2 bundle - you have uploaded. Please verify the information is correct before - proceeding.</p> -</div> - -<div class="row"> - <dl> - <dt>Species</dt> - <dd>{{species.SpeciesName}} ({{species.FullName}})</dd> - - <dt>Population</dt> - <dd>{{population.InbredSetName}}</dd> - - {%if geno_dataset%} - <dt>Genotype Dataset</dt> - <dd>{{geno_dataset.Name}} ({{geno_dataset.FullName}})</dd> - {%endif%} - - {%if tissue%} - <dt>Tissue</dt> - <dd>{{tissue.TissueName}} ({{tissue.Name}}, {{tissue.Short_Name}})</dd> - {%endif%} - - {%if probe_study%} - <dt>ProbeSet Study</dt> - <dd>{{probe_study.Name}} ({{probe_study.FullName}})</dd> - {%endif%} - - {%if probe_dataset%} - <dt>ProbeSet Dataset</dt> - <dd>{{probe_dataset.Name2}} ({{probe_dataset.FullName}})</dd> - {%endif%} - </dl> -</div> - -<div class="row"> - <form id="frm:confirm-rqtl2bundle-details" - action="{{url_for('upload.rqtl2.confirm_bundle_details', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - method="POST" - enctype="multipart/form-data"> - <legend class="heading">Create ProbeSet dataset</legend> - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" value="{{rqtl2_bundle_file}}" /> - <input type="hidden" name="geno-dataset-id" value="{{geno_dataset.Id}}" /> - <input type="hidden" name="probe-study-id" value="{{probe_study.Id}}" /> - <input type="hidden" name="probe-dataset-id" value="{{probe_dataset.Id}}" /> - - <button type="submit" class="btn btn-primary">continue</button> - </form> -</div> -{%endblock%} diff --git a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-01.html b/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-01.html deleted file mode 100644 index 07c240f..0000000 --- a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-01.html +++ /dev/null @@ -1,276 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_all_messages%} -{%from "upload_progress_indicator.html" import upload_progress_indicator%} - -{%block title%}Upload R/qtl2 Bundle{%endblock%} - -{%block contents%} -{%macro rqtl2_file_help()%} -<span class="form-text text-muted"> - <p> - Provide a valid R/qtl2 zip file here. In particular, ensure your zip bundle - contains exactly one control file and the corresponding files mentioned in - the control file. - </p> - <p> - The control file can be either a YAML or JSON file. <em>ALL</em> other data - files in the zip bundle should be CSV files. - </p> - <p>See the - <a href="https://kbroman.org/qtl2/assets/vignettes/input_files.html" - target="_blank"> - R/qtl2 file format specifications - </a> - for more details. - </p> -</span> -{%endmacro%} -{{upload_progress_indicator()}} - -<div id="resumable-file-display-template" - class="panel panel-info" - style="display: none"> - <div class="panel-heading"></div> - <div class="panel-body"></div> -</div> - - -<h2 class="heading">Upload R/qtl2 Bundle</h2> - -<div id="resumable-drop-area" - style="display:none;background:#eeeeee;min-height:12em;border-radius:0.5em;padding:1em;"> - <p> - <a id="resumable-browse-button" href="#" - class="btn btn-info">Browse</a> - </p> - <p class="form-text text-muted"> - You can drag and drop your file here, or click the browse button. - Click on the file to remove it. - </p> - {{rqtl2_file_help()}} - <div id="resumable-selected-files" - style="display:flex;flex-direction:row;flex-wrap: wrap;justify-content:space-around;gap:10px 20px;"></div> - <div id="resumable-class-buttons" style="text-align: right;"> - <button id="resumable-upload-button" - class="btn btn-primary" - style="display: none">start upload</button> - <button id="resumable-cancel-upload-button" - class="btn btn-danger" - style="display: none">cancel upload</button> - </div> - <div id="resumable-progress-bar" class="progress" style="display: none"> - <div class="progress-bar" - role="progress-bar" - aria-valuenow="60" - aria-valuemin="0" - aria-valuemax="100" - style="width: 0%;"> - Uploading: 60% - </div> - </div> -</div> - -<form id="frm-upload-rqtl2-bundle" - action="{{url_for('upload.rqtl2.upload_rqtl2_bundle', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - method="POST" - enctype="multipart/form-data" - data-resumable-target="{{url_for( - 'upload.rqtl2.upload_rqtl2_bundle_chunked_post', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}"> - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - - {{flash_all_messages()}} - - <div class="form-group"> - <legend class="heading">file upload</legend> - <label for="file-rqtl2-bundle" class="form-label">R/qtl2 bundle</label> - <input type="file" id="file-rqtl2-bundle" name="rqtl2_bundle_file" - accept="application/zip, .zip" - required="required" - class="form-control" /> - {{rqtl2_file_help()}} - </div> - - <button type="submit" - class="btn btn-primary" - data-toggle="modal" - data-target="#upload-progress-indicator">upload R/qtl2 bundle</button> -</form> - -{%endblock%} - -{%block javascript%} -<script src="{{url_for('base.node_modules', - filename='resumablejs/resumable.js')}}"></script> -<script type="text/javascript" src="/static/js/upload_progress.js"></script> -<script type="text/javascript"> - function readBinaryFile(file) { - return new Promise((resolve, reject) => { - var _reader = new FileReader(); - _reader.onload = (event) => {resolve(_reader.result);}; - _reader.readAsArrayBuffer(file); - }); - } - - function computeFileChecksum(file) { - return readBinaryFile(file) - .then((content) => { - return window.crypto.subtle.digest( - "SHA-256", new Uint8Array(content)); - }).then((digest) => { - return Uint8ArrayToHex(new Uint8Array(digest)) - }); - } - - function Uint8ArrayToHex(arr) { - var toHex = (val) => { - _hex = val.toString(16); - if(_hex.length < 2) { - return "0" + val; - } - return _hex; - }; - _hexstr = "" - arr.forEach((val) => {_hexstr += toHex(val)}); - return _hexstr - } - - var r = Resumable({ - target: $("#frm-upload-rqtl2-bundle").attr("data-resumable-target"), - fileType: ["zip"], - maxFiles: 1, - forceChunkSize: true, - generateUniqueIdentifier: (file, event) => { - return computeFileChecksum(file).then((checksum) => { - var _relativePath = (file.webkitRelativePath - || file.relativePath - || file.fileName - || file.name); - return checksum + "-" + _relativePath.replace( - /[^a-zA-Z0-9_-]/img, ""); - }); - } - }); - - if(r.support) { - //Hide form and display drag&drop UI - $("#frm-upload-rqtl2-bundle").css("display", "none"); - $("#resumable-drop-area").css("display", "block"); - - // Define UI elements for browse and drag&drop - r.assignDrop(document.getElementById("resumable-drop-area")); - r.assignBrowse(document.getElementById("resumable-browse-button")); - - // Event handlers - - function display_files(files) { - displayArea = $("#resumable-selected-files") - displayArea.empty(); - files.forEach((file) => { - var displayElement = $( - "#resumable-file-display-template").clone(); - displayElement.removeAttr("id"); - displayElement.css("display", ""); - displayElement.find(".panel-heading").text(file.fileName); - list = $("<ul></ul>"); - list.append($("<li><strong>Name</strong>: " - + (file.name - || file.fileName - || file.relativePath - || file.webkitRelativePath) - + "</li>")); - list.append($("<li><strong>Size</strong>: " - + (file.size / (1024*1024)).toFixed(2) - + " MB</li>")); - list.append($("<li><strong>Unique Identifier</strong>: " - + file.uniqueIdentifier + "</li>")); - list.append($("<li><strong>Mime</strong>: " - + file.file.type - + "</li>")); - displayElement.find(".panel-body").append(list); - displayElement.appendTo("#resumable-selected-files"); - }); - } - - r.on("filesAdded", function(files) { - display_files(files); - $("#resumable-upload-button").css("display", ""); - $("#resumable-upload-button").on("click", (event) => { - r.upload(); - }); - }); - - r.on("uploadStart", (event) => { - $("#resumable-upload-button").css("display", "none"); - $("#resumable-cancel-upload-button").css("display", ""); - $("#resumable-cancel-upload-button").on("click", (event) => { - r.files.forEach((file) => { - if(file.isUploading()) { - file.abort(); - } - }); - $("#resumable-cancel-upload-button").css("display", "none"); - $("#resumable-upload-button").on("click", (event) => { - r.files.forEach((file) => {file.retry();}); - }); - $("#resumable-upload-button").css("display", ""); - }); - }); - - r.on("progress", () => { - var progress = (r.progress() * 100).toFixed(2); - var pbar = $("#resumable-progress-bar > .progress-bar"); - $("#resumable-progress-bar").css("display", ""); - pbar.css("width", progress+"%"); - pbar.attr("aria-valuenow", progress); - pbar.text("Uploading: " + progress + "%"); - }) - - r.on("fileSuccess", (file, message) => { - if(message != "OK") { - var uri = (window.location.protocol - + "//" - + window.location.host - + message); - window.location.replace(uri); - } - }); - - r.on("error", (message, file) => { - filename = (file.webkitRelativePath - || file.relativePath - || file.fileName - || file.name); - jsonmsg = JSON.parse(message); - alert("There was an error while uploading your file '" - + filename - + "'. The error message was:\n\n\t" - + jsonmsg.error - + " (" - + jsonmsg.statuscode - + "): " + jsonmsg.message); - }) - } else { - setup_upload_handlers( - "frm-upload-rqtl2-bundle", make_data_uploader( - function (form) { - var formdata = new FormData(); - formdata.append( - "species_id", - form.querySelector('input[name="species_id"]').value); - formdata.append( - "population_id", - form.querySelector('input[name="population_id"]').value); - formdata.append( - "rqtl2_bundle_file", - form.querySelector("#file-rqtl2-bundle").files[0]); - return formdata; - })); - } -</script> -{%endblock%} diff --git a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-02.html b/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-02.html deleted file mode 100644 index 93b1dc9..0000000 --- a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-02.html +++ /dev/null @@ -1,33 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_all_messages%} - -{%block title%}Upload R/qtl2 Bundle{%endblock%} - -{%block contents%} -<h2 class="heading">Upload R/qtl2 Bundle</h2> - -<div class="row"> - <p>You have successfully uploaded the zipped bundle of R/qtl2 files.</p> - <p>The next step is to select the various extra information we need to figure - out what to do with the data. You will select/create the relevant studies - and/or datasets to organise the data in the steps that follow.</p> - <p>Click "Continue" below to proceed.</p> - - <form id="frm-upload-rqtl2-bundle" - action="{{url_for('upload.rqtl2.select_dataset_info', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - method="POST" - enctype="multipart/form-data"> - {{flash_all_messages()}} - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" - value="{{population.InbredSetId}}" /> - <input type="hidden" name="rqtl2_bundle_file" - value="{{rqtl2_bundle_file}}" /> - - <button type="submit" class="btn btn-primary">continue</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/samples/select-population.html b/qc_app/templates/samples/select-population.html deleted file mode 100644 index da19ddc..0000000 --- a/qc_app/templates/samples/select-population.html +++ /dev/null @@ -1,99 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Select Grouping/Population{%endblock%} - -{%block contents%} -<h1 class="heading">Select grouping/population</h1> - -<div> - <p>We organise the samples/cases/strains in a hierarchichal form, starting - with <strong>species</strong> at the very top. Under species, we have a - grouping in terms of the relevant population - (e.g. Inbred populations, cell tissue, etc.)</p> -</div> - -<form method="POST" action="{{url_for('samples.select_population', - species_id=species.SpeciesId)}}"> - <legend class="heading">select grouping/population</legend> - {{flash_messages("error-select-population")}} - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - - <div class="form-group"> - <label for="select:inbredset" class="form-label">grouping/population</label> - <select id="select:inbredset" - name="inbredset_id" - required="required" - class="form-control"> - <option value="">Select a grouping/population</option> - {%for pop in populations%} - <option value="{{pop.InbredSetId}}"> - {{pop.InbredSetName}} ({{pop.FullName}})</option> - {%endfor%} - </select> - </div> - - <button type="submit" class="btn btn-primary">select population</button> -</form> - -<p style="color:#FE3535; padding-left:20em; font-weight:bolder;">OR</p> - -<form method="POST" action="{{url_for('samples.create_population', - species_id=species.SpeciesId)}}"> - <legend class="heading">create new grouping/population</legend> - {{flash_messages("error-create-population")}} - - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <div class="form-group"> - <legend>mandatory</legend> - - <label for="txt:inbredset-name" class="form-label">name</label> - <input id="txt:inbredset-name" - name="inbredset_name" - type="text" - required="required" - placeholder="Enter grouping/population name" - class="form-control" /> - - <label for="txt:" class="form-label">full name</label> - <input id="txt:inbredset-fullname" - name="inbredset_fullname" - type="text" - required = "required" - placeholder="Enter the grouping/population's full name" - class="form-control" /> - </div> - <div class="form-group"> - <legend>Optional</legend> - - <label for="num:public" class="form-label">public?</label> - <input id="num:public" - name="public" - type="number" - min="0" max="2" value="2" - class="form-control" /> - - <label for="txt:inbredset-family" class="form-label">family</label> - <input id="txt:inbredset-family" - name="inbredset_family" - type="text" - placeholder="I am not sure what this is about." - class="form-control" /> - - <label for="txtarea:" class="form-label">Description</label> - <textarea id="txtarea:description" - name="description" - rows="5" - placeholder="Enter a description of this grouping/population" - class="form-control"></textarea> - </div> - - <button type="submit" class="btn btn-primary">create grouping/population</button> -</form> - -{%endblock%} - - -{%block javascript%} -{%endblock%} diff --git a/qc_app/templates/samples/select-species.html b/qc_app/templates/samples/select-species.html deleted file mode 100644 index edadc61..0000000 --- a/qc_app/templates/samples/select-species.html +++ /dev/null @@ -1,30 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_all_messages%} - -{%block title%}Select Grouping/Population{%endblock%} - -{%block contents%} -<h2 class="heading">upload samples/cases</h2> - -<p>We need to know what species your data belongs to.</p> - -{{flash_all_messages()}} - -<form method="POST" action="{{url_for('samples.select_species')}}"> - <legend class="heading">upload samples</legend> - <div class="form-group"> - <label for="select_species02" class="form-label">Species</label> - <select id="select_species02" - name="species_id" - required="required" - class="form-control"> - <option value="">Select species</option> - {%for spec in species%} - <option value="{{spec.SpeciesId}}">{{spec.MenuName}}</option> - {%endfor%} - </select> - </div> - - <button type="submit" class="btn btn-primary">submit</button> -</form> -{%endblock%} diff --git a/qc_app/templates/samples/upload-failure.html b/qc_app/templates/samples/upload-failure.html deleted file mode 100644 index 09e2ecf..0000000 --- a/qc_app/templates/samples/upload-failure.html +++ /dev/null @@ -1,27 +0,0 @@ -{%extends "base.html"%} -{%from "cli-output.html" import cli_output%} - -{%block title%}Samples Upload Failure{%endblock%} - -{%block contents%} -<h1 class="heading">{{job.job_name}}</h2> - -<p>There was a failure attempting to upload the samples.</p> - -<p>Here is some information to help with debugging the issue. Provide this - information to the developer/maintainer.</p> - -<h3>Debugging Information</h3> -<ul> - <li><strong>job id</strong>: {{job.job_id}}</li> - <li><strong>status</strong>: {{job.status}}</li> - <li><strong>job type</strong>: {{job["job-type"]}}</li> -</ul> - -<h4>stdout</h4> -{{cli_output(job, "stdout")}} - -<h4>stderr</h4> -{{cli_output(job, "stderr")}} - -{%endblock%} diff --git a/qc_app/templates/samples/upload-progress.html b/qc_app/templates/samples/upload-progress.html deleted file mode 100644 index 7bb02be..0000000 --- a/qc_app/templates/samples/upload-progress.html +++ /dev/null @@ -1,22 +0,0 @@ -{%extends "base.html"%} -{%from "cli-output.html" import cli_output%} - -{%block extrameta%} -<meta http-equiv="refresh" content="5"> -{%endblock%} - -{%block title%}Job Status{%endblock%} - -{%block contents%} -<h1 class="heading">{{job.job_name}}</h2> - -<p> -<strong>status</strong>: -<span>{{job["status"]}} ({{job.get("message", "-")}})</span><br /> -</p> - -<p>saving to database...</p> - -{{cli_output(job, "stdout")}} - -{%endblock%} diff --git a/qc_app/templates/samples/upload-samples.html b/qc_app/templates/samples/upload-samples.html deleted file mode 100644 index e62de57..0000000 --- a/qc_app/templates/samples/upload-samples.html +++ /dev/null @@ -1,139 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} - -{%block title%}Upload Samples{%endblock%} - -{%block css%}{%endblock%} - -{%block contents%} -<h1 class="heading">upload samples</h1> - -{{flash_messages("alert-success")}} - -<p>You can now upload a character-separated value (CSV) file that contains - details about your samples. The CSV file should have the following fields: - <dl> - <dt>Name</dt> - <dd>The primary name for the sample</dd> - - <dt>Name2</dt> - <dd>A secondary name for the sample. This can simply be the same as - <strong>Name</strong> above. This field <strong>MUST</strong> contain a - value.</dd> - - <dt>Symbol</dt> - <dd>A symbol for the sample. Can be an empty field.</dd> - - <dt>Alias</dt> - <dd>An alias for the sample. Can be an empty field.</dd> - </dl> -</p> - -<form id="form-samples" - method="POST" - action="{{url_for('samples.upload_samples', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}" - enctype="multipart/form-data"> - <legend class="heading">upload samples</legend> - - <div class="form-group"> - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <label class="form-label">species:</label> - <span class="form-text">{{species.SpeciesName}} [{{species.MenuName}}]</span> - </div> - - <div class="form-group"> - <input type="hidden" name="inbredset_id" value="{{population.InbredSetId}}" /> - <label class="form-label">grouping/population:</label> - <span class="form-text">{{population.Name}} [{{population.FullName}}]</span> - </div> - - <div class="form-group"> - <label for="file-samples" class="form-label">select file</label> - <input type="file" name="samples_file" id="file:samples" - accept="text/csv, text/tab-separated-values" - class="form-control" /> - </div> - - <div class="form-group"> - <label for="select:separator" class="form-label">field separator</label> - <select id="select:separator" - name="separator" - required="required" - class="form-control"> - <option value="">Select separator for your file: (default is comma)</option> - <option value="	">TAB</option> - <option value=" ">Space</option> - <option value=",">Comma</option> - <option value=";">Semicolon</option> - <option value="other">Other</option> - </select> - <input id="txt:separator" - type="text" - name="other_separator" - class="form-control" /> - <small class="form-text text-muted"> - If you select '<strong>Other</strong>' for the field separator value, - enter the character that separates the fields in your CSV file in the form - field below. - </small> - </div> - - <div class="form-group form-check"> - <input id="chk:heading" - type="checkbox" - name="first_line_heading" - class="form-check-input" /> - <label for="chk:heading" class="form-check-label"> - first line is a heading?</label> - <small class="form-text text-muted"> - Select this if the first line in your file contains headings for the - columns. - </small> - </div> - - <div class="form-group"> - <label for="txt:delimiter" class="form-label">field delimiter</label> - <input id="txt:delimiter" - type="text" - name="field_delimiter" - maxlength="1" - class="form-control" /> - <small class="form-text text-muted"> - If there is a character delimiting the string texts within particular - fields in your CSV, provide the character here. This can be left blank if - no such delimiters exist in your file. - </small> - </div> - - <button type="submit" - class="btn btn-primary">upload samples file</button> -</form> - -<table id="tbl:samples-preview" class="table"> - <caption class="heading">preview content</caption> - - <thead> - <tr> - <th>Name</th> - <th>Name2</th> - <th>Symbol</th> - <th>Alias</th> - </tr> - </thead> - - <tbody> - <tr id="default-row"> - <td colspan="4"> - Please make some selections to preview the data.</td> - </tr> - </tbody> -</table> - -{%endblock%} - - -{%block javascript%} -<script src="/static/js/upload_samples.js" type="text/javascript"></script> -{%endblock%} diff --git a/qc_app/templates/samples/upload-success.html b/qc_app/templates/samples/upload-success.html deleted file mode 100644 index cb745c3..0000000 --- a/qc_app/templates/samples/upload-success.html +++ /dev/null @@ -1,18 +0,0 @@ -{%extends "base.html"%} -{%from "cli-output.html" import cli_output%} - -{%block title%}Job Status{%endblock%} - -{%block contents%} -<h1 class="heading">{{job.job_name}}</h2> - -<p> -<strong>status</strong>: -<span>{{job["status"]}} ({{job.get("message", "-")}})</span><br /> -</p> - -<p>Successfully uploaded the samples.</p> - -{{cli_output(job, "stdout")}} - -{%endblock%} diff --git a/qc_app/templates/select_dataset.html b/qc_app/templates/select_dataset.html deleted file mode 100644 index 2f07de8..0000000 --- a/qc_app/templates/select_dataset.html +++ /dev/null @@ -1,161 +0,0 @@ -{%extends "base.html"%} -{%from "dbupdate_hidden_fields.html" import hidden_fields%} - -{%block title%}Select Dataset{%endblock%} - -{%block css%} -<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> -{%endblock%} - -{%block contents%} -<h2 class="heading">{{filename}}: select dataset</h2> - -<div class="row"> - <form method="POST" action="{{url_for('dbinsert.final_confirmation')}}" - id="select-dataset-form" class="two-col-sep-col1"> - <legend class="heading">choose existing dataset</legend> - {{hidden_fields( - filename, filetype, species=species, genechipid=genechipid, - studyid=studyid, totallines=totallines)}} - - <div class="form-group"> - <label for="datasetid" class="form-label">dataset:</label> - <select id="datasetid" name="datasetid" class="form-control" - {%if datasets | length == 0:%} - disabled="disabled" - {%endif%}> - {%for dataset in datasets:%} - <option value="{{dataset['Id']}}"> - [{{dataset["Name"]}}] - {{dataset["FullName"]}} - </option> - {%endfor%} - </select> - </div> - - <button type="submit" class="btn btn-primary" - {%if datasets | length == 0:%} - disabled="disabled" - {%endif%} />update database</button> -</form> -</div> - -<div class="row"> - <p class="two-col-sep-separator">OR</p> -</div> - -<div class="row"> - <form method="POST" id="create-dataset-form" - action="{{url_for('dbinsert.create_dataset')}}" - class="two-col-sep-col2"> - <legend class="heading">create new dataset</legend> - {{hidden_fields( - filename, filetype, species=species, genechipid=genechipid, - studyid=studyid, totallines=totallines)}} - - {%with messages = get_flashed_messages(with_categories=true)%} - {%if messages:%} - <ul> - {%for category, message in messages:%} - <li class="{{category}}">{{message}}</li> - {%endfor%} - </ul> - {%endif%} - {%endwith%} - - <div class="form-group"> - <label for="avgid" class="form-label">average:</label> - <select id="avgid" name="avgid" required="required" class="form-control"> - <option value="">Select averaging method</option> - {%for method in avgmethods:%} - <option value="{{method['AvgMethodId']}}" - {%if avgid is defined and method['AvgMethodId'] | int == avgid | int%} - selected="selected" - {%endif%}> - {{method["Name"]}} - </option> - {%endfor%} - </select> - </div> - - <div class="form-group"> - <label for="datasetname" class="form-label">name:</label> - <input id="datasetname" name="datasetname" type="text" - class="form-control" - {%if datasetname is defined %} - value="{{datasetname}}" - {%endif%} /> - </div> - - <div class="form-group"> - <label for="datasetname2" class="form-label">name 2:</label> - <input id="datasetname2" name="datasetname2" type="text" - required="required" class="form-control" - {%if datasetname2 is defined %} - value="{{datasetname2}}" - {%endif%} /> - </div> - - <div class="form-group"> - <label for="datasetfullname" class="form-label">full name:</label> - <input id="datasetfullname" name="datasetfullname" type="text" - required="required" class="form-control" - {%if datasetfullname is defined %} - value="{{datasetfullname}}" - {%endif%} /> - </div> - - <div class="form-group"> - <label for="datasetshortname" class="form-label">short name:</label> - <input id="datasetshortname" name="datasetshortname" type="text" - required="required" class="form-control" - {%if datasetshortname is defined %} - value="{{datasetshortname}}" - {%endif%} /> - </div> - - <div class="form-group"> - <label for="datasetpublic" class="form-label">public:</label> - <input id="datasetpublic" name="datasetpublic" type="number" - required="required" min="0" max="2" - {%if datasetpublic is defined %} - value="{{datasetpublic | int}}" - {%else%} - value="0" - {%endif%} - class="form-control" /> - </div> - - <div class="form-group"> - <label for="datasetconfidentiality">confidentiality:</label> - <input id="datasetconfidentiality" name="datasetconfidentiality" - type="number" required="required" min="0" max="2" - {%if datasetconfidentiality is defined %} - value="{{datasetconfidentiality | int}}" - {%else%} - value="0" - {%endif%} - class="form-control" /> - </div> - - <div class="form-group"> - <label for="datasetdatascale" class="form-label">data scale:</label> - <select id="datasetdatascale" name="datasetdatascale" class="form-control"> - <option value="">None</option> - {%for dscale in datascales:%} - <option value="{{dscale}}" - {%if datasetdatascale is defined and dscale == datasetdatascale%} - selected="selected" - {%elif dscale == "log2":%} - selected="selected" - {%endif%}> - {{dscale}} - </option> - {%endfor%} - </select> - </div> - - <button type="submit" class="btn btn-primary">create new dataset</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/select_platform.html b/qc_app/templates/select_platform.html deleted file mode 100644 index d9bc68f..0000000 --- a/qc_app/templates/select_platform.html +++ /dev/null @@ -1,82 +0,0 @@ -{%extends "base.html"%} - -{%block title%}Select Dataset{%endblock%} - -{%block contents%} -<h2 class="heading">{{filename}}: select platform</h2> - -<div class="row"> - <form method="POST" action="{{url_for('dbinsert.select_study')}}" - id="select-platform-form" data-genechips="{{genechips_data}}"> - <input type="hidden" name="filename" value="{{filename}}" /> - <input type="hidden" name="filetype" value="{{filetype}}" /> - <input type="hidden" name="totallines" value="{{totallines}}" /> - - <div class="form-group"> - <label for="species" class="form-label">species</label> - <select id="species" name="species" class="form-control"> - {%for row in species:%} - <option value="{{row['SpeciesId']}}" - {%if row["Name"] == default_species:%} - selected="selected" - {%endif%}> - {{row["MenuName"]}} - </option> - {%endfor%} - </select> - </div> - - <table id="genechips-table" class="table"> - <caption>select platform</caption> - <thead> - <tr> - <th>Select</th> - <th>GeneChip ID</th> - <th>GeneChip Name</th> - </tr> - </thead> - - <tbody> - {%for chip in genechips:%} - <tr> - <td> - <input type="radio" name="genechipid" value="{{chip['GeneChipId']}}" - required="required" /> - </td> - <td>{{chip["GeneChipId"]}}</td> - <td>{{chip["GeneChipName"]}}</td> - </tr> - {%else%} - <tr> - <td colspan="5">No chips found for selected species</td> - </tr> - {%endfor%} - </tbody> - </table> - - <button type="submit" class="btn btn-primary">submit platform</button> - </form> -</div> -{%endblock%} - -{%block javascript%} -<script type="text/javascript" src="/static/js/utils.js"></script> -<script type="text/javascript" src="/static/js/select_platform.js"></script> -<script type="text/javascript"> - document.getElementById( - "species").addEventListener("change", update_genechips); - document.getElementById( - "genechips-table").getElementsByTagName( - "tbody")[0].addEventListener( - "click", - function(event) { - if(event.target.tagName.toLowerCase() == "td") { - return select_row_radio(event.target.parentElement); - } - if(event.target.tagName.toLowerCase() == "td") { - return select_row_radio(event.target); - } - return false; - }); -</script> -{%endblock%} diff --git a/qc_app/templates/select_species.html b/qc_app/templates/select_species.html deleted file mode 100644 index 3b1a8a9..0000000 --- a/qc_app/templates/select_species.html +++ /dev/null @@ -1,92 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} -{%from "upload_progress_indicator.html" import upload_progress_indicator%} - -{%block title%}expression data: select species{%endblock%} - -{%block contents%} -{{upload_progress_indicator()}} - -<h2 class="heading">expression data: select species</h2> - -<div class="row"> - <form action="{{url_for('entry.upload_file')}}" - method="POST" - enctype="multipart/form-data" - id="frm-upload-expression-data"> - <legend class="heading">upload expression data</legend> - {{flash_messages("error-expr-data")}} - - <div class="form-group"> - <label for="select_species01" class="form-label">Species</label> - <select id="select_species01" - name="speciesid" - required="required" - class="form-control"> - <option value="">Select species</option> - {%for aspecies in species%} - <option value="{{aspecies.SpeciesId}}">{{aspecies.MenuName}}</option> - {%endfor%} - </select> - </div> - - <div class="form-group"> - <legend class="heading">file type</legend> - - <div class="form-check"> - <input type="radio" name="filetype" value="average" id="filetype_average" - required="required" class="form-check-input" /> - <label for="filetype_average" class="form-check-label">average</label> - </div> - - <div class="form-check"> - <input type="radio" name="filetype" value="standard-error" - id="filetype_standard_error" required="required" - class="form-check-input" /> - <label for="filetype_standard_error" class="form-check-label"> - standard error - </label> - </div> - </div> - - <div class="form-group"> - <span id="no-file-error" class="alert-danger" style="display: none;"> - No file selected - </span> - <label for="file_upload" class="form-label">select file</label> - <input type="file" name="qc_text_file" id="file_upload" - accept="text/plain, text/tab-separated-values, application/zip" - class="form-control"/> - </div> - - <button type="submit" - class="btn btn-primary" - data-toggle="modal" - data-target="#upload-progress-indicator">upload file</button> - </form> -</div> -{%endblock%} - - -{%block javascript%} -<script type="text/javascript" src="static/js/upload_progress.js"></script> -<script type="text/javascript"> - function setup_formdata(form) { - var formdata = new FormData(); - formdata.append( - "speciesid", - form.querySelector("#select_species01").value) - formdata.append( - "qc_text_file", - form.querySelector("input[type='file']").files[0]); - formdata.append( - "filetype", - selected_filetype( - Array.from(form.querySelectorAll("input[type='radio']")))); - return formdata; - } - - setup_upload_handlers( - "frm-upload-expression-data", make_data_uploader(setup_formdata)); -</script> -{%endblock%} diff --git a/qc_app/templates/select_study.html b/qc_app/templates/select_study.html deleted file mode 100644 index 648ad4c..0000000 --- a/qc_app/templates/select_study.html +++ /dev/null @@ -1,108 +0,0 @@ -{%extends "base.html"%} -{%from "dbupdate_hidden_fields.html" import hidden_fields%} - -{%block title%}Select Dataset{%endblock%} - -{%block css%} -<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> -{%endblock%} - -{%block contents%} -<h2 class="heading">{{filename}}: select study</h2> - -<div class="row"> - <form method="POST" action="{{url_for('dbinsert.select_dataset')}}" - id="select-platform-form" data-genechips="{{genechips_data}}" - class="two-col-sep-col1"> - <legend class="heading">Select from existing study</legend> - {{hidden_fields(filename, filetype, species=species, genechipid=genechipid, - totallines=totallines)}} - - <div class="form-group"> - <label class="form-label" for="study">study:</label> - <select id="study" name="studyid" class="form-control"> - {%for study in studies:%} - <option value="{{study['Id']}}">{{study["Name"]}}</option> - {%endfor%} - </select> - </div> - - <button type="submit" - class="btn btn-primary" - {%if studies | length == 0:%} - disabled="disabled" - {%endif%} />submit selected study</button> -</form> -</div> - -<div class="row"> - <p class="two-col-sep-separator">OR</p> -</div> - -<div class="row"> - <form method="POST" action="{{url_for('dbinsert.create_study')}}" - id="select-platform-form" data-genechips="{{genechips_data}}" - class="two-col-sep-col2"> - {%with messages = get_flashed_messages(with_categories=true)%} - {%if messages:%} - <ul> - {%for category, message in messages:%} - <li class="{{category}}">{{message}}</li> - {%endfor%} - </ul> - {%endif%} - {%endwith%} - <legend class="heading">Create new study</legend> - {{hidden_fields(filename, filetype, species=species, genechipid=genechipid, - totallines=totallines)}} - - <div class="form-group"> - <label class="form-label" for="studyname">name:</label> - <input type="text" id="studyname" name="studyname" class="form-control" - required="required" - {%if studyname:%} - value="{{studyname}}" - {%endif%} /> - </div> - - <div class="form-group"> - <label class="form-label" for="group">group:</label> - <select id="group" name="inbredsetid" class="form-control" - required="required"> - <option value="">Select group</option> - {%for family in groups:%} - <optgroup label="{{family}}"> - {%for group in groups[family]:%} - <option value="{{group['InbredSetId']}}" - {%if group["InbredSetId"] == selected_group:%} - selected="selected" - {%endif%}> - {{group["FullName"]}} - </option> - {%endfor%} - </optgroup> - {%endfor%} - </select> - </div> - - <div class="form-group"> - <label class="form-label" for="tissue">tissue:</label> - <select id="tissue" name="tissueid" class="form-control" - required="required"> - <option value="">Select type</option> - {%for tissue in tissues:%} - <option value="{{tissue['TissueId']}}" - {%if tissue["TissueId"] == selected_tissue:%} - selected="selected" - {%endif%}> - {{tissue["Name"]}} - </option> - {%endfor%} - </select> - </div> - - <button type="submit" class="btn btn-primary">create study</button> - </form> -</div> - -{%endblock%} diff --git a/qc_app/templates/stdout_output.html b/qc_app/templates/stdout_output.html deleted file mode 100644 index 85345a9..0000000 --- a/qc_app/templates/stdout_output.html +++ /dev/null @@ -1,8 +0,0 @@ -{%macro stdout_output(job)%} - -<h4>STDOUT Output</h4> -<div class="cli-output"> - <pre>{{job.get("stdout", "")}}</pre> -</div> - -{%endmacro%} diff --git a/qc_app/templates/unhandled_exception.html b/qc_app/templates/unhandled_exception.html deleted file mode 100644 index 6e6a051..0000000 --- a/qc_app/templates/unhandled_exception.html +++ /dev/null @@ -1,21 +0,0 @@ -{%extends "base.html"%} - -{%block title%}System Error{%endblock%} - -{%block css%} -<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> -{%endblock%} - -{%block contents%} -<p> - An error has occured, and your request has been aborted. Please notify the - administrator to try and get this sorted. -</p> -<p> - Provide the following information to help the administrator figure out and fix - the issue:<br /> - <hr /><br /> - {{trace}} - <hr /><br /> -</p> -{%endblock%} diff --git a/qc_app/templates/upload_progress_indicator.html b/qc_app/templates/upload_progress_indicator.html deleted file mode 100644 index e274e83..0000000 --- a/qc_app/templates/upload_progress_indicator.html +++ /dev/null @@ -1,35 +0,0 @@ -{%macro upload_progress_indicator()%} -<div id="upload-progress-indicator" class="modal fade" tabindex="-1" role="dialog"> - <div class="modal-dialog" role="document"> - <div class="modal-content"> - <div class="modal-header"> - <h3 class="modal-title">Uploading file</h3> - </div> - - <div class="modal-body"> - <form id="frm-cancel-upload" style="border-style: none;"> - <div class="form-group"> - <span id="progress-filename" class="form-text">No file selected!</span> - <progress id="progress-bar" value="0" max="100" class="form-control"> - 0</progress> - </div> - - <div class="form-group"> - <span class="form-text text-muted" id="progress-text"> - Uploading 0%</span> - <span class="form-text text-muted" id="progress-extra-text"> - Processing</span> - </div> - </form> - </div> - - <div class="modal-footer"> - <button id="btn-cancel-upload" - type="button" - class="btn btn-danger" - data-dismiss="modal">Cancel</button> - </div> - </div> - </div> -</div> -{%endmacro%} diff --git a/qc_app/templates/worker_failure.html b/qc_app/templates/worker_failure.html deleted file mode 100644 index b65b140..0000000 --- a/qc_app/templates/worker_failure.html +++ /dev/null @@ -1,24 +0,0 @@ -{%extends "base.html"%} - -{%block title%}Worker Failure{%endblock%} - -{%block contents%} -<h1 class="heading">Worker Failure</h1> - -<p> - There was a critical failure launching the job to parse your file. - This is our fault and (probably) has nothing to do with the file you uploaded. -</p> - -<p> - Please notify the developers of this issue when you encounter it, - providing the link to this page, or the information below. -</p> - -<h4>Debugging Information</h4> - -<ul> - <li><strong>job id</strong>: {{job_id}}</li> -</ul> - -{%endblock%} diff --git a/qc_app/upload/__init__.py b/qc_app/upload/__init__.py deleted file mode 100644 index 5f120d4..0000000 --- a/qc_app/upload/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Package handling upload of files.""" -from flask import Blueprint - -from .rqtl2 import rqtl2 - -upload = Blueprint("upload", __name__) -upload.register_blueprint(rqtl2, url_prefix="/rqtl2") diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py deleted file mode 100644 index e691636..0000000 --- a/qc_app/upload/rqtl2.py +++ /dev/null @@ -1,1147 +0,0 @@ -"""Module to handle uploading of R/qtl2 bundles."""#pylint: disable=[too-many-lines] -import sys -import json -import traceback -from pathlib import Path -from datetime import date -from uuid import UUID, uuid4 -from functools import partial -from zipfile import ZipFile, is_zipfile -from typing import Union, Callable, Optional - -import MySQLdb as mdb -from redis import Redis -from MySQLdb.cursors import DictCursor -from werkzeug.utils import secure_filename -from flask import ( - flash, - escape, - request, - jsonify, - url_for, - redirect, - Response, - Blueprint, - render_template, - current_app as app) - -from r_qtl import r_qtl2 - -from qc_app import jobs -from qc_app.files import save_file, fullpath -from qc_app.dbinsert import species as all_species -from qc_app.db_utils import with_db_connection, database_connection - -from qc_app.db.platforms import platform_by_id, platforms_by_species -from qc_app.db.averaging import averaging_methods, averaging_method_by_id -from qc_app.db.tissues import all_tissues, tissue_by_id, create_new_tissue -from qc_app.db import ( - species_by_id, - save_population, - populations_by_species, - population_by_species_and_id,) -from qc_app.db.datasets import ( - geno_dataset_by_id, - geno_datasets_by_species_and_population, - - probeset_study_by_id, - probeset_create_study, - probeset_dataset_by_id, - probeset_create_dataset, - probeset_datasets_by_study, - probeset_studies_by_species_and_population) - -rqtl2 = Blueprint("rqtl2", __name__) - -@rqtl2.route("/", methods=["GET", "POST"]) -@rqtl2.route("/select-species", methods=["GET", "POST"]) -def select_species(): - """Select the species.""" - if request.method == "GET": - return render_template("rqtl2/index.html", species=with_db_connection(all_species)) - - species_id = request.form.get("species_id") - species = with_db_connection( - lambda conn: species_by_id(conn, species_id)) - if bool(species): - return redirect(url_for( - "upload.rqtl2.select_population", species_id=species_id)) - flash("Invalid species or no species selected!", "alert-error error-rqtl2") - return redirect(url_for("upload.rqtl2.select_species")) - - -@rqtl2.route("/upload/species/<int:species_id>/select-population", - methods=["GET", "POST"]) -def select_population(species_id: int): - """Select/Create the population to organise data under.""" - with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - if not bool(species): - flash("Invalid species selected!", "alert-error error-rqtl2") - return redirect(url_for("upload.rqtl2.select_species")) - - if request.method == "GET": - return render_template( - "rqtl2/select-population.html", - species=species, - populations=populations_by_species(conn, species_id)) - - population = population_by_species_and_id( - conn, species["SpeciesId"], request.form.get("inbredset_id")) - if not bool(population): - flash("Invalid Population!", "alert-error error-rqtl2") - return redirect( - url_for("upload.rqtl2.select_population", pgsrc="error"), - code=307) - - return redirect(url_for("upload.rqtl2.upload_rqtl2_bundle", - species_id=species["SpeciesId"], - population_id=population["InbredSetId"])) - - -@rqtl2.route("/upload/species/<int:species_id>/create-population", - methods=["POST"]) -def create_population(species_id: int): - """Create a new population for the given species.""" - population_page = redirect(url_for("upload.rqtl2.select_population", - species_id=species_id)) - with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - population_name = request.form.get("inbredset_name", "").strip() - population_fullname = request.form.get("inbredset_fullname", "").strip() - if not bool(species): - flash("Invalid species!", "alert-error error-rqtl2") - return redirect(url_for("upload.rqtl2.select_species")) - if not bool(population_name): - flash("Invalid Population Name!", "alert-error error-rqtl2") - return population_page - if not bool(population_fullname): - flash("Invalid Population Full Name!", "alert-error error-rqtl2") - return population_page - new_population = save_population(conn, { - "SpeciesId": species["SpeciesId"], - "Name": population_name, - "InbredSetName": population_fullname, - "FullName": population_fullname, - "Family": request.form.get("inbredset_family") or None, - "Description": request.form.get("description") or None - }) - - flash("Population created successfully.", "alert-success") - return redirect( - url_for("upload.rqtl2.upload_rqtl2_bundle", - species_id=species_id, - population_id=new_population["population_id"], - pgsrc="create-population"), - code=307) - - -class __RequestError__(Exception): #pylint: disable=[invalid-name] - """Internal class to avoid pylint's `too-many-return-statements` error.""" - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle"), - methods=["GET", "POST"]) -def upload_rqtl2_bundle(species_id: int, population_id: int): - """Allow upload of R/qtl2 bundle.""" - with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - population = population_by_species_and_id( - conn, species["SpeciesId"], population_id) - if not bool(species): - flash("Invalid species!", "alert-error error-rqtl2") - return redirect(url_for("upload.rqtl2.select_species")) - if not bool(population): - flash("Invalid Population!", "alert-error error-rqtl2") - return redirect( - url_for("upload.rqtl2.select_population", pgsrc="error"), - code=307) - if request.method == "GET" or ( - request.method == "POST" - and bool(request.args.get("pgsrc"))): - return render_template("rqtl2/upload-rqtl2-bundle-step-01.html", - species=species, - population=population) - - try: - app.logger.debug("Files in the form: %s", request.files) - the_file = save_file(request.files["rqtl2_bundle_file"], - Path(app.config["UPLOAD_FOLDER"])) - except AssertionError: - app.logger.debug(traceback.format_exc()) - flash("Please provide a valid R/qtl2 zip bundle.", - "alert-error error-rqtl2") - return redirect(url_for("upload.rqtl2.upload_rqtl2_bundle", - species_id=species_id, - population_id=population_id)) - - if not is_zipfile(str(the_file)): - app.logger.debug("The file is not a zip file.") - raise __RequestError__("Invalid file! Expected a zip file.") - - jobid = trigger_rqtl2_bundle_qc( - species_id, - population_id, - the_file, - request.files["rqtl2_bundle_file"].filename)#type: ignore[arg-type] - return redirect(url_for( - "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid)) - - -def trigger_rqtl2_bundle_qc( - species_id: int, - population_id: int, - rqtl2bundle: Path, - originalfilename: str -) -> UUID: - """Trigger QC on the R/qtl2 bundle.""" - redisuri = app.config["REDIS_URL"] - with Redis.from_url(redisuri, decode_responses=True) as rconn: - jobid = uuid4() - redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"] - jobs.launch_job( - jobs.initialise_job( - rconn, - jobs.jobsnamespace(), - str(jobid), - [sys.executable, "-m", "scripts.qc_on_rqtl2_bundle", - app.config["SQL_URI"], app.config["REDIS_URL"], - jobs.jobsnamespace(), str(jobid), str(species_id), - str(population_id), "--redisexpiry", - str(redis_ttl_seconds)], - "rqtl2-bundle-qc-job", - redis_ttl_seconds, - {"job-metadata": json.dumps({ - "speciesid": species_id, - "populationid": population_id, - "rqtl2-bundle-file": str(rqtl2bundle.absolute()), - "original-filename": originalfilename})}), - redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") - return jobid - - -def chunk_name(uploadfilename: str, chunkno: int) -> str: - """Generate chunk name from original filename and chunk number""" - if uploadfilename == "": - raise ValueError("Name cannot be empty!") - if chunkno < 1: - raise ValueError("Chunk number must be greater than zero") - return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}" - - -def chunks_directory(uniqueidentifier: str) -> Path: - """Compute the directory where chunks are temporarily stored.""" - if uniqueidentifier == "": - raise ValueError("Unique identifier cannot be empty!") - return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}") - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle-chunked"), - methods=["GET"]) -def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"] - species_id: int, - population_id: int -): - """ - Extension to the `upload_rqtl2_bundle` endpoint above that provides a way - for testing whether all the chunks have been uploaded and to assist with - resuming a failed upload. - """ - fileid = request.args.get("resumableIdentifier", type=str) or "" - filename = request.args.get("resumableFilename", type=str) or "" - chunk = request.args.get("resumableChunkNumber", type=int) or 0 - if not(fileid or filename or chunk): - return jsonify({ - "message": "At least one required query parameter is missing.", - "error": "BadRequest", - "statuscode": 400 - }), 400 - - if Path(chunks_directory(fileid), - chunk_name(filename, chunk)).exists(): - return "OK" - - return jsonify({ - "message": f"Chunk {chunk} was not found.", - "error": "NotFound", - "statuscode": 404 - }), 404 - - -def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path: - """Merge the chunks into a single file.""" - with open(targetfile, "ab") as _target: - for chunkfile in chunkpaths: - with open(chunkfile, "rb") as _chunkdata: - _target.write(_chunkdata.read()) - - chunkfile.unlink() - return targetfile - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle-chunked"), - methods=["POST"]) -def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int): - """ - Extension to the `upload_rqtl2_bundle` endpoint above that allows large - files to be uploaded in chunks. - - This should hopefully speed up uploads, and if done right, even enable - resumable uploads - """ - _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0 - _chunk = request.form.get("resumableChunkNumber", default=1, type=int) - _uploadfilename = request.form.get( - "resumableFilename", default="", type=str) or "" - _fileid = request.form.get( - "resumableIdentifier", default="", type=str) or "" - _targetfile = Path(app.config["UPLOAD_FOLDER"], _fileid) - - if _targetfile.exists(): - return jsonify({ - "message": ( - "A file with a similar unique identifier has previously been " - "uploaded and possibly is/has being/been processed."), - "error": "BadRequest", - "statuscode": 400 - }), 400 - - # save chunk data - chunks_directory(_fileid).mkdir(exist_ok=True) - request.files["file"].save(Path(chunks_directory(_fileid), - chunk_name(_uploadfilename, _chunk))) - - # Check whether upload is complete - chunkpaths = tuple( - Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk)) - for _achunk in range(1, _totalchunks+1)) - if all(_file.exists() for _file in chunkpaths): - # merge_files and clean up chunks - __merge_chunks__(_targetfile, chunkpaths) - chunks_directory(_fileid).rmdir() - jobid = trigger_rqtl2_bundle_qc( - species_id, population_id, _targetfile, _uploadfilename) - return url_for( - "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid) - - return "OK" - - -@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>", - methods=["GET", "POST"]) -def rqtl2_bundle_qc_status(jobid: UUID): - """Check the status of the QC jobs.""" - with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, - database_connection(app.config["SQL_URI"]) as dbconn): - try: - thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid) - messagelistname = thejob.get("log-messagelist") - logmessages = (rconn.lrange(messagelistname, 0, -1) - if bool(messagelistname) else []) - jobstatus = thejob["status"] - if jobstatus == "error": - return render_template("rqtl2/rqtl2-qc-job-error.html", - job=thejob, - errorsgeneric=json.loads( - thejob.get("errors-generic", "[]")), - errorsgeno=json.loads( - thejob.get("errors-geno", "[]")), - errorspheno=json.loads( - thejob.get("errors-pheno", "[]")), - errorsphenose=json.loads( - thejob.get("errors-phenose", "[]")), - errorsphenocovar=json.loads( - thejob.get("errors-phenocovar", "[]")), - messages=logmessages) - if jobstatus == "success": - jobmeta = json.loads(thejob["job-metadata"]) - species = species_by_id(dbconn, jobmeta["speciesid"]) - return render_template( - "rqtl2/rqtl2-qc-job-results.html", - species=species, - population=population_by_species_and_id( - dbconn, species["SpeciesId"], jobmeta["populationid"]), - rqtl2bundle=Path(jobmeta["rqtl2-bundle-file"]).name, - rqtl2bundleorig=jobmeta["original-filename"]) - - def compute_percentage(thejob, filetype) -> Union[str, None]: - if f"{filetype}-linecount" in thejob: - return "100" - if f"{filetype}-filesize" in thejob: - percent = ((int(thejob.get(f"{filetype}-checked", 0)) - / - int(thejob.get(f"{filetype}-filesize", 1))) - * 100) - return f"{percent:.2f}" - return None - - return render_template( - "rqtl2/rqtl2-qc-job-status.html", - job=thejob, - geno_percent=compute_percentage(thejob, "geno"), - pheno_percent=compute_percentage(thejob, "pheno"), - phenose_percent=compute_percentage(thejob, "phenose"), - messages=logmessages) - except jobs.JobNotFound: - return render_template("rqtl2/no-such-job.html", jobid=jobid) - - -def redirect_on_error(flaskroute, **kwargs): - """Utility to redirect on error""" - return redirect(url_for(flaskroute, **kwargs, pgsrc="error"), - code=(307 if request.method == "POST" else 302)) - - -def check_species(conn: mdb.Connection, formargs: dict) -> Optional[ - tuple[str, Response]]: - """ - Check whether the 'species_id' value is provided, and whether a - corresponding species exists in the database. - - Maybe give the function a better name...""" - speciespage = redirect_on_error("upload.rqtl2.select_species") - if "species_id" not in formargs: - return "You MUST provide the Species identifier.", speciespage - - if not bool(species_by_id(conn, formargs["species_id"])): - return "No species with the provided identifier exists.", speciespage - - return None - - -def check_population(conn: mdb.Connection, - formargs: dict, - species_id) -> Optional[tuple[str, Response]]: - """ - Check whether the 'population_id' value is provided, and whether a - corresponding population exists in the database. - - Maybe give the function a better name...""" - poppage = redirect_on_error( - "upload.rqtl2.select_species", species_id=species_id) - if "population_id" not in formargs: - return "You MUST provide the Population identifier.", poppage - - if not bool(population_by_species_and_id( - conn, species_id, formargs["population_id"])): - return "No population with the provided identifier exists.", poppage - - return None - - -def check_r_qtl2_bundle(formargs: dict, - species_id, - population_id) -> Optional[tuple[str, Response]]: - """Check for the existence of the R/qtl2 bundle.""" - fileuploadpage = redirect_on_error("upload.rqtl2.upload_rqtl2_bundle", - species_id=species_id, - population_id=population_id) - if not "rqtl2_bundle_file" in formargs: - return ( - "You MUST provide a R/qtl2 zip bundle for upload.", fileuploadpage) - - if not Path(fullpath(formargs["rqtl2_bundle_file"])).exists(): - return "No R/qtl2 bundle with the given name exists.", fileuploadpage - - return None - - -def check_geno_dataset(conn: mdb.Connection, - formargs: dict, - species_id, - population_id) -> Optional[tuple[str, Response]]: - """Check for the Genotype dataset.""" - genodsetpg = redirect_on_error("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id) - if not bool(formargs.get("geno-dataset-id")): - return ( - "You MUST provide a valid Genotype dataset identifier", genodsetpg) - - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM GenoFreeze WHERE Id=%s", - (formargs["geno-dataset-id"],)) - results = cursor.fetchall() - if not bool(results): - return ("No genotype dataset with the provided identifier exists.", - genodsetpg) - if len(results) > 1: - return ( - "Data corruption: More than one genotype dataset with the same " - "identifier.", - genodsetpg) - - return None - -def check_tissue( - conn: mdb.Connection,formargs: dict) -> Optional[tuple[str, Response]]: - """Check for tissue/organ/biological material.""" - selectdsetpg = redirect_on_error("upload.rqtl2.select_dataset_info", - species_id=formargs["species_id"], - population_id=formargs["population_id"]) - if not bool(formargs.get("tissueid", "").strip()): - return ("No tissue/organ/biological material provided.", selectdsetpg) - - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM Tissue WHERE Id=%s", - (formargs["tissueid"],)) - results = cursor.fetchall() - if not bool(results): - return ("No tissue/organ with the provided identifier exists.", - selectdsetpg) - - if len(results) > 1: - return ( - "Data corruption: More than one tissue/organ with the same " - "identifier.", - selectdsetpg) - - return None - - -def check_probe_study(conn: mdb.Connection, - formargs: dict, - species_id, - population_id) -> Optional[tuple[str, Response]]: - """Check for the ProbeSet study.""" - dsetinfopg = redirect_on_error("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id) - if not bool(formargs.get("probe-study-id")): - return "No probeset study was selected!", dsetinfopg - - if not bool(probeset_study_by_id(conn, formargs["probe-study-id"])): - return ("No probeset study with the provided identifier exists", - dsetinfopg) - - return None - - -def check_probe_dataset(conn: mdb.Connection, - formargs: dict, - species_id, - population_id) -> Optional[tuple[str, Response]]: - """Check for the ProbeSet dataset.""" - dsetinfopg = redirect_on_error("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id) - if not bool(formargs.get("probe-dataset-id")): - return "No probeset dataset was selected!", dsetinfopg - - if not bool(probeset_dataset_by_id(conn, formargs["probe-dataset-id"])): - return ("No probeset dataset with the provided identifier exists", - dsetinfopg) - - return None - - -def with_errors(endpointthunk: Callable, *checkfns): - """Run 'endpointthunk' with error checking.""" - formargs = {**dict(request.args), **dict(request.form)} - errors = tuple(item for item in (_fn(formargs=formargs) for _fn in checkfns) - if item is not None) - if len(errors) > 0: - flash(errors[0][0], "alert-error error-rqtl2") - return errors[0][1] - - return endpointthunk() - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle/select-geno-dataset"), - methods=["POST"]) -def select_geno_dataset(species_id: int, population_id: int): - """Select from existing geno datasets.""" - with database_connection(app.config["SQL_URI"]) as conn: - def __thunk__(): - geno_dset = geno_datasets_by_species_and_population( - conn, species_id, population_id) - if not bool(geno_dset): - flash("No genotype dataset was provided!", - "alert-error error-rqtl2") - return redirect(url_for("upload.rqtl2.select_geno_dataset", - species_id=species_id, - population_id=population_id, - pgsrc="error"), - code=307) - - flash("Genotype accepted", "alert-success error-rqtl2") - return redirect(url_for("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id, - pgsrc="upload.rqtl2.select_geno_dataset"), - code=307) - - return with_errors(__thunk__, - partial(check_species, conn=conn), - partial(check_population, conn=conn, - species_id=species_id), - partial(check_r_qtl2_bundle, - species_id=species_id, - population_id=population_id), - partial(check_geno_dataset, - conn=conn, - species_id=species_id, - population_id=population_id)) - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle/create-geno-dataset"), - methods=["POST"]) -def create_geno_dataset(species_id: int, population_id: int): - """Create a new geno dataset.""" - with database_connection(app.config["SQL_URI"]) as conn: - def __thunk__(): - sgeno_page = redirect(url_for("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id, - pgsrc="error"), - code=307) - errorclasses = "alert-error error-rqtl2 error-rqtl2-create-geno-dataset" - if not bool(request.form.get("dataset-name")): - flash("You must provide the dataset name", errorclasses) - return sgeno_page - if not bool(request.form.get("dataset-fullname")): - flash("You must provide the dataset full name", errorclasses) - return sgeno_page - public = 2 if request.form.get("dataset-public") == "on" else 0 - - with conn.cursor(cursorclass=DictCursor) as cursor: - datasetname = request.form["dataset-name"] - new_dataset = { - "name": datasetname, - "fname": request.form.get("dataset-fullname"), - "sname": request.form.get("dataset-shortname") or datasetname, - "today": date.today().isoformat(), - "pub": public, - "isetid": population_id - } - cursor.execute("SELECT * FROM GenoFreeze WHERE Name=%s", - (datasetname,)) - results = cursor.fetchall() - if bool(results): - flash( - f"A genotype dataset with name '{escape(datasetname)}' " - "already exists.", - errorclasses) - return redirect(url_for("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id, - pgsrc="error"), - code=307) - cursor.execute( - "INSERT INTO GenoFreeze(" - "Name, FullName, ShortName, CreateTime, public, InbredSetId" - ") " - "VALUES(" - "%(name)s, %(fname)s, %(sname)s, %(today)s, %(pub)s, %(isetid)s" - ")", - new_dataset) - flash("Created dataset successfully.", "alert-success") - return render_template( - "rqtl2/create-geno-dataset-success.html", - species=species_by_id(conn, species_id), - population=population_by_species_and_id( - conn, species_id, population_id), - rqtl2_bundle_file=request.form["rqtl2_bundle_file"], - geno_dataset={**new_dataset, "id": cursor.lastrowid}) - - return with_errors(__thunk__, - partial(check_species, conn=conn), - partial(check_population, conn=conn, species_id=species_id), - partial(check_r_qtl2_bundle, - species_id=species_id, - population_id=population_id)) - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle/select-tissue"), - methods=["POST"]) -def select_tissue(species_id: int, population_id: int): - """Select from existing tissues.""" - with database_connection(app.config["SQL_URI"]) as conn: - def __thunk__(): - if not bool(request.form.get("tissueid", "").strip()): - flash("Invalid tissue selection!", - "alert-error error-select-tissue error-rqtl2") - - return redirect(url_for("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id, - pgsrc="upload.rqtl2.select_geno_dataset"), - code=307) - - return with_errors(__thunk__, - partial(check_species, conn=conn), - partial(check_population, - conn=conn, - species_id=species_id), - partial(check_r_qtl2_bundle, - species_id=species_id, - population_id=population_id), - partial(check_geno_dataset, - conn=conn, - species_id=species_id, - population_id=population_id)) - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle/create-tissue"), - methods=["POST"]) -def create_tissue(species_id: int, population_id: int): - """Add new tissue, organ or biological material to the system.""" - form = request.form - datasetinfopage = redirect( - url_for("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id, - pgsrc="upload.rqtl2.select_geno_dataset"), - code=307) - with database_connection(app.config["SQL_URI"]) as conn: - tissuename = form.get("tissuename", "").strip() - tissueshortname = form.get("tissueshortname", "").strip() - if not bool(tissuename): - flash("Organ/Tissue name MUST be provided.", - "alert-error error-create-tissue error-rqtl2") - return datasetinfopage - - if not bool(tissueshortname): - flash("Organ/Tissue short name MUST be provided.", - "alert-error error-create-tissue error-rqtl2") - return datasetinfopage - - try: - tissue = create_new_tissue(conn, tissuename, tissueshortname) - flash("Tissue created successfully!", "alert-success") - return render_template( - "rqtl2/create-tissue-success.html", - species=species_by_id(conn, species_id), - population=population_by_species_and_id( - conn, species_id, population_id), - rqtl2_bundle_file=request.form["rqtl2_bundle_file"], - geno_dataset=geno_dataset_by_id( - conn, - int(request.form["geno-dataset-id"])), - tissue=tissue) - except mdb.IntegrityError as _ierr: - flash("Tissue/Organ with that short name already exists!", - "alert-error error-create-tissue error-rqtl2") - return datasetinfopage - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle/select-probeset-study"), - methods=["POST"]) -def select_probeset_study(species_id: int, population_id: int): - """Select or create a probeset study.""" - with database_connection(app.config["SQL_URI"]) as conn: - def __thunk__(): - summary_page = redirect(url_for("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id), - code=307) - if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))): - flash("Invalid study selected!", "alert-error error-rqtl2") - return summary_page - - return summary_page - return with_errors(__thunk__, - partial(check_species, conn=conn), - partial(check_population, - conn=conn, - species_id=species_id), - partial(check_r_qtl2_bundle, - species_id=species_id, - population_id=population_id), - partial(check_geno_dataset, - conn=conn, - species_id=species_id, - population_id=population_id), - partial(check_tissue, conn=conn), - partial(check_probe_study, - conn=conn, - species_id=species_id, - population_id=population_id)) - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle/select-probeset-dataset"), - methods=["POST"]) -def select_probeset_dataset(species_id: int, population_id: int): - """Select or create a probeset dataset.""" - with database_connection(app.config["SQL_URI"]) as conn: - def __thunk__(): - summary_page = redirect(url_for("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id), - code=307) - if not bool(probeset_study_by_id(conn, int(request.form["probe-study-id"]))): - flash("Invalid study selected!", "alert-error error-rqtl2") - return summary_page - - return summary_page - - return with_errors(__thunk__, - partial(check_species, conn=conn), - partial(check_population, - conn=conn, - species_id=species_id), - partial(check_r_qtl2_bundle, - species_id=species_id, - population_id=population_id), - partial(check_geno_dataset, - conn=conn, - species_id=species_id, - population_id=population_id), - partial(check_tissue, conn=conn), - partial(check_probe_study, - conn=conn, - species_id=species_id, - population_id=population_id), - partial(check_probe_dataset, - conn=conn, - species_id=species_id, - population_id=population_id)) - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle/create-probeset-study"), - methods=["POST"]) -def create_probeset_study(species_id: int, population_id: int): - """Create a new probeset study.""" - errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-study" - with database_connection(app.config["SQL_URI"]) as conn: - def __thunk__(): - form = request.form - dataset_info_page = redirect( - url_for("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id), - code=307) - - if not (bool(form.get("platformid")) and - bool(platform_by_id(conn, int(form["platformid"])))): - flash("Invalid platform selected.", errorclasses) - return dataset_info_page - - if not (bool(form.get("tissueid")) and - bool(tissue_by_id(conn, int(form["tissueid"])))): - flash("Invalid tissue selected.", errorclasses) - return dataset_info_page - - studyname = form["studyname"] - try: - study = probeset_create_study( - conn, population_id, int(form["platformid"]), int(form["tissueid"]), - studyname, form.get("studyfullname") or "", - form.get("studyshortname") or "") - except mdb.IntegrityError as _ierr: - flash(f"ProbeSet study with name '{escape(studyname)}' already " - "exists.", - errorclasses) - return dataset_info_page - return render_template( - "rqtl2/create-probe-study-success.html", - species=species_by_id(conn, species_id), - population=population_by_species_and_id( - conn, species_id, population_id), - rqtl2_bundle_file=request.form["rqtl2_bundle_file"], - geno_dataset=geno_dataset_by_id( - conn, - int(request.form["geno-dataset-id"])), - study=study) - - return with_errors(__thunk__, - partial(check_species, conn=conn), - partial(check_population, - conn=conn, - species_id=species_id), - partial(check_r_qtl2_bundle, - species_id=species_id, - population_id=population_id), - partial(check_geno_dataset, - conn=conn, - species_id=species_id, - population_id=population_id), - partial(check_tissue, conn=conn)) - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle/create-probeset-dataset"), - methods=["POST"]) -def create_probeset_dataset(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements] - """Create a new probeset dataset.""" - errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-dataset" - with database_connection(app.config["SQL_URI"]) as conn: - def __thunk__():#pylint: disable=[too-many-return-statements] - form = request.form - summary_page = redirect(url_for("upload.rqtl2.select_dataset_info", - species_id=species_id, - population_id=population_id), - code=307) - if not bool(form.get("averageid")): - flash("Averaging method not selected!", errorclasses) - return summary_page - if not bool(form.get("datasetname")): - flash("Dataset name not provided!", errorclasses) - return summary_page - if not bool(form.get("datasetfullname")): - flash("Dataset full name not provided!", errorclasses) - return summary_page - - tissue = tissue_by_id(conn, form.get("tissueid", "").strip()) - - study = probeset_study_by_id(conn, int(form["probe-study-id"])) - if not bool(study): - flash("Invalid ProbeSet study provided!", errorclasses) - return summary_page - - avgmethod = averaging_method_by_id(conn, int(form["averageid"])) - if not bool(avgmethod): - flash("Invalid averaging method provided!", errorclasses) - return summary_page - - try: - dset = probeset_create_dataset(conn, - int(form["probe-study-id"]), - int(form["averageid"]), - form["datasetname"], - form["datasetfullname"], - form["datasetshortname"], - form["datasetpublic"] == "on", - form.get( - "datasetdatascale", "log2")) - except mdb.IntegrityError as _ierr: - app.logger.debug("Possible integrity error: %s", traceback.format_exc()) - flash(("IntegrityError: The data you provided has some errors: " - f"{_ierr.args}"), - errorclasses) - return summary_page - except Exception as _exc:# pylint: disable=[broad-except] - app.logger.debug("Error creating ProbeSet dataset: %s", - traceback.format_exc()) - flash(("There was a problem creating your dataset. Please try " - "again."), - errorclasses) - return summary_page - return render_template( - "rqtl2/create-probe-dataset-success.html", - species=species_by_id(conn, species_id), - population=population_by_species_and_id( - conn, species_id, population_id), - rqtl2_bundle_file=request.form["rqtl2_bundle_file"], - geno_dataset=geno_dataset_by_id( - conn, - int(request.form["geno-dataset-id"])), - tissue=tissue, - study=study, - avgmethod=avgmethod, - dataset=dset) - - return with_errors(__thunk__, - partial(check_species, conn=conn), - partial(check_population, - conn=conn, - species_id=species_id), - partial(check_r_qtl2_bundle, - species_id=species_id, - population_id=population_id), - partial(check_geno_dataset, - conn=conn, - species_id=species_id, - population_id=population_id), - partial(check_tissue, conn=conn), - partial(check_probe_study, - conn=conn, - species_id=species_id, - population_id=population_id)) - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle/dataset-info"), - methods=["POST"]) -def select_dataset_info(species_id: int, population_id: int): - """ - If `geno` files exist in the R/qtl2 bundle, prompt user to provide the - dataset the genotypes belong to. - """ - form = request.form - with database_connection(app.config["SQL_URI"]) as conn: - def __thunk__(): - species = species_by_id(conn, species_id) - population = population_by_species_and_id( - conn, species_id, population_id) - thefile = fullpath(form["rqtl2_bundle_file"]) - with ZipFile(str(thefile), "r") as zfile: - cdata = r_qtl2.control_data(zfile) - - geno_dataset = geno_dataset_by_id( - conn,form.get("geno-dataset-id", "").strip()) - if "geno" in cdata and not bool(form.get("geno-dataset-id")): - return render_template( - "rqtl2/select-geno-dataset.html", - species=species, - population=population, - rqtl2_bundle_file=thefile.name, - datasets=geno_datasets_by_species_and_population( - conn, species_id, population_id)) - - tissue = tissue_by_id(conn, form.get("tissueid", "").strip()) - if "pheno" in cdata and not bool(tissue): - return render_template( - "rqtl2/select-tissue.html", - species=species, - population=population, - rqtl2_bundle_file=thefile.name, - geno_dataset=geno_dataset, - studies=probeset_studies_by_species_and_population( - conn, species_id, population_id), - platforms=platforms_by_species(conn, species_id), - tissues=all_tissues(conn)) - - probeset_study = probeset_study_by_id( - conn, form.get("probe-study-id", "").strip()) - if "pheno" in cdata and not bool(probeset_study): - return render_template( - "rqtl2/select-probeset-study-id.html", - species=species, - population=population, - rqtl2_bundle_file=thefile.name, - geno_dataset=geno_dataset, - studies=probeset_studies_by_species_and_population( - conn, species_id, population_id), - platforms=platforms_by_species(conn, species_id), - tissue=tissue) - probeset_study = probeset_study_by_id( - conn, int(form["probe-study-id"])) - - probeset_dataset = probeset_dataset_by_id( - conn, form.get("probe-dataset-id", "").strip()) - if "pheno" in cdata and not bool(probeset_dataset): - return render_template( - "rqtl2/select-probeset-dataset.html", - species=species, - population=population, - rqtl2_bundle_file=thefile.name, - geno_dataset=geno_dataset, - probe_study=probeset_study, - tissue=tissue, - datasets=probeset_datasets_by_study( - conn, int(form["probe-study-id"])), - avgmethods=averaging_methods(conn)) - - return render_template("rqtl2/summary-info.html", - species=species, - population=population, - rqtl2_bundle_file=thefile.name, - geno_dataset=geno_dataset, - tissue=tissue, - probe_study=probeset_study, - probe_dataset=probeset_dataset) - - return with_errors(__thunk__, - partial(check_species, conn=conn), - partial(check_population, - conn=conn, - species_id=species_id), - partial(check_r_qtl2_bundle, - species_id=species_id, - population_id=population_id)) - - -@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" - "/rqtl2-bundle/confirm-bundle-details"), - methods=["POST"]) -def confirm_bundle_details(species_id: int, population_id: int): - """Confirm the details and trigger R/qtl2 bundle processing...""" - redisuri = app.config["REDIS_URL"] - with (database_connection(app.config["SQL_URI"]) as conn, - Redis.from_url(redisuri, decode_responses=True) as rconn): - def __thunk__(): - redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"] - jobid = str(uuid4()) - _job = jobs.launch_job( - jobs.initialise_job( - rconn, - jobs.jobsnamespace(), - jobid, - [ - sys.executable, "-m", "scripts.process_rqtl2_bundle", - app.config["SQL_URI"], app.config["REDIS_URL"], - jobs.jobsnamespace(), jobid, "--redisexpiry", - str(redis_ttl_seconds)], - "R/qtl2 Bundle Upload", - redis_ttl_seconds, - { - "bundle-metadata": json.dumps({ - "speciesid": species_id, - "populationid": population_id, - "rqtl2-bundle-file": str(fullpath( - request.form["rqtl2_bundle_file"])), - "geno-dataset-id": request.form.get( - "geno-dataset-id", ""), - "probe-study-id": request.form.get( - "probe-study-id", ""), - "probe-dataset-id": request.form.get( - "probe-dataset-id", ""), - **({ - "platformid": probeset_study_by_id( - conn, - int(request.form["probe-study-id"]))["ChipId"] - } if bool(request.form.get("probe-study-id")) else {}) - }) - }), - redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") - - return redirect(url_for("upload.rqtl2.rqtl2_processing_status", - jobid=jobid)) - - return with_errors(__thunk__, - partial(check_species, conn=conn), - partial(check_population, - conn=conn, - species_id=species_id), - partial(check_r_qtl2_bundle, - species_id=species_id, - population_id=population_id), - partial(check_geno_dataset, - conn=conn, - species_id=species_id, - population_id=population_id), - partial(check_probe_study, - conn=conn, - species_id=species_id, - population_id=population_id), - partial(check_probe_dataset, - conn=conn, - species_id=species_id, - population_id=population_id)) - - -@rqtl2.route("/status/<uuid:jobid>") -def rqtl2_processing_status(jobid: UUID): - """Retrieve the status of the job processing the uploaded R/qtl2 bundle.""" - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: - try: - thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid) - - messagelistname = thejob.get("log-messagelist") - logmessages = (rconn.lrange(messagelistname, 0, -1) - if bool(messagelistname) else []) - - if thejob["status"] == "error": - return render_template( - "rqtl2/rqtl2-job-error.html", job=thejob, messages=logmessages) - if thejob["status"] == "success": - return render_template("rqtl2/rqtl2-job-results.html", - job=thejob, - messages=logmessages) - - return render_template( - "rqtl2/rqtl2-job-status.html", job=thejob, messages=logmessages) - except jobs.JobNotFound as _exc: - return render_template("rqtl2/no-such-job.html", jobid=jobid) |
