From 0b37b9b3fa4fead86787a705713645fa14530a54 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 23 Sep 2024 14:28:49 -0500 Subject: Initialise the "Expression Data" section. --- uploader/__init__.py | 3 - uploader/dbinsert.py | 406 --------------------- uploader/expression_data/__init__.py | 11 +- uploader/expression_data/dbinsert.py | 406 +++++++++++++++++++++ uploader/expression_data/index.py | 125 ------- uploader/expression_data/parse.py | 178 --------- uploader/expression_data/views.py | 384 +++++++++++++++++++ uploader/population/views.py | 2 + uploader/templates/data_review.html | 85 ----- uploader/templates/expression-data/base.html | 13 + .../templates/expression-data/data-review.html | 85 +++++ uploader/templates/expression-data/index.html | 82 +---- .../templates/expression-data/job-progress.html | 47 +++ .../templates/expression-data/no-such-job.html | 15 + .../templates/expression-data/parse-failure.html | 26 ++ .../templates/expression-data/parse-results.html | 39 ++ .../templates/expression-data/select-file.html | 115 ++++++ .../expression-data/select-population.html | 29 ++ uploader/templates/job_progress.html | 40 -- uploader/templates/no_such_job.html | 14 - uploader/templates/parse_failure.html | 26 -- uploader/templates/parse_results.html | 30 -- uploader/templates/select_species.html | 92 ----- 23 files changed, 1172 insertions(+), 1081 deletions(-) delete mode 100644 uploader/dbinsert.py create mode 100644 uploader/expression_data/dbinsert.py delete mode 100644 uploader/expression_data/index.py delete mode 100644 uploader/expression_data/parse.py create mode 100644 uploader/expression_data/views.py delete mode 100644 uploader/templates/data_review.html create mode 100644 uploader/templates/expression-data/base.html create mode 100644 uploader/templates/expression-data/data-review.html create mode 100644 uploader/templates/expression-data/job-progress.html create mode 100644 uploader/templates/expression-data/no-such-job.html create mode 100644 uploader/templates/expression-data/parse-failure.html create mode 100644 uploader/templates/expression-data/parse-results.html create mode 100644 uploader/templates/expression-data/select-file.html create mode 100644 uploader/templates/expression-data/select-population.html delete mode 100644 uploader/templates/job_progress.html delete mode 100644 uploader/templates/no_such_job.html delete mode 100644 uploader/templates/parse_failure.html delete mode 100644 uploader/templates/parse_results.html delete mode 100644 uploader/templates/select_species.html diff --git a/uploader/__init__.py b/uploader/__init__.py index 1af159b..9fdb383 100644 --- a/uploader/__init__.py +++ b/uploader/__init__.py @@ -12,7 +12,6 @@ from uploader.oauth2.client import user_logged_in, authserver_authorise_uri from . import session from .base_routes import base from .species import speciesbp -from .dbinsert import dbinsertbp from .oauth2.views import oauth2 from .expression_data import exprdatabp from .errors import register_error_handlers @@ -85,8 +84,6 @@ def create_app(): app.register_blueprint(base, url_prefix="/") app.register_blueprint(oauth2, url_prefix="/oauth2") app.register_blueprint(speciesbp, url_prefix="/species") - app.register_blueprint(dbinsertbp, url_prefix="/dbinsert") - app.register_blueprint(exprdatabp, url_prefix="/expression-data") register_error_handlers(app) return app diff --git a/uploader/dbinsert.py b/uploader/dbinsert.py deleted file mode 100644 index 2116031..0000000 --- a/uploader/dbinsert.py +++ /dev/null @@ -1,406 +0,0 @@ -"Handle inserting data into the database" -import os -import json -from typing import Union -from functools import reduce -from datetime import datetime - -from redis import Redis -from MySQLdb.cursors import DictCursor -from flask import ( - flash, request, url_for, Blueprint, redirect, render_template, - current_app as app) - -from uploader.authorisation import require_login -from uploader.population.models import populations_by_species -from uploader.db_utils import with_db_connection, database_connection -from uploader.species.models import species_by_id, all_species as species - -from . import jobs - -dbinsertbp = Blueprint("dbinsert", __name__) - -def render_error(error_msg): - "Render the generic error page" - return render_template("dbupdate_error.html", error_message=error_msg), 400 - -def make_menu_items_grouper(grouping_fn=lambda item: item): - "Build function to be used to group menu items." - def __grouper__(acc, row): - grouping = grouping_fn(row[2]) - row_values = (row[0].strip(), row[1].strip()) - if acc.get(grouping) is None: - return {**acc, grouping: (row_values,)} - return {**acc, grouping: (acc[grouping] + (row_values,))} - return __grouper__ - -def genechips(): - "Retrieve the genechip information from the database" - def __organise_by_species__(acc, chip): - speciesid = chip["SpeciesId"] - if acc.get(speciesid) is None: - return {**acc, speciesid: (chip,)} - return {**acc, speciesid: acc[speciesid] + (chip,)} - - with database_connection(app.config["SQL_URI"]) as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM GeneChip ORDER BY GeneChipName ASC") - return reduce(__organise_by_species__, cursor.fetchall(), {}) - - return {} - -def platform_by_id(genechipid:int) -> Union[dict, None]: - "Retrieve the gene platform by id" - with database_connection(app.config["SQL_URI"]) as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - "SELECT * FROM GeneChip WHERE GeneChipId=%s", - (genechipid,)) - return cursor.fetchone() - -def studies_by_species_and_platform(speciesid:int, genechipid:int) -> tuple: - "Retrieve the studies by the related species and gene platform" - with database_connection(app.config["SQL_URI"]) as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - query = ( - "SELECT Species.SpeciesId, ProbeFreeze.* " - "FROM Species INNER JOIN InbredSet " - "ON Species.SpeciesId=InbredSet.SpeciesId " - "INNER JOIN ProbeFreeze " - "ON InbredSet.InbredSetId=ProbeFreeze.InbredSetId " - "WHERE Species.SpeciesId = %s " - "AND ProbeFreeze.ChipId = %s") - cursor.execute(query, (speciesid, genechipid)) - return tuple(cursor.fetchall()) - - return tuple() - -def organise_groups_by_family(acc:dict, group:dict) -> dict: - "Organise the group (InbredSet) information by the group field" - family = group["Family"] - if acc.get(family): - return {**acc, family: acc[family] + (group,)} - return {**acc, family: (group,)} - -def tissues() -> tuple: - "Retrieve type (Tissue) information from the database." - with database_connection(app.config["SQL_URI"]) as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM Tissue ORDER BY Name") - return tuple(cursor.fetchall()) - - return tuple() - -@dbinsertbp.route("/platform", methods=["POST"]) -@require_login -def select_platform(): - "Select the platform (GeneChipId) used for the data." - job_id = request.form["job_id"] - with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, - database_connection(app.config["SQL_URI"]) as conn): - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - if job: - filename = job["filename"] - filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" - if os.path.exists(filepath): - default_species = 1 - gchips = genechips() - return render_template( - "select_platform.html", filename=filename, - filetype=job["filetype"], totallines=int(job["currentline"]), - default_species=default_species, species=species(conn), - genechips=gchips[default_species], - genechips_data=json.dumps(gchips)) - return render_error(f"File '{filename}' no longer exists.") - return render_error(f"Job '{job_id}' no longer exists.") - return render_error("Unknown error") - -@dbinsertbp.route("/study", methods=["POST"]) -@require_login -def select_study(): - "View to select/create the study (ProbeFreeze) associated with the data." - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - - speciesid = form["species"] - genechipid = form["genechipid"] - - the_studies = studies_by_species_and_platform(speciesid, genechipid) - the_groups = reduce( - organise_groups_by_family, - with_db_connection( - lambda conn: populations_by_species(conn, speciesid)), - {}) - return render_template( - "select_study.html", filename=form["filename"], - filetype=form["filetype"], totallines=form["totallines"], - species=speciesid, genechipid=genechipid, studies=the_studies, - groups=the_groups, tissues = tissues(), - selected_group=int(form.get("inbredsetid", -13)), - selected_tissue=int(form.get("tissueid", -13))) - except AssertionError as aserr: - return render_error(f"Missing data: {aserr.args[0]}") - -@dbinsertbp.route("/create-study", methods=["POST"]) -@require_login -def create_study(): - "Create a new study (ProbeFreeze)." - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - assert form.get("studyname"), "study name" - assert form.get("inbredsetid"), "group" - assert form.get("tissueid"), "type/tissue" - - with database_connection(app.config["SQL_URI"]) as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - values = ( - form["genechipid"], - form["tissueid"], - form["studyname"], - form.get("studyfullname", ""), - form.get("studyshortname", ""), - datetime.now().date().strftime("%Y-%m-%d"), - form["inbredsetid"]) - query = ( - "INSERT INTO ProbeFreeze(" - "ChipId, TissueId, Name, FullName, ShortName, CreateTime, " - "InbredSetId" - ") VALUES (%s, %s, %s, %s, %s, %s, %s)") - cursor.execute(query, values) - new_studyid = cursor.lastrowid - cursor.execute( - "UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s", - (new_studyid, new_studyid)) - flash("Study created successfully", "alert-success") - return render_template( - "continue_from_create_study.html", - filename=form["filename"], filetype=form["filetype"], - totallines=form["totallines"], species=form["species"], - genechipid=form["genechipid"], studyid=new_studyid) - except AssertionError as aserr: - flash(f"Missing data: {aserr.args[0]}", "alert-error") - return redirect(url_for("dbinsert.select_study"), code=307) - -def datasets_by_study(studyid:int) -> tuple: - "Retrieve datasets associated with a study with the ID `studyid`." - with database_connection(app.config["SQL_URI"]) as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - query = "SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s" - cursor.execute(query, (studyid,)) - return tuple(cursor.fetchall()) - - return tuple() - -def averaging_methods() -> tuple: - "Retrieve averaging methods from database" - with database_connection(app.config["SQL_URI"]) as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute("SELECT * FROM AvgMethod") - return tuple(cursor.fetchall()) - - return tuple() - -def dataset_datascales() -> tuple: - "Retrieve datascales from database" - with database_connection(app.config["SQL_URI"]) as conn: - with conn.cursor() as cursor: - cursor.execute( - 'SELECT DISTINCT DataScale FROM ProbeSetFreeze ' - 'WHERE DataScale IS NOT NULL AND DataScale != ""') - return tuple( - item for item in - (res[0].strip() for res in cursor.fetchall()) - if (item is not None and item != "")) - - return tuple() - -@dbinsertbp.route("/dataset", methods=["POST"]) -@require_login -def select_dataset(): - "Select the dataset to add the file contents against" - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - assert form.get("studyid"), "study" - - studyid = form["studyid"] - datasets = datasets_by_study(studyid) - return render_template( - "select_dataset.html", **{**form, "studyid": studyid}, - datasets=datasets, avgmethods=averaging_methods(), - datascales=dataset_datascales()) - except AssertionError as aserr: - return render_error(f"Missing data: {aserr.args[0]}") - -@dbinsertbp.route("/create-dataset", methods=["POST"]) -@require_login -def create_dataset(): - "Select the dataset to add the file contents against" - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - assert form.get("studyid"), "study" - assert form.get("avgid"), "averaging method" - assert form.get("datasetname2"), "Dataset Name 2" - assert form.get("datasetfullname"), "Dataset Full Name" - assert form.get("datasetshortname"), "Dataset Short Name" - assert form.get("datasetpublic"), "Dataset public specification" - assert form.get("datasetconfidentiality"), "Dataset confidentiality" - assert form.get("datasetdatascale"), "Dataset Datascale" - - with database_connection(app.config["SQL_URI"]) as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - datasetname = form["datasetname"] - cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Name=%s", - (datasetname,)) - results = cursor.fetchall() - if bool(results): - flash("A dataset with that name already exists.", - "alert-error") - return redirect(url_for("dbinsert.select_dataset"), code=307) - values = ( - form["studyid"], form["avgid"], - datasetname, form["datasetname2"], - form["datasetfullname"], form["datasetshortname"], - datetime.now().date().strftime("%Y-%m-%d"), - form["datasetpublic"], form["datasetconfidentiality"], - "williamslab", form["datasetdatascale"]) - query = ( - "INSERT INTO ProbeSetFreeze(" - "ProbeFreezeId, AvgID, Name, Name2, FullName, " - "ShortName, CreateTime, OrderList, public, " - "confidentiality, AuthorisedUsers, DataScale) " - "VALUES" - "(%s, %s, %s, %s, %s, %s, %s, NULL, %s, %s, %s, %s)") - cursor.execute(query, values) - new_datasetid = cursor.lastrowid - return render_template( - "continue_from_create_dataset.html", - filename=form["filename"], filetype=form["filetype"], - species=form["species"], genechipid=form["genechipid"], - studyid=form["studyid"], datasetid=new_datasetid, - totallines=form["totallines"]) - except AssertionError as aserr: - flash(f"Missing data {aserr.args[0]}", "alert-error") - return redirect(url_for("dbinsert.select_dataset"), code=307) - -def study_by_id(studyid:int) -> Union[dict, None]: - "Get a study by its Id" - with database_connection(app.config["SQL_URI"]) as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - "SELECT * FROM ProbeFreeze WHERE Id=%s", - (studyid,)) - return cursor.fetchone() - -def dataset_by_id(datasetid:int) -> Union[dict, None]: - "Retrieve a dataset by its id" - with database_connection(app.config["SQL_URI"]) as conn: - with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute( - ("SELECT AvgMethod.Name AS AvgMethodName, ProbeSetFreeze.* " - "FROM ProbeSetFreeze INNER JOIN AvgMethod " - "ON ProbeSetFreeze.AvgId=AvgMethod.AvgMethodId " - "WHERE ProbeSetFreeze.Id=%s"), - (datasetid,)) - return cursor.fetchone() - -def selected_keys(original: dict, keys: tuple) -> dict: - "Return a new dict from the `original` dict with only `keys` present." - return {key: value for key,value in original.items() if key in keys} - -@dbinsertbp.route("/final-confirmation", methods=["POST"]) -@require_login -def final_confirmation(): - "Preview the data before triggering entry into the database" - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - assert form.get("studyid"), "study" - assert form.get("datasetid"), "dataset" - - speciesid = form["species"] - genechipid = form["genechipid"] - studyid = form["studyid"] - datasetid=form["datasetid"] - return render_template( - "final_confirmation.html", filename=form["filename"], - filetype=form["filetype"], totallines=form["totallines"], - species=speciesid, genechipid=genechipid, studyid=studyid, - datasetid=datasetid, the_species=selected_keys( - with_db_connection(lambda conn: species_by_id(conn, speciesid)), - ("SpeciesName", "Name", "MenuName")), - platform=selected_keys( - platform_by_id(genechipid), - ("GeneChipName", "Name", "GeoPlatform", "Title", "GO_tree_value")), - study=selected_keys( - study_by_id(studyid), ("Name", "FullName", "ShortName")), - dataset=selected_keys( - dataset_by_id(datasetid), - ("AvgMethodName", "Name", "Name2", "FullName", "ShortName", - "DataScale"))) - except AssertionError as aserr: - return render_error(f"Missing data: {aserr.args[0]}") - -@dbinsertbp.route("/insert-data", methods=["POST"]) -@require_login -def insert_data(): - "Trigger data insertion" - form = request.form - try: - assert form.get("filename"), "filename" - assert form.get("filetype"), "filetype" - assert form.get("species"), "species" - assert form.get("genechipid"), "platform" - assert form.get("studyid"), "study" - assert form.get("datasetid"), "dataset" - - filename = form["filename"] - filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" - redisurl = app.config["REDIS_URL"] - if os.path.exists(filepath): - with Redis.from_url(redisurl, decode_responses=True) as rconn: - job = jobs.launch_job( - jobs.data_insertion_job( - rconn, filepath, form["filetype"], form["totallines"], - form["species"], form["genechipid"], form["datasetid"], - app.config["SQL_URI"], redisurl, - app.config["JOBS_TTL_SECONDS"]), - redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors") - - return redirect(url_for("dbinsert.insert_status", job_id=job["jobid"])) - return render_error(f"File '{filename}' no longer exists.") - except AssertionError as aserr: - return render_error(f"Missing data: {aserr.args[0]}") - -@dbinsertbp.route("/status/", methods=["GET"]) -def insert_status(job_id: str): - "Retrieve status of data insertion." - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - - if job: - job_status = job["status"] - if job_status == "success": - return render_template("insert_success.html", job=job) - if job["status"] == "error": - return render_template("insert_error.html", job=job) - return render_template("insert_progress.html", job=job) - return render_template("no_such_job.html", job_id=job_id), 400 diff --git a/uploader/expression_data/__init__.py b/uploader/expression_data/__init__.py index 206a764..fc8bd41 100644 --- a/uploader/expression_data/__init__.py +++ b/uploader/expression_data/__init__.py @@ -1,11 +1,2 @@ """Package handling upload of files.""" -from flask import Blueprint - -from .rqtl2 import rqtl2 -from .index import indexbp -from .parse import parsebp - -exprdatabp = Blueprint("expression-data", __name__) -exprdatabp.register_blueprint(indexbp, url_prefix="/") -exprdatabp.register_blueprint(rqtl2, url_prefix="/rqtl2") -exprdatabp.register_blueprint(parsebp, url_prefix="/parse") +from .views import exprdatabp diff --git a/uploader/expression_data/dbinsert.py b/uploader/expression_data/dbinsert.py new file mode 100644 index 0000000..2116031 --- /dev/null +++ b/uploader/expression_data/dbinsert.py @@ -0,0 +1,406 @@ +"Handle inserting data into the database" +import os +import json +from typing import Union +from functools import reduce +from datetime import datetime + +from redis import Redis +from MySQLdb.cursors import DictCursor +from flask import ( + flash, request, url_for, Blueprint, redirect, render_template, + current_app as app) + +from uploader.authorisation import require_login +from uploader.population.models import populations_by_species +from uploader.db_utils import with_db_connection, database_connection +from uploader.species.models import species_by_id, all_species as species + +from . import jobs + +dbinsertbp = Blueprint("dbinsert", __name__) + +def render_error(error_msg): + "Render the generic error page" + return render_template("dbupdate_error.html", error_message=error_msg), 400 + +def make_menu_items_grouper(grouping_fn=lambda item: item): + "Build function to be used to group menu items." + def __grouper__(acc, row): + grouping = grouping_fn(row[2]) + row_values = (row[0].strip(), row[1].strip()) + if acc.get(grouping) is None: + return {**acc, grouping: (row_values,)} + return {**acc, grouping: (acc[grouping] + (row_values,))} + return __grouper__ + +def genechips(): + "Retrieve the genechip information from the database" + def __organise_by_species__(acc, chip): + speciesid = chip["SpeciesId"] + if acc.get(speciesid) is None: + return {**acc, speciesid: (chip,)} + return {**acc, speciesid: acc[speciesid] + (chip,)} + + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM GeneChip ORDER BY GeneChipName ASC") + return reduce(__organise_by_species__, cursor.fetchall(), {}) + + return {} + +def platform_by_id(genechipid:int) -> Union[dict, None]: + "Retrieve the gene platform by id" + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT * FROM GeneChip WHERE GeneChipId=%s", + (genechipid,)) + return cursor.fetchone() + +def studies_by_species_and_platform(speciesid:int, genechipid:int) -> tuple: + "Retrieve the studies by the related species and gene platform" + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + query = ( + "SELECT Species.SpeciesId, ProbeFreeze.* " + "FROM Species INNER JOIN InbredSet " + "ON Species.SpeciesId=InbredSet.SpeciesId " + "INNER JOIN ProbeFreeze " + "ON InbredSet.InbredSetId=ProbeFreeze.InbredSetId " + "WHERE Species.SpeciesId = %s " + "AND ProbeFreeze.ChipId = %s") + cursor.execute(query, (speciesid, genechipid)) + return tuple(cursor.fetchall()) + + return tuple() + +def organise_groups_by_family(acc:dict, group:dict) -> dict: + "Organise the group (InbredSet) information by the group field" + family = group["Family"] + if acc.get(family): + return {**acc, family: acc[family] + (group,)} + return {**acc, family: (group,)} + +def tissues() -> tuple: + "Retrieve type (Tissue) information from the database." + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Tissue ORDER BY Name") + return tuple(cursor.fetchall()) + + return tuple() + +@dbinsertbp.route("/platform", methods=["POST"]) +@require_login +def select_platform(): + "Select the platform (GeneChipId) used for the data." + job_id = request.form["job_id"] + with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, + database_connection(app.config["SQL_URI"]) as conn): + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + if job: + filename = job["filename"] + filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + if os.path.exists(filepath): + default_species = 1 + gchips = genechips() + return render_template( + "select_platform.html", filename=filename, + filetype=job["filetype"], totallines=int(job["currentline"]), + default_species=default_species, species=species(conn), + genechips=gchips[default_species], + genechips_data=json.dumps(gchips)) + return render_error(f"File '{filename}' no longer exists.") + return render_error(f"Job '{job_id}' no longer exists.") + return render_error("Unknown error") + +@dbinsertbp.route("/study", methods=["POST"]) +@require_login +def select_study(): + "View to select/create the study (ProbeFreeze) associated with the data." + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + + speciesid = form["species"] + genechipid = form["genechipid"] + + the_studies = studies_by_species_and_platform(speciesid, genechipid) + the_groups = reduce( + organise_groups_by_family, + with_db_connection( + lambda conn: populations_by_species(conn, speciesid)), + {}) + return render_template( + "select_study.html", filename=form["filename"], + filetype=form["filetype"], totallines=form["totallines"], + species=speciesid, genechipid=genechipid, studies=the_studies, + groups=the_groups, tissues = tissues(), + selected_group=int(form.get("inbredsetid", -13)), + selected_tissue=int(form.get("tissueid", -13))) + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/create-study", methods=["POST"]) +@require_login +def create_study(): + "Create a new study (ProbeFreeze)." + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyname"), "study name" + assert form.get("inbredsetid"), "group" + assert form.get("tissueid"), "type/tissue" + + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + values = ( + form["genechipid"], + form["tissueid"], + form["studyname"], + form.get("studyfullname", ""), + form.get("studyshortname", ""), + datetime.now().date().strftime("%Y-%m-%d"), + form["inbredsetid"]) + query = ( + "INSERT INTO ProbeFreeze(" + "ChipId, TissueId, Name, FullName, ShortName, CreateTime, " + "InbredSetId" + ") VALUES (%s, %s, %s, %s, %s, %s, %s)") + cursor.execute(query, values) + new_studyid = cursor.lastrowid + cursor.execute( + "UPDATE ProbeFreeze SET ProbeFreezeId=%s WHERE Id=%s", + (new_studyid, new_studyid)) + flash("Study created successfully", "alert-success") + return render_template( + "continue_from_create_study.html", + filename=form["filename"], filetype=form["filetype"], + totallines=form["totallines"], species=form["species"], + genechipid=form["genechipid"], studyid=new_studyid) + except AssertionError as aserr: + flash(f"Missing data: {aserr.args[0]}", "alert-error") + return redirect(url_for("dbinsert.select_study"), code=307) + +def datasets_by_study(studyid:int) -> tuple: + "Retrieve datasets associated with a study with the ID `studyid`." + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + query = "SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s" + cursor.execute(query, (studyid,)) + return tuple(cursor.fetchall()) + + return tuple() + +def averaging_methods() -> tuple: + "Retrieve averaging methods from database" + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM AvgMethod") + return tuple(cursor.fetchall()) + + return tuple() + +def dataset_datascales() -> tuple: + "Retrieve datascales from database" + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor() as cursor: + cursor.execute( + 'SELECT DISTINCT DataScale FROM ProbeSetFreeze ' + 'WHERE DataScale IS NOT NULL AND DataScale != ""') + return tuple( + item for item in + (res[0].strip() for res in cursor.fetchall()) + if (item is not None and item != "")) + + return tuple() + +@dbinsertbp.route("/dataset", methods=["POST"]) +@require_login +def select_dataset(): + "Select the dataset to add the file contents against" + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + + studyid = form["studyid"] + datasets = datasets_by_study(studyid) + return render_template( + "select_dataset.html", **{**form, "studyid": studyid}, + datasets=datasets, avgmethods=averaging_methods(), + datascales=dataset_datascales()) + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/create-dataset", methods=["POST"]) +@require_login +def create_dataset(): + "Select the dataset to add the file contents against" + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + assert form.get("avgid"), "averaging method" + assert form.get("datasetname2"), "Dataset Name 2" + assert form.get("datasetfullname"), "Dataset Full Name" + assert form.get("datasetshortname"), "Dataset Short Name" + assert form.get("datasetpublic"), "Dataset public specification" + assert form.get("datasetconfidentiality"), "Dataset confidentiality" + assert form.get("datasetdatascale"), "Dataset Datascale" + + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + datasetname = form["datasetname"] + cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Name=%s", + (datasetname,)) + results = cursor.fetchall() + if bool(results): + flash("A dataset with that name already exists.", + "alert-error") + return redirect(url_for("dbinsert.select_dataset"), code=307) + values = ( + form["studyid"], form["avgid"], + datasetname, form["datasetname2"], + form["datasetfullname"], form["datasetshortname"], + datetime.now().date().strftime("%Y-%m-%d"), + form["datasetpublic"], form["datasetconfidentiality"], + "williamslab", form["datasetdatascale"]) + query = ( + "INSERT INTO ProbeSetFreeze(" + "ProbeFreezeId, AvgID, Name, Name2, FullName, " + "ShortName, CreateTime, OrderList, public, " + "confidentiality, AuthorisedUsers, DataScale) " + "VALUES" + "(%s, %s, %s, %s, %s, %s, %s, NULL, %s, %s, %s, %s)") + cursor.execute(query, values) + new_datasetid = cursor.lastrowid + return render_template( + "continue_from_create_dataset.html", + filename=form["filename"], filetype=form["filetype"], + species=form["species"], genechipid=form["genechipid"], + studyid=form["studyid"], datasetid=new_datasetid, + totallines=form["totallines"]) + except AssertionError as aserr: + flash(f"Missing data {aserr.args[0]}", "alert-error") + return redirect(url_for("dbinsert.select_dataset"), code=307) + +def study_by_id(studyid:int) -> Union[dict, None]: + "Get a study by its Id" + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT * FROM ProbeFreeze WHERE Id=%s", + (studyid,)) + return cursor.fetchone() + +def dataset_by_id(datasetid:int) -> Union[dict, None]: + "Retrieve a dataset by its id" + with database_connection(app.config["SQL_URI"]) as conn: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + ("SELECT AvgMethod.Name AS AvgMethodName, ProbeSetFreeze.* " + "FROM ProbeSetFreeze INNER JOIN AvgMethod " + "ON ProbeSetFreeze.AvgId=AvgMethod.AvgMethodId " + "WHERE ProbeSetFreeze.Id=%s"), + (datasetid,)) + return cursor.fetchone() + +def selected_keys(original: dict, keys: tuple) -> dict: + "Return a new dict from the `original` dict with only `keys` present." + return {key: value for key,value in original.items() if key in keys} + +@dbinsertbp.route("/final-confirmation", methods=["POST"]) +@require_login +def final_confirmation(): + "Preview the data before triggering entry into the database" + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + assert form.get("datasetid"), "dataset" + + speciesid = form["species"] + genechipid = form["genechipid"] + studyid = form["studyid"] + datasetid=form["datasetid"] + return render_template( + "final_confirmation.html", filename=form["filename"], + filetype=form["filetype"], totallines=form["totallines"], + species=speciesid, genechipid=genechipid, studyid=studyid, + datasetid=datasetid, the_species=selected_keys( + with_db_connection(lambda conn: species_by_id(conn, speciesid)), + ("SpeciesName", "Name", "MenuName")), + platform=selected_keys( + platform_by_id(genechipid), + ("GeneChipName", "Name", "GeoPlatform", "Title", "GO_tree_value")), + study=selected_keys( + study_by_id(studyid), ("Name", "FullName", "ShortName")), + dataset=selected_keys( + dataset_by_id(datasetid), + ("AvgMethodName", "Name", "Name2", "FullName", "ShortName", + "DataScale"))) + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/insert-data", methods=["POST"]) +@require_login +def insert_data(): + "Trigger data insertion" + form = request.form + try: + assert form.get("filename"), "filename" + assert form.get("filetype"), "filetype" + assert form.get("species"), "species" + assert form.get("genechipid"), "platform" + assert form.get("studyid"), "study" + assert form.get("datasetid"), "dataset" + + filename = form["filename"] + filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + redisurl = app.config["REDIS_URL"] + if os.path.exists(filepath): + with Redis.from_url(redisurl, decode_responses=True) as rconn: + job = jobs.launch_job( + jobs.data_insertion_job( + rconn, filepath, form["filetype"], form["totallines"], + form["species"], form["genechipid"], form["datasetid"], + app.config["SQL_URI"], redisurl, + app.config["JOBS_TTL_SECONDS"]), + redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors") + + return redirect(url_for("dbinsert.insert_status", job_id=job["jobid"])) + return render_error(f"File '{filename}' no longer exists.") + except AssertionError as aserr: + return render_error(f"Missing data: {aserr.args[0]}") + +@dbinsertbp.route("/status/", methods=["GET"]) +def insert_status(job_id: str): + "Retrieve status of data insertion." + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + job_status = job["status"] + if job_status == "success": + return render_template("insert_success.html", job=job) + if job["status"] == "error": + return render_template("insert_error.html", job=job) + return render_template("insert_progress.html", job=job) + return render_template("no_such_job.html", job_id=job_id), 400 diff --git a/uploader/expression_data/index.py b/uploader/expression_data/index.py deleted file mode 100644 index db23136..0000000 --- a/uploader/expression_data/index.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Entry-point module""" -import os -import mimetypes -from typing import Tuple -from zipfile import ZipFile, is_zipfile - -from werkzeug.utils import secure_filename -from flask import ( - flash, - request, - url_for, - redirect, - Blueprint, - render_template, - current_app as app) - -from uploader.species.models import all_species as species -from uploader.authorisation import require_login -from uploader.db_utils import with_db_connection - -indexbp = Blueprint("index", __name__) - - -def errors(rqst) -> Tuple[str, ...]: - """Return a tuple of the errors found in the request `rqst`. If no error is - found, then an empty tuple is returned.""" - def __filetype_error__(): - return ( - ("Invalid file type provided.",) - if rqst.form.get("filetype") not in ("average", "standard-error") - else tuple()) - - def __file_missing_error__(): - return ( - ("No file was uploaded.",) - if ("qc_text_file" not in rqst.files or - rqst.files["qc_text_file"].filename == "") - else tuple()) - - def __file_mimetype_error__(): - text_file = rqst.files["qc_text_file"] - return ( - ( - ("Invalid file! Expected a tab-separated-values file, or a zip " - "file of the a tab-separated-values file."),) - if text_file.mimetype not in ( - "text/plain", "text/tab-separated-values", - "application/zip") - else tuple()) - - return ( - __filetype_error__() + - (__file_missing_error__() or __file_mimetype_error__())) - -def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]: - """Check the uploaded zip file for errors.""" - zfile_errors: Tuple[str, ...] = tuple() - if is_zipfile(filepath): - with ZipFile(filepath, "r") as zfile: - infolist = zfile.infolist() - if len(infolist) != 1: - zfile_errors = zfile_errors + ( - ("Expected exactly one (1) member file within the uploaded zip " - f"file. Got {len(infolist)} member files."),) - if len(infolist) == 1 and infolist[0].is_dir(): - zfile_errors = zfile_errors + ( - ("Expected a member text file in the uploaded zip file. Got a " - "directory/folder."),) - - if len(infolist) == 1 and not infolist[0].is_dir(): - zfile.extract(infolist[0], path=upload_dir) - mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}") - if mime[0] != "text/tab-separated-values": - zfile_errors = zfile_errors + ( - ("Expected the member text file in the uploaded zip file to" - " be a tab-separated file."),) - - return zfile_errors - - -@indexbp.route("/", methods=["GET"]) -@require_login -def index(): - """Display the expression data index page.""" - return render_template("expression-data/index.html") - - -@indexbp.route("/upload", methods=["GET", "POST"]) -@require_login -def upload_file(): - """Enables uploading the files""" - if request.method == "GET": - return render_template( - "select_species.html", species=with_db_connection(species)) - - upload_dir = app.config["UPLOAD_FOLDER"] - request_errors = errors(request) - if request_errors: - for error in request_errors: - flash(error, "alert-danger error-expr-data") - return redirect(url_for("expression-data.index.upload_file")) - - filename = secure_filename(request.files["qc_text_file"].filename) - if not os.path.exists(upload_dir): - os.mkdir(upload_dir) - - filepath = os.path.join(upload_dir, filename) - request.files["qc_text_file"].save(os.path.join(upload_dir, filename)) - - zip_errors = zip_file_errors(filepath, upload_dir) - if zip_errors: - for error in zip_errors: - flash(error, "alert-danger error-expr-data") - return redirect(url_for("expression-data.index.upload_file")) - - return redirect(url_for("expression-data.parse.parse", - speciesid=request.form["speciesid"], - filename=filename, - filetype=request.form["filetype"])) - -@indexbp.route("/data-review", methods=["GET"]) -@require_login -def data_review(): - """Provide some help on data expectations to the user.""" - return render_template("data_review.html") diff --git a/uploader/expression_data/parse.py b/uploader/expression_data/parse.py deleted file mode 100644 index fc1c3f0..0000000 --- a/uploader/expression_data/parse.py +++ /dev/null @@ -1,178 +0,0 @@ -"""File parsing module""" -import os - -import jsonpickle -from redis import Redis -from flask import flash, request, url_for, redirect, Blueprint, render_template -from flask import current_app as app - -from quality_control.errors import InvalidValue, DuplicateHeading - -from uploader import jobs -from uploader.dbinsert import species_by_id -from uploader.db_utils import with_db_connection -from uploader.authorisation import require_login - -parsebp = Blueprint("parse", __name__) - -def isinvalidvalue(item): - """Check whether item is of type InvalidValue""" - return isinstance(item, InvalidValue) - -def isduplicateheading(item): - """Check whether item is of type DuplicateHeading""" - return isinstance(item, DuplicateHeading) - -@parsebp.route("/parse", methods=["GET"]) -@require_login -def parse(): - """Trigger file parsing""" - errors = False - speciesid = request.args.get("speciesid") - filename = request.args.get("filename") - filetype = request.args.get("filetype") - if speciesid is None: - flash("No species selected", "alert-error error-expr-data") - errors = True - else: - try: - speciesid = int(speciesid) - species = with_db_connection( - lambda con: species_by_id(con, speciesid)) - if not bool(species): - flash("No such species.", "alert-error error-expr-data") - errors = True - except ValueError: - flash("Invalid speciesid provided. Expected an integer.", - "alert-error error-expr-data") - errors = True - - if filename is None: - flash("No file provided", "alert-error error-expr-data") - errors = True - - if filetype is None: - flash("No filetype provided", "alert-error error-expr-data") - errors = True - - if filetype not in ("average", "standard-error"): - flash("Invalid filetype provided", "alert-error error-expr-data") - errors = True - - if filename: - filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename) - if not os.path.exists(filepath): - flash("Selected file does not exist (any longer)", - "alert-error error-expr-data") - errors = True - - if errors: - return redirect(url_for("expression-data.index.upload_file")) - - redisurl = app.config["REDIS_URL"] - with Redis.from_url(redisurl, decode_responses=True) as rconn: - job = jobs.launch_job( - jobs.build_file_verification_job( - rconn, app.config["SQL_URI"], redisurl, - speciesid, filepath, filetype, - app.config["JOBS_TTL_SECONDS"]), - redisurl, - f"{app.config['UPLOAD_FOLDER']}/job_errors") - - return redirect(url_for("expression-data.parse.parse_status", job_id=job["jobid"])) - -@parsebp.route("/status/", methods=["GET"]) -def parse_status(job_id: str): - "Retrieve the status of the job" - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: - try: - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - except jobs.JobNotFound as _exc: - return render_template("no_such_job.html", job_id=job_id), 400 - - error_filename = jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") - if os.path.exists(error_filename): - stat = os.stat(error_filename) - if stat.st_size > 0: - return redirect(url_for("parse.fail", job_id=job_id)) - - job_id = job["jobid"] - progress = float(job["percent"]) - status = job["status"] - filename = job.get("filename", "uploaded file") - errors = jsonpickle.decode( - job.get("errors", jsonpickle.encode(tuple()))) - if status in ("success", "aborted"): - return redirect(url_for("expression-data.parse.results", job_id=job_id)) - - if status == "parse-error": - return redirect(url_for("parse.fail", job_id=job_id)) - - app.jinja_env.globals.update( - isinvalidvalue=isinvalidvalue, - isduplicateheading=isduplicateheading) - return render_template( - "job_progress.html", - job_id = job_id, - job_status = status, - progress = progress, - message = job.get("message", ""), - job_name = f"Parsing '{filename}'", - errors=errors) - -@parsebp.route("/results/", methods=["GET"]) -def results(job_id: str): - """Show results of parsing...""" - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - - if job: - filename = job["filename"] - errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple()))) - app.jinja_env.globals.update( - isinvalidvalue=isinvalidvalue, - isduplicateheading=isduplicateheading) - return render_template( - "parse_results.html", - errors=errors, - job_name = f"Parsing '{filename}'", - user_aborted = job.get("user_aborted"), - job_id=job["jobid"]) - - return render_template("no_such_job.html", job_id=job_id) - -@parsebp.route("/fail/", methods=["GET"]) -def fail(job_id: str): - """Handle parsing failure""" - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - - if job: - error_filename = jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") - if os.path.exists(error_filename): - stat = os.stat(error_filename) - if stat.st_size > 0: - return render_template( - "worker_failure.html", job_id=job_id) - - return render_template("parse_failure.html", job=job) - - return render_template("no_such_job.html", job_id=job_id) - -@parsebp.route("/abort", methods=["POST"]) -@require_login -def abort(): - """Handle user request to abort file processing""" - job_id = request.form["job_id"] - - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: - job = jobs.job(rconn, jobs.jobsnamespace(), job_id) - - if job: - rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id), - key="user_aborted", - value=int(True)) - - return redirect(url_for("expression-data.parse.parse_status", job_id=job_id)) diff --git a/uploader/expression_data/views.py b/uploader/expression_data/views.py new file mode 100644 index 0000000..6900c51 --- /dev/null +++ b/uploader/expression_data/views.py @@ -0,0 +1,384 @@ +"""Views for expression data""" +import os +import uuid +import mimetypes +from typing import Tuple +from zipfile import ZipFile, is_zipfile + +import jsonpickle +from redis import Redis +from werkzeug.utils import secure_filename +from flask import (flash, + request, + url_for, + redirect, + Blueprint, + current_app as app) + +from quality_control.errors import InvalidValue, DuplicateHeading + +from uploader import jobs +from uploader.ui import make_template_renderer +from uploader.authorisation import require_login +from uploader.species.models import all_species, species_by_id +from uploader.db_utils import with_db_connection, database_connection +from uploader.datautils import safe_int, order_by_family, enumerate_sequence +from uploader.population.models import (populations_by_species, + population_by_species_and_id) + +exprdatabp = Blueprint("expression-data", __name__) +render_template = make_template_renderer("expression-data") + +def isinvalidvalue(item): + """Check whether item is of type InvalidValue""" + return isinstance(item, InvalidValue) + + +def isduplicateheading(item): + """Check whether item is of type DuplicateHeading""" + return isinstance(item, DuplicateHeading) + + +def errors(rqst) -> Tuple[str, ...]: + """Return a tuple of the errors found in the request `rqst`. If no error is + found, then an empty tuple is returned.""" + def __filetype_error__(): + return ( + ("Invalid file type provided.",) + if rqst.form.get("filetype") not in ("average", "standard-error") + else tuple()) + + def __file_missing_error__(): + return ( + ("No file was uploaded.",) + if ("qc_text_file" not in rqst.files or + rqst.files["qc_text_file"].filename == "") + else tuple()) + + def __file_mimetype_error__(): + text_file = rqst.files["qc_text_file"] + return ( + ( + ("Invalid file! Expected a tab-separated-values file, or a zip " + "file of the a tab-separated-values file."),) + if text_file.mimetype not in ( + "text/plain", "text/tab-separated-values", + "application/zip") + else tuple()) + + return ( + __filetype_error__() + + (__file_missing_error__() or __file_mimetype_error__())) + + +def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]: + """Check the uploaded zip file for errors.""" + zfile_errors: Tuple[str, ...] = tuple() + if is_zipfile(filepath): + with ZipFile(filepath, "r") as zfile: + infolist = zfile.infolist() + if len(infolist) != 1: + zfile_errors = zfile_errors + ( + ("Expected exactly one (1) member file within the uploaded zip " + f"file. Got {len(infolist)} member files."),) + if len(infolist) == 1 and infolist[0].is_dir(): + zfile_errors = zfile_errors + ( + ("Expected a member text file in the uploaded zip file. Got a " + "directory/folder."),) + + if len(infolist) == 1 and not infolist[0].is_dir(): + zfile.extract(infolist[0], path=upload_dir) + mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}") + if mime[0] != "text/tab-separated-values": + zfile_errors = zfile_errors + ( + ("Expected the member text file in the uploaded zip file to" + " be a tab-separated file."),) + + return zfile_errors + + +@exprdatabp.route("populations/expression-data", methods=["GET"]) +@require_login +def index(): + """Display the expression data index page.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(request.args.get("species_id")): + return render_template("expression-data/index.html", + species=order_by_family(all_species(conn)), + activelink="genotypes") + species = species_by_id(conn, request.args.get("species_id")) + if not bool(species): + flash(f"Could not find species with ID '{request.args.get('species_id')}'!", + "alert-danger") + return redirect(url_for("species.populations.expression-data.index")) + return redirect(url_for("species.populations.expression-data.select_population", + species_id=species["SpeciesId"])) + return render_template() + + +@exprdatabp.route("/populations/expression-data/select-population", + methods=["GET"]) +@require_login +def select_population(species_id: int): + """Select the expression data's population.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("Invalid species provided!", "alert-danger") + return redirect(url_for("species.populations.expression-data.index")) + + if not bool(request.args.get("population_id")): + return render_template("expression-data/select-population.html", + species=species, + populations=order_by_family( + populations_by_species(conn, species_id), + order_key="FamilyOrder"), + activelink="genotypes") + + population = population_by_species_and_id( + conn, species_id, request.args.get("population_id")) + if not bool(population): + flash("Invalid population selected!", "alert-danger") + return redirect(url_for( + "species.populations.expression-data.select_population", + species_id=species_id)) + + return redirect(url_for("species.populations.expression-data.upload_file", + species_id=species_id, + population_id=population["Id"])) + + +@exprdatabp.route("/populations//" + "expression-data/upload", + methods=["GET", "POST"]) +@require_login +def upload_file(species_id: int, population_id: int): + """Enables uploading the files""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + population = population_by_species_and_id(conn, species_id, population_id) + if request.method == "GET": + return render_template("expression-data/select-file.html", + species=species, + population=population) + + upload_dir = app.config["UPLOAD_FOLDER"] + request_errors = errors(request) + if request_errors: + for error in request_errors: + flash(error, "alert-danger error-expr-data") + return redirect(url_for("species.populations.expression-data.upload_file")) + + filename = secure_filename(request.files["qc_text_file"].filename) + if not os.path.exists(upload_dir): + os.mkdir(upload_dir) + + filepath = os.path.join(upload_dir, filename) + request.files["qc_text_file"].save(os.path.join(upload_dir, filename)) + + zip_errors = zip_file_errors(filepath, upload_dir) + if zip_errors: + for error in zip_errors: + flash(error, "alert-danger error-expr-data") + return redirect(url_for("species.populations.expression-data.index.upload_file")) + + return redirect(url_for("species.populations.expression-data.parse_file", + species_id=species_id, + population_id=population_id, + filename=filename, + filetype=request.form["filetype"])) + + +@exprdatabp.route("/data-review", methods=["GET"]) +@require_login +def data_review(): + """Provide some help on data expectations to the user.""" + return render_template("expression-data/data-review.html") + + +@exprdatabp.route( + "/populations//expression-data/parse", + methods=["GET"]) +@require_login +def parse_file(species_id: int, population_id: int): + """Trigger file parsing""" + errors = False + filename = request.args.get("filename") + filetype = request.args.get("filetype") + + species = with_db_connection(lambda con: species_by_id(con, species_id)) + if not bool(species): + flash("No such species.", "alert-danger") + errors = True + + if filename is None: + flash("No file provided", "alert-danger") + errors = True + + if filetype is None: + flash("No filetype provided", "alert-danger") + errors = True + + if filetype not in ("average", "standard-error"): + flash("Invalid filetype provided", "alert-danger") + errors = True + + if filename: + filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename) + if not os.path.exists(filepath): + flash("Selected file does not exist (any longer)", "alert-danger") + errors = True + + if errors: + return redirect(url_for("species.populations.expression-data.upload_file")) + + redisurl = app.config["REDIS_URL"] + with Redis.from_url(redisurl, decode_responses=True) as rconn: + job = jobs.launch_job( + jobs.build_file_verification_job( + rconn, app.config["SQL_URI"], redisurl, + species_id, filepath, filetype, + app.config["JOBS_TTL_SECONDS"]), + redisurl, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + + return redirect(url_for("species.populations.expression-data.parse_status", + species_id=species_id, + population_id=population_id, + job_id=job["jobid"])) + + +@exprdatabp.route( + "/populations//expression-data/parse/" + "status/", + methods=["GET"]) +@require_login +def parse_status(species_id: int, population_id: int, job_id: str): + "Retrieve the status of the job" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + except jobs.JobNotFound as _exc: + return render_template("no_such_job.html", job_id=job_id), 400 + + error_filename = jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + if os.path.exists(error_filename): + stat = os.stat(error_filename) + if stat.st_size > 0: + return redirect(url_for("parse.fail", job_id=job_id)) + + job_id = job["jobid"] + progress = float(job["percent"]) + status = job["status"] + filename = job.get("filename", "uploaded file") + errors = jsonpickle.decode( + job.get("errors", jsonpickle.encode(tuple()))) + if status in ("success", "aborted"): + return redirect(url_for("species.populations.expression-data.results", + species_id=species_id, + population_id=population_id, + job_id=job_id)) + + if status == "parse-error": + return redirect(url_for("species.populations.expression-data.fail", job_id=job_id)) + + app.jinja_env.globals.update( + isinvalidvalue=isinvalidvalue, + isduplicateheading=isduplicateheading) + return render_template( + "expression-data/job-progress.html", + job_id = job_id, + job_status = status, + progress = progress, + message = job.get("message", ""), + job_name = f"Parsing '{filename}'", + errors=errors, + species=with_db_connection( + lambda conn: species_by_id(conn, species_id)), + population=with_db_connection( + lambda conn: population_by_species_and_id( + conn, species_id, population_id))) + + +@exprdatabp.route( + "/populations//expression-data/parse/" + "/results", + methods=["GET"]) +@require_login +def results(species_id: int, population_id: int, job_id: uuid.UUID): + """Show results of parsing...""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + filename = job["filename"] + errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple()))) + app.jinja_env.globals.update( + isinvalidvalue=isinvalidvalue, + isduplicateheading=isduplicateheading) + return render_template( + "expression-data/parse-results.html", + errors=errors, + job_name = f"Parsing '{filename}'", + user_aborted = job.get("user_aborted"), + job_id=job["jobid"], + species=with_db_connection( + lambda conn: species_by_id(conn, species_id)), + population=with_db_connection( + lambda conn: population_by_species_and_id( + conn, species_id, population_id))) + + return render_template("expression-data/no-such-job.html", job_id=job_id) + + +@exprdatabp.route( + "/populations//expression-data/parse/" + "/fail", + methods=["GET"]) +@require_login +def fail(species_id: int, population_id: int, job_id: str): + """Handle parsing failure""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + error_filename = jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + if os.path.exists(error_filename): + stat = os.stat(error_filename) + if stat.st_size > 0: + return render_template( + "worker_failure.html", job_id=job_id) + + return render_template("parse_failure.html", job=job) + + return render_template("expression-data/no-such-job.html", + **with_db_connection(lambda conn: { + "species_id": species_by_id(conn, species_id), + "population_id": population_by_species_and_id( + conn, species_id, population_id)}), + job_id=job_id) + + +@exprdatabp.route( + "/populations//expression-data/parse/" + "abort", + methods=["POST"]) +@require_login +def abort(species_id: int, population_id: int): + """Handle user request to abort file processing""" + job_id = request.form["job_id"] + + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + job = jobs.job(rconn, jobs.jobsnamespace(), job_id) + + if job: + rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id), + key="user_aborted", + value=int(True)) + + return redirect(url_for("species.populations.expression-data.parse_status", + species_id=species_id, + population_id=population_id, + job_id=job_id)) diff --git a/uploader/population/views.py b/uploader/population/views.py index 39a5762..631f0be 100644 --- a/uploader/population/views.py +++ b/uploader/population/views.py @@ -18,6 +18,7 @@ from uploader.oauth2.client import oauth2_post from uploader.ui import make_template_renderer from uploader.authorisation import require_login from uploader.genotypes.views import genotypesbp +from uploader.expression_data.views import exprdatabp from uploader.db_utils import database_connection from uploader.datautils import enumerate_sequence from uploader.species.models import (all_species, @@ -34,6 +35,7 @@ __active_link__ = "populations" popbp = Blueprint("populations", __name__) popbp.register_blueprint(samplesbp, url_prefix="/") popbp.register_blueprint(genotypesbp, url_prefix="/") +popbp.register_blueprint(exprdatabp, url_prefix="/") render_template = make_template_renderer("populations") diff --git a/uploader/templates/data_review.html b/uploader/templates/data_review.html deleted file mode 100644 index 4e5c586..0000000 --- a/uploader/templates/data_review.html +++ /dev/null @@ -1,85 +0,0 @@ -{%extends "base.html"%} - -{%block title%}Data Review{%endblock%} - -{%block contents%} -

data review

- -
-

Data Concerns

-

The following are some of the requirements that the data in your file - MUST fulfil before it is considered valid for this system: -

- -
    -
  1. File headings -
      -
    • The first row in the file should contains the headings. The number of - headings in this first row determines the number of columns expected for - all other lines in the file.
      • -
    • Each heading value in the first row MUST appear in the first row - ONE AND ONLY ONE time
      • -
    • The sample/cases (previously 'strains') headers in your first row will be - against those in the - GeneNetwork database.
      - - If you encounter an error saying your sample(s)/case(s) do not exist - in the GeneNetwork database, then you will have to use the - Upload Samples/Cases - option on this system to upload them. - -
    -
    2. - -
  2. Data -
      -
    1. NONE of the data cells/fields is allowed to be empty. - All fields/cells MUST contain a value.
      2. -
    2. The first column of the data rows will be considered a textual field, - holding the "identifier" for that row
    3. -
    4. Except for the first column/field for each data row, - NONE of the data columns/cells/fields should contain - spurious characters like `eeeee`, `5.555iloveguix`, etc...
      - All of them should be decimal values
      5. -
    5. decimal numbers must conform to the following criteria: -
        -
      • when checking an average file decimal numbers must have exactly three - decimal places to the right of the decimal point.
        • -
      • when checking a standard error file decimal numbers must have six or - greater decimal places to the right of the decimal point.
        • -
      • there must be a number to the left side of the decimal place - (e.g. 0.55555 is allowed but .55555 is not).
        • -
      -
      6. -
    -
    3. -
-
- - -
-

Supported File Types

- We support the following file types: - -
    -
  • Tab-Separated value files (.tsv) -
      -
    • The TAB character is used to separate the fields of each - column
      • -
    • The values of each field ARE NOT quoted.
      • -
    • Here is an - - example file with a single data row.
      • -
    -
    • -
  • .txt files: Content has the same format as .tsv file above
    • -
  • .zip files: each zip file should contain - ONE AND ONLY ONE file of the .tsv or .txt type above. -
    Any zip file with more than one file is invalid, and so is an empty - zip file.
    • -
- -
-{%endblock%} diff --git a/uploader/templates/expression-data/base.html b/uploader/templates/expression-data/base.html new file mode 100644 index 0000000..d63fd7e --- /dev/null +++ b/uploader/templates/expression-data/base.html @@ -0,0 +1,13 @@ +{%extends "populations/base.html"%} + +{%block lvl3_breadcrumbs%} +
  • + + Expression Data +
    • +{%block lvl4_breadcrumbs%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/expression-data/data-review.html b/uploader/templates/expression-data/data-review.html new file mode 100644 index 0000000..c985b03 --- /dev/null +++ b/uploader/templates/expression-data/data-review.html @@ -0,0 +1,85 @@ +{%extends "base.html"%} + +{%block title%}Data Review{%endblock%} + +{%block contents%} +

    data review

    + +
    +

    Data Concerns

    +

    The following are some of the requirements that the data in your file + MUST fulfil before it is considered valid for this system: +

    + +
      +
    1. File headings +
        +
      • The first row in the file should contains the headings. The number of + headings in this first row determines the number of columns expected for + all other lines in the file.
        • +
      • Each heading value in the first row MUST appear in the first row + ONE AND ONLY ONE time
        • +
      • The sample/cases (previously 'strains') headers in your first row will be + against those in the + GeneNetwork database.
        + + If you encounter an error saying your sample(s)/case(s) do not exist + in the GeneNetwork database, then you will have to use the + Upload Samples/Cases + option on this system to upload them. + +
      +
      2. + +
    2. Data +
        +
      1. NONE of the data cells/fields is allowed to be empty. + All fields/cells MUST contain a value.
        2. +
      2. The first column of the data rows will be considered a textual field, + holding the "identifier" for that row
      3. +
      4. Except for the first column/field for each data row, + NONE of the data columns/cells/fields should contain + spurious characters like `eeeee`, `5.555iloveguix`, etc...
        + All of them should be decimal values
        5. +
      5. decimal numbers must conform to the following criteria: +
          +
        • when checking an average file decimal numbers must have exactly three + decimal places to the right of the decimal point.
          • +
        • when checking a standard error file decimal numbers must have six or + greater decimal places to the right of the decimal point.
          • +
        • there must be a number to the left side of the decimal place + (e.g. 0.55555 is allowed but .55555 is not).
          • +
        +
        6. +
      +
      3. +
    +
    + + +
    +

    Supported File Types

    + We support the following file types: + +
      +
    • Tab-Separated value files (.tsv) +
        +
      • The TAB character is used to separate the fields of each + column
        • +
      • The values of each field ARE NOT quoted.
        • +
      • Here is an + example file with a single data row.
        • +
      +
      • +
    • .txt files: Content has the same format as .tsv file above
      • +
    • .zip files: each zip file should contain + ONE AND ONLY ONE file of the .tsv or .txt type above. +
      Any zip file with more than one file is invalid, and so is an empty + zip file.
      • +
    + +
    +{%endblock%} diff --git a/uploader/templates/expression-data/index.html b/uploader/templates/expression-data/index.html index ed5d8dd..9ba3582 100644 --- a/uploader/templates/expression-data/index.html +++ b/uploader/templates/expression-data/index.html @@ -1,5 +1,6 @@ -{%extends "base.html"%} +{%extends "expression-data/base.html"%} {%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-select-species.html" import select_species_form%} {%block title%}Expression Data{%endblock%} @@ -10,86 +11,23 @@ Home
  • - Expression Data + + Expression Data
    • {%endblock%} {%block contents%} -
    - {{flash_all_messages()}} - -

    data upload

    - -
    -

    Each of the sections below gives you a different option for data expression-data. - Please read the documentation for each section carefully to understand what - each section is about.

    -
    -
    - -
    -

    R/qtl2 Bundles

    - -
    -

    This feature combines and extends the two upload methods below. Instead of - uploading one item at a time, the R/qtl2 bundle you upload can contain both - the genotypes data (samples/individuals/cases and their data) and the - expression data.

    -

    The R/qtl2 bundle, additionally, can contain extra metadata, that neither - of the methods below can handle.

    - - - -
    -
    - -

    Expression Data

    + {{flash_all_messages()}} -
    -

    This feature enables you to upload expression data. It expects the data to - be in tab-separated values (TSV) files. The data should be - a simple matrix of phenotype × sample, i.e. The first column is a - list of the phenotypes and the first row is a list of - samples/cases.

    - -

    If you haven't done so please go to this page to learn the requirements for - file formats and helpful suggestions to enter your data in a fast and easy - way.

    - -
      -
    1. PLEASE REVIEW YOUR DATA.Make sure your data complies - with our system requirements. ( - Help - )
      2. -
    2. UPLOAD YOUR DATA FOR DATA VERIFICATION. We accept - .csv, .txt and .zip - files (Help)
      3. -
    -
    - - upload expression data +

    This section allows you to enter the expression data for your experiment. + You will need to select the species that your data concerns below.

    -

    samples/cases

    - -
    -

    For the expression data above, you need the samples/cases in your file to - already exist in the GeneNetwork database. If there are any samples that do - not already exist the upload of the expression data will fail.

    -

    This section gives you the opportunity to upload any missing samples

    -
    - - upload Samples/Cases + {{select_species_form(url_for("species.populations.expression-data.index"), + species)}}
    - {%endblock%} diff --git a/uploader/templates/expression-data/job-progress.html b/uploader/templates/expression-data/job-progress.html new file mode 100644 index 0000000..ef264e1 --- /dev/null +++ b/uploader/templates/expression-data/job-progress.html @@ -0,0 +1,47 @@ +{%extends "base.html"%} +{%from "errors_display.html" import errors_display%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block extrameta%} + +{%endblock%} + +{%block title%}Job Status{%endblock%} + +{%block contents%} +

    {{job_name}}

    + +
    +
    + Status +
    + + {{job_status}}: {{message}}
    +
    + +
    + + + {{progress}} + {{"%.2f" | format(progress)}}% +
    + + + + +
    +
    + +
    + {{errors_display(errors, "No errors found so far", "We have found the following errors so far", False)}} +
    + +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/expression-data/no-such-job.html b/uploader/templates/expression-data/no-such-job.html new file mode 100644 index 0000000..d22c429 --- /dev/null +++ b/uploader/templates/expression-data/no-such-job.html @@ -0,0 +1,15 @@ +{%extends "base.html"%} + +{%block extrameta%} + +{%endblock%} + +{%block title%}No Such Job{%endblock%} + +{%block contents%} +

    No Such Job: {{job_id}}

    + +

    No job, with the id '{{job_id}}' was found!

    + +{%endblock%} diff --git a/uploader/templates/expression-data/parse-failure.html b/uploader/templates/expression-data/parse-failure.html new file mode 100644 index 0000000..31f6be8 --- /dev/null +++ b/uploader/templates/expression-data/parse-failure.html @@ -0,0 +1,26 @@ +{%extends "base.html"%} + +{%block title%}Worker Failure{%endblock%} + +{%block contents%} +

    Worker Failure

    + +

    + There was an error while parsing your file. +

    + +

    + Please notify the developers of this issue when you encounter it, + providing the information below. +

    + +

    Debugging Information

    + +
      +
    • job id: {{job["job_id"]}}
      • +
    • filename: {{job["filename"]}}
      • +
    • line number: {{job["line_number"]}}
      • +
    • Progress: {{job["percent"]}} %
      • +
    + +{%endblock%} diff --git a/uploader/templates/expression-data/parse-results.html b/uploader/templates/expression-data/parse-results.html new file mode 100644 index 0000000..03a23e2 --- /dev/null +++ b/uploader/templates/expression-data/parse-results.html @@ -0,0 +1,39 @@ +{%extends "base.html"%} +{%from "errors_display.html" import errors_display%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Parse Results{%endblock%} + +{%block contents%} + +
    +

    {{job_name}}: parse results

    + + {%if user_aborted%} + Job aborted by the user + {%endif%} + + {{errors_display(errors, "No errors found in the file", "We found the following errors", True)}} + + {%if errors | length == 0 and not user_aborted %} +
    + + +
    + {%endif%} + + {%if errors | length > 0 or user_aborted %} +
    + Go back + + {%endif%} +
    +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/expression-data/select-file.html b/uploader/templates/expression-data/select-file.html new file mode 100644 index 0000000..4ca461e --- /dev/null +++ b/uploader/templates/expression-data/select-file.html @@ -0,0 +1,115 @@ +{%extends "expression-data/base.html"%} +{%from "flash_messages.html" import flash_messages%} +{%from "upload_progress_indicator.html" import upload_progress_indicator%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Expression Data — Upload Data{%endblock%} + +{%block pagetitle%}Expression Data — Upload Data{%endblock%} + +{%block contents%} +{{upload_progress_indicator()}} + +
    +

    Upload Expression Data

    + +

    This feature enables you to upload expression data. It expects the data to + be in tab-separated values (TSV) files. The data should be + a simple matrix of phenotype × sample, i.e. The first column is a + list of the phenotypes and the first row is a list of + samples/cases.

    + +

    If you haven't done so please go to this page to learn the requirements for + file formats and helpful suggestions to enter your data in a fast and easy + way.

    + +
      +
    1. PLEASE REVIEW YOUR DATA.Make sure your data complies + with our system requirements. ( + Help + )
      2. +
    2. UPLOAD YOUR DATA FOR DATA VERIFICATION. We accept + .csv, .txt and .zip + files (Help)
      3. +
    +
    + +
    +
    + {{flash_messages("error-expr-data")}} + +
    + File Type + +
    + +

    + The averages data …

    +
    + +
    + +

    + The standard errors computed from the averages …

    +
    +
    + +
    + + + +

    + Select the file to upload.

    +
    + + +
    +
    +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} + +{%block javascript%} + + +{%endblock%} diff --git a/uploader/templates/expression-data/select-population.html b/uploader/templates/expression-data/select-population.html new file mode 100644 index 0000000..8555e27 --- /dev/null +++ b/uploader/templates/expression-data/select-population.html @@ -0,0 +1,29 @@ +{%extends "expression-data/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "species/macro-display-species-card.html" import display_species_card%} +{%from "populations/macro-select-population.html" import select_population_form%} + +{%block title%}Expression Data{%endblock%} + +{%block pagetitle%}Expression Data{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +
    +

    You have selected the species. Now you need to select the population that + the expression data belongs to.

    +
    + +
    + {{select_population_form(url_for( + "species.populations.expression-data.select_population", + species_id=species.SpeciesId), + populations)}} +
    +{%endblock%} + +{%block sidebarcontents%} +{{display_species_card(species)}} +{%endblock%} diff --git a/uploader/templates/job_progress.html b/uploader/templates/job_progress.html deleted file mode 100644 index 2feaa89..0000000 --- a/uploader/templates/job_progress.html +++ /dev/null @@ -1,40 +0,0 @@ -{%extends "base.html"%} -{%from "errors_display.html" import errors_display%} - -{%block extrameta%} - -{%endblock%} - -{%block title%}Job Status{%endblock%} - -{%block contents%} -

    {{job_name}}

    - -
    -
    - Status -
    - - {{job_status}}: {{message}}
    -
    - -
    - - - {{progress}} - {{"%.2f" | format(progress)}}% -
    - - - - -
    -
    - -
    - {{errors_display(errors, "No errors found so far", "We have found the following errors so far", False)}} -
    - -{%endblock%} diff --git a/uploader/templates/no_such_job.html b/uploader/templates/no_such_job.html deleted file mode 100644 index 874d047..0000000 --- a/uploader/templates/no_such_job.html +++ /dev/null @@ -1,14 +0,0 @@ -{%extends "base.html"%} - -{%block extrameta%} - -{%endblock%} - -{%block title%}No Such Job{%endblock%} - -{%block contents%} -

    No Such Job: {{job_id}}

    - -

    No job, with the id '{{job_id}}' was found!

    - -{%endblock%} diff --git a/uploader/templates/parse_failure.html b/uploader/templates/parse_failure.html deleted file mode 100644 index 31f6be8..0000000 --- a/uploader/templates/parse_failure.html +++ /dev/null @@ -1,26 +0,0 @@ -{%extends "base.html"%} - -{%block title%}Worker Failure{%endblock%} - -{%block contents%} -

    Worker Failure

    - -

    - There was an error while parsing your file. -

    - -

    - Please notify the developers of this issue when you encounter it, - providing the information below. -

    - -

    Debugging Information

    - -
      -
    • job id: {{job["job_id"]}}
      • -
    • filename: {{job["filename"]}}
      • -
    • line number: {{job["line_number"]}}
      • -
    • Progress: {{job["percent"]}} %
      • -
    - -{%endblock%} diff --git a/uploader/templates/parse_results.html b/uploader/templates/parse_results.html deleted file mode 100644 index 46fbaaf..0000000 --- a/uploader/templates/parse_results.html +++ /dev/null @@ -1,30 +0,0 @@ -{%extends "base.html"%} -{%from "errors_display.html" import errors_display%} - -{%block title%}Parse Results{%endblock%} - -{%block contents%} -

    {{job_name}}: parse results

    - -{%if user_aborted%} -Job aborted by the user -{%endif%} - -{{errors_display(errors, "No errors found in the file", "We found the following errors", True)}} - -{%if errors | length == 0 and not user_aborted %} -
    - - -
    -{%endif%} - -{%if errors | length > 0 or user_aborted %} -
    - - Go back - -{%endif%} - -{%endblock%} diff --git a/uploader/templates/select_species.html b/uploader/templates/select_species.html deleted file mode 100644 index 1642401..0000000 --- a/uploader/templates/select_species.html +++ /dev/null @@ -1,92 +0,0 @@ -{%extends "base.html"%} -{%from "flash_messages.html" import flash_messages%} -{%from "upload_progress_indicator.html" import upload_progress_indicator%} - -{%block title%}expression data: select species{%endblock%} - -{%block contents%} -{{upload_progress_indicator()}} - -

    expression data: select species

    - -
    -
    - upload expression data - {{flash_messages("error-expr-data")}} - -
    - - -
    - -
    - file type - -
    - - -
    - -
    - - -
    -
    - -
    - - - -
    - - -
    -
    -{%endblock%} - - -{%block javascript%} - - -{%endblock%} -- cgit v1.2.3