Initialise the "Expression Data" section.

18 files changed, 606 insertions, 515 deletions

diff --git a/uploader/__init__.py b/uploader/__init__.py
index 1af159b..9fdb383 100644
--- a/uploader/__init__.py
+++ b/uploader/__init__.py
@@ -12,7 +12,6 @@ from uploader.oauth2.client import user_logged_in, authserver_authorise_uri
 from . import session
 from .base_routes import base
 from .species import speciesbp
-from .dbinsert import dbinsertbp
 from .oauth2.views import oauth2
 from .expression_data import exprdatabp
 from .errors import register_error_handlers
@@ -85,8 +84,6 @@ def create_app():
     app.register_blueprint(base, url_prefix="/")
     app.register_blueprint(oauth2, url_prefix="/oauth2")
     app.register_blueprint(speciesbp, url_prefix="/species")
-    app.register_blueprint(dbinsertbp, url_prefix="/dbinsert")
-    app.register_blueprint(exprdatabp, url_prefix="/expression-data")
 
     register_error_handlers(app)
     return app
diff --git a/uploader/expression_data/__init__.py b/uploader/expression_data/__init__.py
index 206a764..fc8bd41 100644
--- a/uploader/expression_data/__init__.py
+++ b/uploader/expression_data/__init__.py
@@ -1,11 +1,2 @@
 """Package handling upload of files."""
-from flask import Blueprint
-
-from .rqtl2 import rqtl2
-from .index import indexbp
-from .parse import parsebp
-
-exprdatabp = Blueprint("expression-data", __name__)
-exprdatabp.register_blueprint(indexbp, url_prefix="/")
-exprdatabp.register_blueprint(rqtl2, url_prefix="/rqtl2")
-exprdatabp.register_blueprint(parsebp, url_prefix="/parse")
+from .views import exprdatabp
diff --git a/uploader/dbinsert.py b/uploader/expression_data/dbinsert.py
index 2116031..2116031 100644
--- a/uploader/dbinsert.py
+++ b/uploader/expression_data/dbinsert.py
diff --git a/uploader/expression_data/index.py b/uploader/expression_data/index.py
deleted file mode 100644
index db23136..0000000
--- a/uploader/expression_data/index.py
+++ /dev/null
@@ -1,125 +0,0 @@
-"""Entry-point module"""
-import os
-import mimetypes
-from typing import Tuple
-from zipfile import ZipFile, is_zipfile
-
-from werkzeug.utils import secure_filename
-from flask import (
-    flash,
-    request,
-    url_for,
-    redirect,
-    Blueprint,
-    render_template,
-    current_app as app)
-
-from uploader.species.models import all_species as species
-from uploader.authorisation import require_login
-from uploader.db_utils import with_db_connection
-
-indexbp = Blueprint("index", __name__)
-
-
-def errors(rqst) -> Tuple[str, ...]:
-    """Return a tuple of the errors found in the request `rqst`. If no error is
-    found, then an empty tuple is returned."""
-    def __filetype_error__():
-        return (
-            ("Invalid file type provided.",)
-            if rqst.form.get("filetype") not in ("average", "standard-error")
-            else tuple())
-
-    def __file_missing_error__():
-        return (
-            ("No file was uploaded.",)
-            if ("qc_text_file" not in rqst.files or
-                rqst.files["qc_text_file"].filename == "")
-            else tuple())
-
-    def __file_mimetype_error__():
-        text_file = rqst.files["qc_text_file"]
-        return (
-            (
-                ("Invalid file! Expected a tab-separated-values file, or a zip "
-                 "file of the a tab-separated-values file."),)
-            if text_file.mimetype not in (
-                    "text/plain", "text/tab-separated-values",
-                    "application/zip")
-            else tuple())
-
-    return (
-        __filetype_error__() +
-        (__file_missing_error__() or __file_mimetype_error__()))
-
-def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]:
-    """Check the uploaded zip file for errors."""
-    zfile_errors: Tuple[str, ...] = tuple()
-    if is_zipfile(filepath):
-        with ZipFile(filepath, "r") as zfile:
-            infolist = zfile.infolist()
-            if len(infolist) != 1:
-                zfile_errors = zfile_errors + (
-                    ("Expected exactly one (1) member file within the uploaded zip "
-                     f"file. Got {len(infolist)} member files."),)
-            if len(infolist) == 1 and infolist[0].is_dir():
-                zfile_errors = zfile_errors + (
-                    ("Expected a member text file in the uploaded zip file. Got a "
-                     "directory/folder."),)
-
-            if len(infolist) == 1 and not infolist[0].is_dir():
-                zfile.extract(infolist[0], path=upload_dir)
-                mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}")
-                if mime[0] != "text/tab-separated-values":
-                    zfile_errors = zfile_errors + (
-                        ("Expected the member text file in the uploaded zip file to"
-                         " be a tab-separated file."),)
-
-    return zfile_errors
-
-
-@indexbp.route("/", methods=["GET"])
-@require_login
-def index():
-    """Display the expression data index page."""
-    return render_template("expression-data/index.html")
-
-
-@indexbp.route("/upload", methods=["GET", "POST"])
-@require_login
-def upload_file():
-    """Enables uploading the files"""
-    if request.method == "GET":
-        return render_template(
-            "select_species.html", species=with_db_connection(species))
-
-    upload_dir = app.config["UPLOAD_FOLDER"]
-    request_errors = errors(request)
-    if request_errors:
-        for error in request_errors:
-            flash(error, "alert-danger error-expr-data")
-        return redirect(url_for("expression-data.index.upload_file"))
-
-    filename = secure_filename(request.files["qc_text_file"].filename)
-    if not os.path.exists(upload_dir):
-        os.mkdir(upload_dir)
-
-    filepath = os.path.join(upload_dir, filename)
-    request.files["qc_text_file"].save(os.path.join(upload_dir, filename))
-
-    zip_errors = zip_file_errors(filepath, upload_dir)
-    if zip_errors:
-        for error in zip_errors:
-            flash(error, "alert-danger error-expr-data")
-        return redirect(url_for("expression-data.index.upload_file"))
-
-    return redirect(url_for("expression-data.parse.parse",
-                            speciesid=request.form["speciesid"],
-                            filename=filename,
-                            filetype=request.form["filetype"]))
-
-@indexbp.route("/data-review", methods=["GET"])
-@require_login
-def data_review():
-    """Provide some help on data expectations to the user."""
-    return render_template("data_review.html")
diff --git a/uploader/expression_data/parse.py b/uploader/expression_data/parse.py
deleted file mode 100644
index fc1c3f0..0000000
--- a/uploader/expression_data/parse.py
+++ /dev/null
@@ -1,178 +0,0 @@
-"""File parsing module"""
-import os
-
-import jsonpickle
-from redis import Redis
-from flask import flash, request, url_for, redirect, Blueprint, render_template
-from flask import current_app as app
-
-from quality_control.errors import InvalidValue, DuplicateHeading
-
-from uploader import jobs
-from uploader.dbinsert import species_by_id
-from uploader.db_utils import with_db_connection
-from uploader.authorisation import require_login
-
-parsebp = Blueprint("parse", __name__)
-
-def isinvalidvalue(item):
-    """Check whether item is of type InvalidValue"""
-    return isinstance(item, InvalidValue)
-
-def isduplicateheading(item):
-    """Check whether item is of type DuplicateHeading"""
-    return isinstance(item, DuplicateHeading)
-
-@parsebp.route("/parse", methods=["GET"])
-@require_login
-def parse():
-    """Trigger file parsing"""
-    errors = False
-    speciesid = request.args.get("speciesid")
-    filename = request.args.get("filename")
-    filetype = request.args.get("filetype")
-    if speciesid is None:
-        flash("No species selected", "alert-error error-expr-data")
-        errors = True
-    else:
-        try:
-            speciesid = int(speciesid)
-            species = with_db_connection(
-                lambda con: species_by_id(con, speciesid))
-            if not bool(species):
-                flash("No such species.", "alert-error error-expr-data")
-                errors = True
-        except ValueError:
-            flash("Invalid speciesid provided. Expected an integer.",
-                  "alert-error error-expr-data")
-            errors = True
-
-    if filename is None:
-        flash("No file provided", "alert-error error-expr-data")
-        errors = True
-
-    if filetype is None:
-        flash("No filetype provided", "alert-error error-expr-data")
-        errors = True
-
-    if filetype not in ("average", "standard-error"):
-        flash("Invalid filetype provided", "alert-error error-expr-data")
-        errors = True
-
-    if filename:
-        filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename)
-        if not os.path.exists(filepath):
-            flash("Selected file does not exist (any longer)",
-                  "alert-error error-expr-data")
-            errors = True
-
-    if errors:
-        return redirect(url_for("expression-data.index.upload_file"))
-
-    redisurl = app.config["REDIS_URL"]
-    with Redis.from_url(redisurl, decode_responses=True) as rconn:
-        job = jobs.launch_job(
-            jobs.build_file_verification_job(
-                rconn, app.config["SQL_URI"], redisurl,
-                speciesid, filepath, filetype,
-                app.config["JOBS_TTL_SECONDS"]),
-            redisurl,
-            f"{app.config['UPLOAD_FOLDER']}/job_errors")
-
-    return redirect(url_for("expression-data.parse.parse_status", job_id=job["jobid"]))
-
-@parsebp.route("/status/<job_id>", methods=["GET"])
-def parse_status(job_id: str):
-    "Retrieve the status of the job"
-    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
-        try:
-            job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
-        except jobs.JobNotFound as _exc:
-            return render_template("no_such_job.html", job_id=job_id), 400
-
-    error_filename = jobs.error_filename(
-        job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")
-    if os.path.exists(error_filename):
-        stat = os.stat(error_filename)
-        if stat.st_size > 0:
-            return redirect(url_for("parse.fail", job_id=job_id))
-
-    job_id = job["jobid"]
-    progress = float(job["percent"])
-    status = job["status"]
-    filename = job.get("filename", "uploaded file")
-    errors = jsonpickle.decode(
-        job.get("errors", jsonpickle.encode(tuple())))
-    if status in ("success", "aborted"):
-        return redirect(url_for("expression-data.parse.results", job_id=job_id))
-
-    if status == "parse-error":
-        return redirect(url_for("parse.fail", job_id=job_id))
-
-    app.jinja_env.globals.update(
-        isinvalidvalue=isinvalidvalue,
-        isduplicateheading=isduplicateheading)
-    return render_template(
-        "job_progress.html",
-        job_id = job_id,
-        job_status = status,
-        progress = progress,
-        message = job.get("message", ""),
-        job_name = f"Parsing '{filename}'",
-        errors=errors)
-
-@parsebp.route("/results/<job_id>", methods=["GET"])
-def results(job_id: str):
-    """Show results of parsing..."""
-    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
-        job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
-
-    if job:
-        filename = job["filename"]
-        errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple())))
-        app.jinja_env.globals.update(
-            isinvalidvalue=isinvalidvalue,
-            isduplicateheading=isduplicateheading)
-        return render_template(
-            "parse_results.html",
-            errors=errors,
-            job_name = f"Parsing '{filename}'",
-            user_aborted = job.get("user_aborted"),
-            job_id=job["jobid"])
-
-    return render_template("no_such_job.html", job_id=job_id)
-
-@parsebp.route("/fail/<job_id>", methods=["GET"])
-def fail(job_id: str):
-    """Handle parsing failure"""
-    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
-        job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
-
-    if job:
-        error_filename = jobs.error_filename(
-            job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")
-        if os.path.exists(error_filename):
-            stat = os.stat(error_filename)
-            if stat.st_size > 0:
-                return render_template(
-                    "worker_failure.html", job_id=job_id)
-
-        return render_template("parse_failure.html", job=job)
-
-    return render_template("no_such_job.html", job_id=job_id)
-
-@parsebp.route("/abort", methods=["POST"])
-@require_login
-def abort():
-    """Handle user request to abort file processing"""
-    job_id = request.form["job_id"]
-
-    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
-        job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
-
-        if job:
-            rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id),
-                       key="user_aborted",
-                       value=int(True))
-
-    return redirect(url_for("expression-data.parse.parse_status", job_id=job_id))
diff --git a/uploader/expression_data/views.py b/uploader/expression_data/views.py
new file mode 100644
index 0000000..6900c51
--- /dev/null
+++ b/uploader/expression_data/views.py
@@ -0,0 +1,384 @@
+"""Views for expression data"""
+import os
+import uuid
+import mimetypes
+from typing import Tuple
+from zipfile import ZipFile, is_zipfile
+
+import jsonpickle
+from redis import Redis
+from werkzeug.utils import secure_filename
+from flask import (flash,
+                   request,
+                   url_for,
+                   redirect,
+                   Blueprint,
+                   current_app as app)
+
+from quality_control.errors import InvalidValue, DuplicateHeading
+
+from uploader import jobs
+from uploader.ui import make_template_renderer
+from uploader.authorisation import require_login
+from uploader.species.models import all_species, species_by_id
+from uploader.db_utils import with_db_connection, database_connection
+from uploader.datautils import safe_int, order_by_family, enumerate_sequence
+from uploader.population.models import (populations_by_species,
+                                        population_by_species_and_id)
+
+exprdatabp = Blueprint("expression-data", __name__)
+render_template = make_template_renderer("expression-data")
+
+def isinvalidvalue(item):
+    """Check whether item is of type InvalidValue"""
+    return isinstance(item, InvalidValue)
+
+
+def isduplicateheading(item):
+    """Check whether item is of type DuplicateHeading"""
+    return isinstance(item, DuplicateHeading)
+
+
+def errors(rqst) -> Tuple[str, ...]:
+    """Return a tuple of the errors found in the request `rqst`. If no error is
+    found, then an empty tuple is returned."""
+    def __filetype_error__():
+        return (
+            ("Invalid file type provided.",)
+            if rqst.form.get("filetype") not in ("average", "standard-error")
+            else tuple())
+
+    def __file_missing_error__():
+        return (
+            ("No file was uploaded.",)
+            if ("qc_text_file" not in rqst.files or
+                rqst.files["qc_text_file"].filename == "")
+            else tuple())
+
+    def __file_mimetype_error__():
+        text_file = rqst.files["qc_text_file"]
+        return (
+            (
+                ("Invalid file! Expected a tab-separated-values file, or a zip "
+                 "file of the a tab-separated-values file."),)
+            if text_file.mimetype not in (
+                    "text/plain", "text/tab-separated-values",
+                    "application/zip")
+            else tuple())
+
+    return (
+        __filetype_error__() +
+        (__file_missing_error__() or __file_mimetype_error__()))
+
+
+def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]:
+    """Check the uploaded zip file for errors."""
+    zfile_errors: Tuple[str, ...] = tuple()
+    if is_zipfile(filepath):
+        with ZipFile(filepath, "r") as zfile:
+            infolist = zfile.infolist()
+            if len(infolist) != 1:
+                zfile_errors = zfile_errors + (
+                    ("Expected exactly one (1) member file within the uploaded zip "
+                     f"file. Got {len(infolist)} member files."),)
+            if len(infolist) == 1 and infolist[0].is_dir():
+                zfile_errors = zfile_errors + (
+                    ("Expected a member text file in the uploaded zip file. Got a "
+                     "directory/folder."),)
+
+            if len(infolist) == 1 and not infolist[0].is_dir():
+                zfile.extract(infolist[0], path=upload_dir)
+                mime = mimetypes.guess_type(f"{upload_dir}/{infolist[0].filename}")
+                if mime[0] != "text/tab-separated-values":
+                    zfile_errors = zfile_errors + (
+                        ("Expected the member text file in the uploaded zip file to"
+                         " be a tab-separated file."),)
+
+    return zfile_errors
+
+
+@exprdatabp.route("populations/expression-data", methods=["GET"])
+@require_login
+def index():
+    """Display the expression data index page."""
+    with database_connection(app.config["SQL_URI"]) as conn:
+        if not bool(request.args.get("species_id")):
+            return render_template("expression-data/index.html",
+                                   species=order_by_family(all_species(conn)),
+                                   activelink="genotypes")
+        species = species_by_id(conn, request.args.get("species_id"))
+        if not bool(species):
+            flash(f"Could not find species with ID '{request.args.get('species_id')}'!",
+                  "alert-danger")
+            return redirect(url_for("species.populations.expression-data.index"))
+        return redirect(url_for("species.populations.expression-data.select_population",
+                                species_id=species["SpeciesId"]))
+    return render_template()
+
+
+@exprdatabp.route("<int:species_id>/populations/expression-data/select-population",
+                  methods=["GET"])
+@require_login
+def select_population(species_id: int):
+    """Select the expression data's population."""
+    with database_connection(app.config["SQL_URI"]) as conn:
+        species = species_by_id(conn, species_id)
+        if not bool(species):
+            flash("Invalid species provided!", "alert-danger")
+            return redirect(url_for("species.populations.expression-data.index"))
+
+        if not bool(request.args.get("population_id")):
+            return render_template("expression-data/select-population.html",
+                                   species=species,
+                                   populations=order_by_family(
+                                       populations_by_species(conn, species_id),
+                                       order_key="FamilyOrder"),
+                                   activelink="genotypes")
+
+        population = population_by_species_and_id(
+            conn, species_id, request.args.get("population_id"))
+        if not bool(population):
+            flash("Invalid population selected!", "alert-danger")
+            return redirect(url_for(
+                "species.populations.expression-data.select_population",
+                species_id=species_id))
+
+        return redirect(url_for("species.populations.expression-data.upload_file",
+                                species_id=species_id,
+                                population_id=population["Id"]))
+
+
+@exprdatabp.route("<int:species_id>/populations/<int:population_id>/"
+                  "expression-data/upload",
+                  methods=["GET", "POST"])
+@require_login
+def upload_file(species_id: int, population_id: int):
+    """Enables uploading the files"""
+    with database_connection(app.config["SQL_URI"]) as conn:
+        species = species_by_id(conn, species_id)
+        population = population_by_species_and_id(conn, species_id, population_id)
+        if request.method == "GET":
+            return render_template("expression-data/select-file.html",
+                                   species=species,
+                                   population=population)
+
+        upload_dir = app.config["UPLOAD_FOLDER"]
+        request_errors = errors(request)
+        if request_errors:
+            for error in request_errors:
+                flash(error, "alert-danger error-expr-data")
+            return redirect(url_for("species.populations.expression-data.upload_file"))
+
+        filename = secure_filename(request.files["qc_text_file"].filename)
+        if not os.path.exists(upload_dir):
+            os.mkdir(upload_dir)
+
+        filepath = os.path.join(upload_dir, filename)
+        request.files["qc_text_file"].save(os.path.join(upload_dir, filename))
+
+        zip_errors = zip_file_errors(filepath, upload_dir)
+        if zip_errors:
+            for error in zip_errors:
+                flash(error, "alert-danger error-expr-data")
+            return redirect(url_for("species.populations.expression-data.index.upload_file"))
+
+        return redirect(url_for("species.populations.expression-data.parse_file",
+                                species_id=species_id,
+                                population_id=population_id,
+                                filename=filename,
+                                filetype=request.form["filetype"]))
+
+
+@exprdatabp.route("/data-review", methods=["GET"])
+@require_login
+def data_review():
+    """Provide some help on data expectations to the user."""
+    return render_template("expression-data/data-review.html")
+
+
+@exprdatabp.route(
+    "<int:species_id>/populations/<int:population_id>/expression-data/parse",
+    methods=["GET"])
+@require_login
+def parse_file(species_id: int, population_id: int):
+    """Trigger file parsing"""
+    errors = False
+    filename = request.args.get("filename")
+    filetype = request.args.get("filetype")
+
+    species = with_db_connection(lambda con: species_by_id(con, species_id))
+    if not bool(species):
+        flash("No such species.", "alert-danger")
+        errors = True
+
+    if filename is None:
+        flash("No file provided", "alert-danger")
+        errors = True
+
+    if filetype is None:
+        flash("No filetype provided", "alert-danger")
+        errors = True
+
+    if filetype not in ("average", "standard-error"):
+        flash("Invalid filetype provided", "alert-danger")
+        errors = True
+
+    if filename:
+        filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename)
+        if not os.path.exists(filepath):
+            flash("Selected file does not exist (any longer)", "alert-danger")
+            errors = True
+
+    if errors:
+        return redirect(url_for("species.populations.expression-data.upload_file"))
+
+    redisurl = app.config["REDIS_URL"]
+    with Redis.from_url(redisurl, decode_responses=True) as rconn:
+        job = jobs.launch_job(
+            jobs.build_file_verification_job(
+                rconn, app.config["SQL_URI"], redisurl,
+                species_id, filepath, filetype,
+                app.config["JOBS_TTL_SECONDS"]),
+            redisurl,
+            f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+    return redirect(url_for("species.populations.expression-data.parse_status",
+                            species_id=species_id,
+                            population_id=population_id,
+                            job_id=job["jobid"]))
+
+
+@exprdatabp.route(
+    "<int:species_id>/populations/<int:population_id>/expression-data/parse/"
+    "status/<uuid:job_id>",
+    methods=["GET"])
+@require_login
+def parse_status(species_id: int, population_id: int, job_id: str):
+    "Retrieve the status of the job"
+    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+        try:
+            job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+        except jobs.JobNotFound as _exc:
+            return render_template("no_such_job.html", job_id=job_id), 400
+
+    error_filename = jobs.error_filename(
+        job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+    if os.path.exists(error_filename):
+        stat = os.stat(error_filename)
+        if stat.st_size > 0:
+            return redirect(url_for("parse.fail", job_id=job_id))
+
+    job_id = job["jobid"]
+    progress = float(job["percent"])
+    status = job["status"]
+    filename = job.get("filename", "uploaded file")
+    errors = jsonpickle.decode(
+        job.get("errors", jsonpickle.encode(tuple())))
+    if status in ("success", "aborted"):
+        return redirect(url_for("species.populations.expression-data.results",
+                                species_id=species_id,
+                                population_id=population_id,
+                                job_id=job_id))
+
+    if status == "parse-error":
+        return redirect(url_for("species.populations.expression-data.fail", job_id=job_id))
+
+    app.jinja_env.globals.update(
+        isinvalidvalue=isinvalidvalue,
+        isduplicateheading=isduplicateheading)
+    return render_template(
+        "expression-data/job-progress.html",
+        job_id = job_id,
+        job_status = status,
+        progress = progress,
+        message = job.get("message", ""),
+        job_name = f"Parsing '{filename}'",
+        errors=errors,
+        species=with_db_connection(
+            lambda conn: species_by_id(conn, species_id)),
+        population=with_db_connection(
+            lambda conn: population_by_species_and_id(
+                conn, species_id, population_id)))
+
+
+@exprdatabp.route(
+    "<int:species_id>/populations/<int:population_id>/expression-data/parse/"
+    "<uuid:job_id>/results",
+    methods=["GET"])
+@require_login
+def results(species_id: int, population_id: int, job_id: uuid.UUID):
+    """Show results of parsing..."""
+    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+        job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+    if job:
+        filename = job["filename"]
+        errors = jsonpickle.decode(job.get("errors", jsonpickle.encode(tuple())))
+        app.jinja_env.globals.update(
+            isinvalidvalue=isinvalidvalue,
+            isduplicateheading=isduplicateheading)
+        return render_template(
+            "expression-data/parse-results.html",
+            errors=errors,
+            job_name = f"Parsing '{filename}'",
+            user_aborted = job.get("user_aborted"),
+            job_id=job["jobid"],
+            species=with_db_connection(
+                lambda conn: species_by_id(conn, species_id)),
+            population=with_db_connection(
+                lambda conn: population_by_species_and_id(
+                    conn, species_id, population_id)))
+
+    return render_template("expression-data/no-such-job.html", job_id=job_id)
+
+
+@exprdatabp.route(
+    "<int:species_id>/populations/<int:population_id>/expression-data/parse/"
+    "<uuid:job_id>/fail",
+    methods=["GET"])
+@require_login
+def fail(species_id: int, population_id: int, job_id: str):
+    """Handle parsing failure"""
+    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+        job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+    if job:
+        error_filename = jobs.error_filename(
+            job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")
+        if os.path.exists(error_filename):
+            stat = os.stat(error_filename)
+            if stat.st_size > 0:
+                return render_template(
+                    "worker_failure.html", job_id=job_id)
+
+        return render_template("parse_failure.html", job=job)
+
+    return render_template("expression-data/no-such-job.html",
+                           **with_db_connection(lambda conn: {
+                               "species_id": species_by_id(conn, species_id),
+                               "population_id": population_by_species_and_id(
+                                   conn, species_id, population_id)}),
+                           job_id=job_id)
+
+
+@exprdatabp.route(
+    "<int:species_id>/populations/<int:population_id>/expression-data/parse/"
+    "abort",
+    methods=["POST"])
+@require_login
+def abort(species_id: int, population_id: int):
+    """Handle user request to abort file processing"""
+    job_id = request.form["job_id"]
+
+    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+        job = jobs.job(rconn, jobs.jobsnamespace(), job_id)
+
+        if job:
+            rconn.hset(name=jobs.job_key(jobs.jobsnamespace(), job_id),
+                       key="user_aborted",
+                       value=int(True))
+
+    return redirect(url_for("species.populations.expression-data.parse_status",
+                            species_id=species_id,
+                            population_id=population_id,
+                            job_id=job_id))
diff --git a/uploader/population/views.py b/uploader/population/views.py
index 39a5762..631f0be 100644
--- a/uploader/population/views.py
+++ b/uploader/population/views.py
@@ -18,6 +18,7 @@ from uploader.oauth2.client import oauth2_post
 from uploader.ui import make_template_renderer
 from uploader.authorisation import require_login
 from uploader.genotypes.views import genotypesbp
+from uploader.expression_data.views import exprdatabp
 from uploader.db_utils import database_connection
 from uploader.datautils import enumerate_sequence
 from uploader.species.models import (all_species,
@@ -34,6 +35,7 @@ __active_link__ = "populations"
 popbp = Blueprint("populations", __name__)
 popbp.register_blueprint(samplesbp, url_prefix="/")
 popbp.register_blueprint(genotypesbp, url_prefix="/")
+popbp.register_blueprint(exprdatabp, url_prefix="/")
 render_template = make_template_renderer("populations")
 
 
diff --git a/uploader/templates/expression-data/base.html b/uploader/templates/expression-data/base.html
new file mode 100644
index 0000000..d63fd7e
--- /dev/null
+++ b/uploader/templates/expression-data/base.html
@@ -0,0 +1,13 @@
+{%extends "populations/base.html"%}
+
+{%block lvl3_breadcrumbs%}
+<li {%if activelink=="expression-data"%}
+    class="breadcrumb-item active"
+    {%else%}
+    class="breadcrumb-item"
+    {%endif%}>
+  <a href="{{url_for('species.populations.expression-data.index')}}">
+    Expression Data</a>
+</li>
+{%block lvl4_breadcrumbs%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/data_review.html b/uploader/templates/expression-data/data-review.html
index 4e5c586..c985b03 100644
--- a/uploader/templates/data_review.html
+++ b/uploader/templates/expression-data/data-review.html
@@ -26,7 +26,7 @@
           <small class="text-muted">
             If you encounter an error saying your sample(s)/case(s) do not exist
             in the GeneNetwork database, then you will have to use the
-            <a href="{{url_for('expression-data.samples.select_species')}}"
+            <a href="{{url_for('species.populations.samples.index')}}"
                title="Upload samples/cases feature">Upload Samples/Cases</a>
             option on this system to upload them.
           </small>
@@ -70,8 +70,8 @@
 	  column</li>
 	<li>The values of each field <strong>ARE NOT</strong> quoted.</li>
 	<li>Here is an
-	  <a href="https://gitlab.com/fredmanglis/gnqc_py/-/blob/main/tests/test_data/no_data_errors.tsv">
-	    example file</a> with a single data row.</li>
+	  <a href="https://gitlab.com/fredmanglis/gnqc_py/-/blob/main/tests/test_data/no_data_errors.tsv"
+             target="_blank">example file</a> with a single data row.</li>
       </ul>
     </li>
     <li>.txt files: Content has the same format as .tsv file above</li>
diff --git a/uploader/templates/expression-data/index.html b/uploader/templates/expression-data/index.html
index ed5d8dd..9ba3582 100644
--- a/uploader/templates/expression-data/index.html
+++ b/uploader/templates/expression-data/index.html
@@ -1,5 +1,6 @@
-{%extends "base.html"%}
+{%extends "expression-data/base.html"%}
 {%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-select-species.html" import select_species_form%}
 
 {%block title%}Expression Data{%endblock%}
 
@@ -10,86 +11,23 @@
   <a href="{{url_for('base.index')}}">Home</a>
 </li>
 <li class="breadcrumb-item active">
-  <a href="{{url_for('expression-data.index.index')}}">Expression Data</a>
+  <a href="{{url_for('species.populations.expression-data.index')}}"
+     title="Upload expression data.">
+    Expression Data</a>
 </li>
 {%endblock%}
 
 {%block contents%}
 <div class="row">
-  {{flash_all_messages()}}
-
-  <h1 class="heading">data upload</h1>
-
-  <div class="explainer">
-    <p>Each of the sections below gives you a different option for data expression-data.
-      Please read the documentation for each section carefully to understand what
-      each section is about.</p>
-  </div>
-</div>
-
-<div class="row">
-  <h2 class="heading">R/qtl2 Bundles</h2>
-
-  <div class="explainer">
-    <p>This feature combines and extends the two upload methods below. Instead of
-      uploading one item at a time, the R/qtl2 bundle you upload can contain both
-      the genotypes data (samples/individuals/cases and their data) and the
-      expression data.</p>
-    <p>The R/qtl2 bundle, additionally, can contain extra metadata, that neither
-      of the methods below can handle.</p>
-
-    <a href="{{url_for('expression-data.rqtl2.select_species')}}"
-       title="Upload a zip bundle of R/qtl2 files">
-      <button class="btn btn-primary">upload R/qtl2 bundle</button></a>
-  </div>
-</div>
-
-
-<div class="row">
   <h2 class="heading">Expression Data</h2>
+  {{flash_all_messages()}}
 
-  <div class="explainer">
-    <p>This feature enables you to upload expression data. It expects the data to
-      be in <strong>tab-separated values (TSV)</strong> files. The data should be
-      a simple matrix of <em>phenotype × sample</em>, i.e. The first column is a
-      list of the <em>phenotypes</em> and the first row is a list of
-      <em>samples/cases</em>.</p>
-
-    <p>If you haven't done so please go to this page to learn the requirements for
-      file formats and helpful suggestions to enter your data in a fast and easy
-      way.</p>
-
-    <ol>
-      <li><strong>PLEASE REVIEW YOUR DATA.</strong>Make sure your data complies
-        with our system requirements. (
-        <a href="{{url_for('expression-data.index.data_review')}}#data-concerns"
-	   title="Details for the data expectations.">Help</a>
-        )</li>
-      <li><strong>UPLOAD YOUR DATA FOR DATA VERIFICATION.</strong> We accept
-        <strong>.csv</strong>, <strong>.txt</strong> and <strong>.zip</strong>
-        files (<a href="{{url_for('expression-data.index.data_review')}}#file-types"
-	          title="Details for the data expectations.">Help</a>)</li>
-    </ol>
-  </div>
-
-  <a href="{{url_for('expression-data.index.upload_file')}}"
-     title="Upload your expression data"
-     class="btn btn-primary">upload expression data</a>
+  <p>This section allows you to enter the expression data for your experiment.
+    You will need to select the species that your data concerns below.</p>
 </div>
 
 <div class="row">
-  <h2 class="heading">samples/cases</h2>
-
-  <div class="explainer">
-    <p>For the expression data above, you need the samples/cases in your file to
-      already exist in the GeneNetwork database. If there are any samples that do
-      not already exist the upload of the expression data will fail.</p>
-    <p>This section gives you the opportunity to upload any missing samples</p>
-  </div>
-
-  <a href="{{url_for('expression-data.samples.select_species')}}"
-     title="Upload samples/cases/individuals for your data"
-     class="btn btn-primary">upload Samples/Cases</a>
+  {{select_species_form(url_for("species.populations.expression-data.index"),
+  species)}}
 </div>
-
 {%endblock%}
diff --git a/uploader/templates/job_progress.html b/uploader/templates/expression-data/job-progress.html
index 2feaa89..ef264e1 100644
--- a/uploader/templates/job_progress.html
+++ b/uploader/templates/expression-data/job-progress.html
@@ -1,5 +1,6 @@
 {%extends "base.html"%}
 {%from "errors_display.html" import errors_display%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
 
 {%block extrameta%}
 <meta http-equiv="refresh" content="5">
@@ -11,7 +12,9 @@
 <h1 class="heading">{{job_name}}</h2>
 
 <div class="row">
-  <form action="{{url_for('expression-data.parse.abort')}}" method="POST">
+  <form action="{{url_for('species.populations.expression-data.abort',
+                species_id=species.SpeciesId,
+                population_id=population.Id)}}" method="POST">
     <legend class="heading">Status</legend>
     <div class="form-group">
       <label for="job_status" class="form-label">status:</label>
@@ -38,3 +41,7 @@
 </div>
 
 {%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/no_such_job.html b/uploader/templates/expression-data/no-such-job.html
index 874d047..d22c429 100644
--- a/uploader/templates/no_such_job.html
+++ b/uploader/templates/expression-data/no-such-job.html
@@ -1,7 +1,8 @@
 {%extends "base.html"%}
 
 {%block extrameta%}
-<meta http-equiv="refresh" content="5;url={{url_for('expression-data.index.upload_file')}}">
+<meta http-equiv="refresh"
+      content="5;url={{url_for('species.populations.expression-data.index.upload_file')}}">
 {%endblock%}
 
 {%block title%}No Such Job{%endblock%}
diff --git a/uploader/templates/parse_failure.html b/uploader/templates/expression-data/parse-failure.html
index 31f6be8..31f6be8 100644
--- a/uploader/templates/parse_failure.html
+++ b/uploader/templates/expression-data/parse-failure.html
diff --git a/uploader/templates/expression-data/parse-results.html b/uploader/templates/expression-data/parse-results.html
new file mode 100644
index 0000000..03a23e2
--- /dev/null
+++ b/uploader/templates/expression-data/parse-results.html
@@ -0,0 +1,39 @@
+{%extends "base.html"%}
+{%from "errors_display.html" import errors_display%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Parse Results{%endblock%}
+
+{%block contents%}
+
+<div class="row">
+  <h2 class="heading">{{job_name}}: parse results</h2>
+
+  {%if user_aborted%}
+  <span class="alert-warning">Job aborted by the user</span>
+  {%endif%}
+
+  {{errors_display(errors, "No errors found in the file", "We found the following errors", True)}}
+
+  {%if errors | length == 0 and not user_aborted %}
+  <form method="post" action="{{url_for('dbinsert.select_platform')}}">
+    <input type="hidden" name="job_id" value="{{job_id}}" />
+    <input type="submit" value="update database" class="btn btn-primary" />
+  </form>
+  {%endif%}
+
+  {%if errors | length > 0 or user_aborted %}
+  <br />
+  <a href="{{url_for('species.populations.expression-data.upload_file',
+           species_id=species.SpeciesId,
+           population_id=population.Id)}}"
+     title="Back to index page."
+     class="btn btn-primary">Go back</a>
+
+  {%endif%}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/expression-data/select-file.html b/uploader/templates/expression-data/select-file.html
new file mode 100644
index 0000000..4ca461e
--- /dev/null
+++ b/uploader/templates/expression-data/select-file.html
@@ -0,0 +1,115 @@
+{%extends "expression-data/base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+{%from "upload_progress_indicator.html" import upload_progress_indicator%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Expression Data &mdash; Upload Data{%endblock%}
+
+{%block pagetitle%}Expression Data &mdash; Upload Data{%endblock%}
+
+{%block contents%}
+{{upload_progress_indicator()}}
+
+<div class="row">
+  <h2 class="heading">Upload Expression Data</h2>
+
+  <p>This feature enables you to upload expression data. It expects the data to
+    be in <strong>tab-separated values (TSV)</strong> files. The data should be
+    a simple matrix of <em>phenotype × sample</em>, i.e. The first column is a
+    list of the <em>phenotypes</em> and the first row is a list of
+    <em>samples/cases</em>.</p>
+
+  <p>If you haven't done so please go to this page to learn the requirements for
+    file formats and helpful suggestions to enter your data in a fast and easy
+    way.</p>
+
+  <ol>
+    <li><strong>PLEASE REVIEW YOUR DATA.</strong>Make sure your data complies
+      with our system requirements. (
+      <a href="{{url_for('species.populations.expression-data.data_review')}}#data-concerns"
+	 title="Details for the data expectations.">Help</a>
+      )</li>
+    <li><strong>UPLOAD YOUR DATA FOR DATA VERIFICATION.</strong> We accept
+      <strong>.csv</strong>, <strong>.txt</strong> and <strong>.zip</strong>
+      files (<a href="{{url_for('species.populations.expression-data.data_review')}}#file-types"
+	        title="Details for the data expectations.">Help</a>)</li>
+  </ol>
+</div>
+
+<div class="row">
+  <form action="{{url_for(
+                'species.populations.expression-data.upload_file',
+                species_id=species.SpeciesId,
+                population_id=population.Id)}}"
+        method="POST"
+        enctype="multipart/form-data"
+        id="frm-upload-expression-data">
+    {{flash_messages("error-expr-data")}}
+
+    <div class="form-group">
+      <legend class="heading">File Type</legend>
+
+      <div class="radio">
+        <label for="filetype_average" class="form-check-label">
+          <input type="radio" name="filetype" value="average" id="filetype_average"
+	         required="required" class="form-check-input" />
+          Average</label>
+        <p class="form-text text-muted">
+          <small>The averages data …</small></p>
+      </div>
+
+      <div class="radio">
+        <label for="filetype_standard_error" class="form-check-label">
+          <input type="radio" name="filetype" value="standard-error"
+	         id="filetype_standard_error" required="required"
+	         class="form-check-input" />
+          Standard Error
+        </label>
+        <p class="form-text text-muted">
+          <small>The standard errors computed from the averages …</small></p>
+      </div>
+    </div>
+
+    <div class="form-group">
+      <span id="no-file-error" class="alert-danger" style="display: none;">
+        No file selected
+      </span>
+      <label for="file_upload" class="form-label">Select File</label>
+      <input type="file" name="qc_text_file" id="file_upload"
+	     accept="text/plain, text/tab-separated-values, application/zip"
+	     class="form-control"/>
+      <p class="form-text text-muted">
+        <small>Select the file to upload.</small></p>
+    </div>
+
+    <button type="submit"
+            class="btn btn-primary"
+            data-toggle="modal"
+            data-target="#upload-progress-indicator">upload file</button>
+  </form>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/upload_progress.js"></script>
+<script type="text/javascript">
+  function setup_formdata(form) {
+      var formdata = new FormData();
+      formdata.append(
+	  "qc_text_file",
+	  form.querySelector("input[type='file']").files[0]);
+      formdata.append(
+	  "filetype",
+	  selected_filetype(
+	      Array.from(form.querySelectorAll("input[type='radio']"))));
+      return formdata;
+  }
+
+  setup_upload_handlers(
+      "frm-upload-expression-data", make_data_uploader(setup_formdata));
+</script>
+{%endblock%}
diff --git a/uploader/templates/expression-data/select-population.html b/uploader/templates/expression-data/select-population.html
new file mode 100644
index 0000000..8555e27
--- /dev/null
+++ b/uploader/templates/expression-data/select-population.html
@@ -0,0 +1,29 @@
+{%extends "expression-data/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+{%from "populations/macro-select-population.html" import select_population_form%}
+
+{%block title%}Expression Data{%endblock%}
+
+{%block pagetitle%}Expression Data{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+  <p>You have selected the species. Now you need to select the population that
+  the expression data belongs to.</p>
+</div>
+
+<div class="row">
+  {{select_population_form(url_for(
+  "species.populations.expression-data.select_population",
+  species_id=species.SpeciesId),
+  populations)}}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}
diff --git a/uploader/templates/parse_results.html b/uploader/templates/parse_results.html
deleted file mode 100644
index 46fbaaf..0000000
--- a/uploader/templates/parse_results.html
+++ /dev/null
@@ -1,30 +0,0 @@
-{%extends "base.html"%}
-{%from "errors_display.html" import errors_display%}
-
-{%block title%}Parse Results{%endblock%}
-
-{%block contents%}
-<h1 class="heading">{{job_name}}: parse results</h2>
-
-{%if user_aborted%}
-<span class="alert-warning">Job aborted by the user</span>
-{%endif%}
-
-{{errors_display(errors, "No errors found in the file", "We found the following errors", True)}}
-
-{%if errors | length == 0 and not user_aborted %}
-<form method="post" action="{{url_for('dbinsert.select_platform')}}">
-  <input type="hidden" name="job_id" value="{{job_id}}" />
-  <input type="submit" value="update database" class="btn btn-primary" />
-</form>
-{%endif%}
-
-{%if errors | length > 0 or user_aborted %}
-<br />
-<a href="{{url_for('expression-data.index.upload_file')}}" title="Back to index page."
-   class="btn btn-primary">
-  Go back
-</a>
-{%endif%}
-
-{%endblock%}
diff --git a/uploader/templates/select_species.html b/uploader/templates/select_species.html
deleted file mode 100644
index 1642401..0000000
--- a/uploader/templates/select_species.html
+++ /dev/null
@@ -1,92 +0,0 @@
-{%extends "base.html"%}
-{%from "flash_messages.html" import flash_messages%}
-{%from "upload_progress_indicator.html" import upload_progress_indicator%}
-
-{%block title%}expression data: select species{%endblock%}
-
-{%block contents%}
-{{upload_progress_indicator()}}
-
-<h2 class="heading">expression data: select species</h2>
-
-<div class="row">
-  <form action="{{url_for('expression-data.index.upload_file')}}"
-        method="POST"
-        enctype="multipart/form-data"
-        id="frm-upload-expression-data">
-    <legend class="heading">upload expression data</legend>
-    {{flash_messages("error-expr-data")}}
-
-    <div class="form-group">
-      <label for="select_species01" class="form-label">Species</label>
-      <select id="select_species01"
-	      name="speciesid"
-	      required="required"
-              class="form-control">
-        <option value="">Select species</option>
-        {%for aspecies in species%}
-        <option value="{{aspecies.SpeciesId}}">{{aspecies.MenuName}}</option>
-        {%endfor%}
-      </select>
-    </div>
-
-    <div class="form-group">
-      <legend class="heading">file type</legend>
-
-      <div class="form-check">
-        <input type="radio" name="filetype" value="average" id="filetype_average"
-	       required="required" class="form-check-input" />
-        <label for="filetype_average" class="form-check-label">average</label>
-      </div>
-
-      <div class="form-check">
-        <input type="radio" name="filetype" value="standard-error"
-	       id="filetype_standard_error" required="required"
-	       class="form-check-input" />
-        <label for="filetype_standard_error" class="form-check-label">
-          standard error
-        </label>
-      </div>
-    </div>
-
-    <div class="form-group">
-      <span id="no-file-error" class="alert-danger" style="display: none;">
-        No file selected
-      </span>
-      <label for="file_upload" class="form-label">select file</label>
-      <input type="file" name="qc_text_file" id="file_upload"
-	     accept="text/plain, text/tab-separated-values, application/zip"
-	     class="form-control"/>
-    </div>
-
-    <button type="submit"
-            class="btn btn-primary"
-            data-toggle="modal"
-            data-target="#upload-progress-indicator">upload file</button>
-  </form>
-</div>
-{%endblock%}
-
-
-{%block javascript%}
-<script type="text/javascript" src="static/js/upload_progress.js"></script>
-<script type="text/javascript">
-  function setup_formdata(form) {
-      var formdata = new FormData();
-      formdata.append(
-	  "speciesid",
-	  form.querySelector("#select_species01").value)
-      formdata.append(
-	  "qc_text_file",
-	  form.querySelector("input[type='file']").files[0]);
-      formdata.append(
-	  "filetype",
-	  selected_filetype(
-	      Array.from(form.querySelectorAll("input[type='radio']"))));
-      return formdata;
-  }
-
-  setup_upload_handlers(
-      "frm-upload-expression-data", make_data_uploader(setup_formdata));
-</script>
-{%endblock%}