diff options
Diffstat (limited to 'uploader/samples/views.py')
| -rw-r--r-- | uploader/samples/views.py | 300 |
1 files changed, 300 insertions, 0 deletions
diff --git a/uploader/samples/views.py b/uploader/samples/views.py new file mode 100644 index 0000000..9ba1df8 --- /dev/null +++ b/uploader/samples/views.py @@ -0,0 +1,300 @@ +"""Code regarding samples""" +import os +import sys +import uuid +from pathlib import Path +from typing import Iterator + +import MySQLdb as mdb +from redis import Redis +from MySQLdb.cursors import DictCursor +from flask import (flash, + request, + url_for, + redirect, + Blueprint, + current_app as app) + +from uploader import jobs +from uploader.files import save_file +from uploader.ui import make_template_renderer +from uploader.authorisation import require_login +from uploader.input_validation import is_integer_input +from uploader.datautils import order_by_family, enumerate_sequence +from uploader.db_utils import ( + with_db_connection, + database_connection, + with_redis_connection) +from uploader.species.models import (all_species, + species_by_id, + order_species_by_family) +from uploader.population.models import(save_population, + population_by_id, + populations_by_species, + population_by_species_and_id) + +from .models import samples_by_species_and_population + +samplesbp = Blueprint("samples", __name__) +render_template = make_template_renderer("samples") + +@samplesbp.route("/samples", methods=["GET"]) +@require_login +def index(): + """Direct entry-point for uploading/handling the samples.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(request.args.get("species_id")): + return render_template( + "samples/index.html", + species=order_species_by_family(all_species(conn)), + activelink="samples") + species = species_by_id(conn, request.args.get("species_id")) + if not bool(species): + flash("No such species!", "alert-danger") + return redirect(url_for("species.populations.samples.index")) + return redirect(url_for("species.populations.samples.select_population", + species_id=species["SpeciesId"])) + + +@samplesbp.route("<int:species_id>/samples/select-population", methods=["GET"]) +@require_login +def select_population(species_id: int): + """Select the population to use for the samples.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("Invalid species!", "alert-danger") + return redirect(url_for("species.populations.samples.index")) + + if not bool(request.args.get("population_id")): + return render_template("samples/select-population.html", + species=species, + populations=order_by_family( + populations_by_species( + conn, + species_id), + order_key="FamilyOrder"), + activelink="samples") + + population = population_by_id(conn, request.args.get("population_id")) + if not bool(population): + flash("Population not found!", "alert-danger") + return redirect(url_for( + "species.populations.samples.select_population", + species_id=species_id)) + + return redirect(url_for("species.populations.samples.list_samples", + species_id=species_id, + population_id=population["Id"])) + +@samplesbp.route("<int:species_id>/populations/<int:population_id>/samples") +@require_login +def list_samples(species_id: int, population_id: int): + """ + List the samples in a particular population and give the ability to upload + new ones. + """ + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("Invalid species!", "alert-danger") + return redirect(url_for("species.populations.samples.index")) + + population = population_by_id(conn, population_id) + if not bool(population): + flash("Population not found!", "alert-danger") + return redirect(url_for( + "species.populations.samples.select_population", + species_id=species_id)) + + all_samples = enumerate_sequence(samples_by_species_and_population( + conn, species_id, population_id)) + total_samples = len(all_samples) + offset = int(request.args.get("from") or 0) + if offset < 0: + offset = 0 + count = int(request.args.get("count") or 20) + return render_template("samples/list-samples.html", + species=species, + population=population, + samples=all_samples[offset:offset+count], + offset=offset, + count=count, + total_samples=total_samples, + activelink="list-samples") + + +def build_sample_upload_job(# pylint: disable=[too-many-arguments] + speciesid: int, + populationid: int, + samplesfile: Path, + separator: str, + firstlineheading: bool, + quotechar: str): + """Define the async command to run the actual samples data upload.""" + return [ + sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"], + str(speciesid), str(populationid), str(samplesfile.absolute()), + separator, f"--redisuri={app.config['REDIS_URL']}", + f"--quotechar={quotechar}" + ] + (["--firstlineheading"] if firstlineheading else []) + + +@samplesbp.route("<int:species_id>/populations/<int:population_id>/upload-samples", + methods=["GET", "POST"]) +@require_login +def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements] + """Upload the samples.""" + samples_uploads_page = redirect(url_for( + "species.populations.samples.upload_samples", + species_id=species_id, + population_id=population_id)) + if not is_integer_input(species_id): + flash("You did not provide a valid species. Please select one to " + "continue.", + "alert-danger") + return redirect(url_for("expression-data.samples.select_species")) + species = with_db_connection(lambda conn: species_by_id(conn, species_id)) + if not bool(species): + flash("Species with given ID was not found.", "alert-danger") + return redirect(url_for("expression-data.samples.select_species")) + + if not is_integer_input(population_id): + flash("You did not provide a valid population. Please select one " + "to continue.", + "alert-danger") + return redirect(url_for("species.populations.samples.select_population", + species_id=species_id), + code=307) + population = with_db_connection( + lambda conn: population_by_id(conn, int(population_id))) + if not bool(population): + flash("Invalid grouping/population!", "alert-error") + return redirect(url_for("species.populations.samples.select_population", + species_id=species_id), + code=307) + + if request.method == "GET" or request.files.get("samples_file") is None: + return render_template("samples/upload-samples.html", + species=species, + population=population) + + try: + samples_file = save_file(request.files["samples_file"], + Path(app.config["UPLOAD_FOLDER"])) + except AssertionError: + flash("You need to provide a file with the samples data.", + "alert-error") + return samples_uploads_page + + firstlineheading = (request.form.get("first_line_heading") == "on") + + separator = request.form.get("separator", ",") + if separator == "other": + separator = request.form.get("other_separator", ",") + if not bool(separator): + flash("You need to provide a separator character.", "alert-error") + return samples_uploads_page + + quotechar = (request.form.get("field_delimiter", '"') or '"') + + redisuri = app.config["REDIS_URL"] + with Redis.from_url(redisuri, decode_responses=True) as rconn: + #TODO: Add a QC step here — what do we check? + # 1. Does any sample in the uploaded file exist within the database? + # If yes, what is/are its/their species and population? + # 2. If yes 1. above, provide error with notes on which species and + # populations already own the samples. + the_job = jobs.launch_job( + jobs.initialise_job( + rconn, + jobs.jobsnamespace(), + str(uuid.uuid4()), + build_sample_upload_job( + species["SpeciesId"], + population["InbredSetId"], + samples_file, + separator, + firstlineheading, + quotechar), + "samples_upload", + app.config["JOBS_TTL_SECONDS"], + {"job_name": f"Samples Upload: {samples_file.name}"}), + redisuri, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + return redirect(url_for( + "species.populations.samples.upload_status", + species_id=species_id, + population_id=population_id, + job_id=the_job["jobid"])) + + +@samplesbp.route("<int:species_id>/populations/<int:population_id>/" + "upload-samples/status/<uuid:job_id>", + methods=["GET"]) +@require_login +def upload_status(species_id: int, population_id: int, job_id: uuid.UUID): + """Check on the status of a samples upload job.""" + with database_connection(app.config["SQL_URI"]) as conn: + species = species_by_id(conn, species_id) + if not bool(species): + flash("You must provide a valid species.", "alert-danger") + return redirect(url_for("species.populations.samples.index")) + + population = population_by_species_and_id( + conn, species_id, population_id) + if not bool(population): + flash("You must provide a valid population.", "alert-danger") + return redirect(url_for( + "species.populations.samples.select_population", + species_id=species_id)) + + job = with_redis_connection(lambda rconn: jobs.job( + rconn, jobs.jobsnamespace(), job_id)) + if job: + status = job["status"] + if status == "success": + return render_template("samples/upload-success.html", + job=job, + species=species, + population=population,) + + if status == "error": + return redirect(url_for( + "species.populations.samples.upload_failure", job_id=job_id)) + + error_filename = Path(jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) + if error_filename.exists(): + stat = os.stat(error_filename) + if stat.st_size > 0: + return redirect(url_for( + "samples.upload_failure", job_id=job_id)) + + return render_template("samples/upload-progress.html", + species=species, + population=population, + job=job) # maybe also handle this? + + return render_template("no_such_job.html", + job_id=job_id, + species=species, + population=population), 400 + +@samplesbp.route("/upload/failure/<uuid:job_id>", methods=["GET"]) +@require_login +def upload_failure(job_id: uuid.UUID): + """Display the errors of the samples upload failure.""" + job = with_redis_connection(lambda rconn: jobs.job( + rconn, jobs.jobsnamespace(), job_id)) + if not bool(job): + return render_template("no_such_job.html", job_id=job_id), 400 + + error_filename = Path(jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) + if error_filename.exists(): + stat = os.stat(error_filename) + if stat.st_size > 0: + return render_template("worker_failure.html", job_id=job_id) + + return render_template("samples/upload-failure.html", job=job) |