diff options
Diffstat (limited to 'uploader/samples')
| -rw-r--r-- | uploader/samples/models.py | 13 | ||||
| -rw-r--r-- | uploader/samples/views.py | 209 |
2 files changed, 63 insertions, 159 deletions
diff --git a/uploader/samples/models.py b/uploader/samples/models.py index d7d5384..1e9293f 100644 --- a/uploader/samples/models.py +++ b/uploader/samples/models.py @@ -15,11 +15,11 @@ def samples_by_species_and_population( """Fetch the samples by their species and population.""" with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute( - "SELECT iset.InbredSetId, s.* FROM InbredSet AS iset " - "INNER JOIN StrainXRef AS sxr ON iset.InbredSetId=sxr.InbredSetId " - "INNER JOIN Strain AS s ON sxr.StrainId=s.Id " - "WHERE s.SpeciesId=%(species_id)s " - "AND iset.InbredSetId=%(population_id)s", + "SELECT InbredSet.InbredSetId, Strain.* FROM InbredSet " + "INNER JOIN StrainXRef ON InbredSet.InbredSetId=StrainXRef.InbredSetId " + "INNER JOIN Strain ON StrainXRef.StrainId=Strain.Id " + "WHERE Strain.SpeciesId=%(species_id)s " + "AND InbredSet.InbredSetId=%(population_id)s", {"species_id": species_id, "population_id": population_id}) return tuple(cursor.fetchall()) @@ -34,8 +34,7 @@ def read_samples_file(filepath, separator: str, firstlineheading: bool, **kwargs else ("Name", "Name2", "Symbol", "Alias")), delimiter=separator, quotechar=kwargs.get("quotechar", '"')) - for row in reader: - yield row + yield from reader def save_samples_data(conn: mdb.Connection, diff --git a/uploader/samples/views.py b/uploader/samples/views.py index ed79101..f318bf0 100644 --- a/uploader/samples/views.py +++ b/uploader/samples/views.py @@ -2,30 +2,32 @@ import os import sys import uuid +import logging from pathlib import Path -from redis import Redis from flask import (flash, request, - url_for, redirect, Blueprint, current_app as app) -from uploader import jobs +from gn_libs import sqlite3 +from gn_libs import jobs as jobs + +from uploader import session from uploader.files import save_file +from uploader.flask_extensions import url_for from uploader.ui import make_template_renderer from uploader.authorisation import require_login -from uploader.request_checks import with_population from uploader.input_validation import is_integer_input -from uploader.datautils import safe_int, order_by_family, enumerate_sequence -from uploader.population.models import population_by_id, populations_by_species +from uploader.population.models import population_by_id +from uploader.route_utils import generic_select_population +from uploader.datautils import safe_int, enumerate_sequence +from uploader.species.models import all_species, species_by_id +from uploader.request_checks import with_species, with_population from uploader.db_utils import (with_db_connection, database_connection, with_redis_connection) -from uploader.species.models import (all_species, - species_by_id, - order_species_by_family) from .models import samples_by_species_and_population @@ -40,8 +42,15 @@ def index(): if not bool(request.args.get("species_id")): return render_template( "samples/index.html", - species=order_species_by_family(all_species(conn)), + species=all_species(conn), activelink="samples") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.samples.select_population")) + species = species_by_id(conn, request.args.get("species_id")) if not bool(species): flash("No such species!", "alert-danger") @@ -52,57 +61,31 @@ def index(): @samplesbp.route("<int:species_id>/samples/select-population", methods=["GET"]) @require_login -def select_population(species_id: int): +@with_species(redirect_uri="species.populations.samples.index") +def select_population(species: dict, **kwargs):# pylint: disable=[unused-argument] """Select the population to use for the samples.""" - with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - if not bool(species): - flash("Invalid species!", "alert-danger") - return redirect(url_for("species.populations.samples.index")) - - if not bool(request.args.get("population_id")): - return render_template("samples/select-population.html", - species=species, - populations=order_by_family( - populations_by_species( - conn, - species_id), - order_key="FamilyOrder"), - activelink="samples") - - population = population_by_id(conn, request.args.get("population_id")) - if not bool(population): - flash("Population not found!", "alert-danger") - return redirect(url_for( - "species.populations.samples.select_population", - species_id=species_id)) - - return redirect(url_for("species.populations.samples.list_samples", - species_id=species_id, - population_id=population["Id"])) + return generic_select_population( + species, + "samples/select-population.html", + request.args.get("population_id") or "", + "species.populations.samples.select_population", + "species.populations.samples.list_samples", + "samples", + "Population not found!") @samplesbp.route("<int:species_id>/populations/<int:population_id>/samples") @require_login -def list_samples(species_id: int, population_id: int): +@with_population( + species_redirect_uri="species.populations.samples.index", + redirect_uri="species.populations.samples.select_population") +def list_samples(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """ List the samples in a particular population and give the ability to upload new ones. """ with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - if not bool(species): - flash("Invalid species!", "alert-danger") - return redirect(url_for("species.populations.samples.index")) - - population = population_by_id(conn, population_id) - if not bool(population): - flash("Population not found!", "alert-danger") - return redirect(url_for( - "species.populations.samples.select_population", - species_id=species_id)) - all_samples = enumerate_sequence(samples_by_species_and_population( - conn, species_id, population_id)) + conn, species["SpeciesId"], population["Id"])) total_samples = len(all_samples) offset = max(safe_int(request.args.get("from") or 0), 0) count = int(request.args.get("count") or 20) @@ -116,22 +99,6 @@ def list_samples(species_id: int, population_id: int): activelink="list-samples") -def build_sample_upload_job(# pylint: disable=[too-many-arguments] - speciesid: int, - populationid: int, - samplesfile: Path, - separator: str, - firstlineheading: bool, - quotechar: str): - """Define the async command to run the actual samples data upload.""" - return [ - sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"], - str(speciesid), str(populationid), str(samplesfile.absolute()), - separator, f"--redisuri={app.config['REDIS_URL']}", - f"--quotechar={quotechar}" - ] + (["--firstlineheading"] if firstlineheading else []) - - @samplesbp.route("<int:species_id>/populations/<int:population_id>/upload-samples", methods=["GET", "POST"]) @require_login @@ -179,7 +146,7 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma "alert-error") return samples_uploads_page - firstlineheading = (request.form.get("first_line_heading") == "on") + firstlineheading = request.form.get("first_line_heading") == "on" separator = request.form.get("separator", ",") if separator == "other": @@ -190,91 +157,29 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma quotechar = (request.form.get("field_delimiter", '"') or '"') - redisuri = app.config["REDIS_URL"] - with Redis.from_url(redisuri, decode_responses=True) as rconn: - #TODO: Add a QC step here — what do we check? - # 1. Does any sample in the uploaded file exist within the database? - # If yes, what is/are its/their species and population? - # 2. If yes 1. above, provide error with notes on which species and - # populations already own the samples. - the_job = jobs.launch_job( + _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + with sqlite3.connection(_jobs_db) as conn: + job = jobs.launch_job( jobs.initialise_job( - rconn, - jobs.jobsnamespace(), + conn, str(uuid.uuid4()), - build_sample_upload_job( - species["SpeciesId"], - population["InbredSetId"], - samples_file, + [ + sys.executable, "-m", "scripts.insert_samples", + app.config["SQL_URI"], + str(species["SpeciesId"]), + str(population["InbredSetId"]), + str(samples_file.absolute()), separator, - firstlineheading, - quotechar), + f"--quotechar={quotechar}" + ] + (["--firstlineheading"] if firstlineheading else []), "samples_upload", - app.config["JOBS_TTL_SECONDS"], - {"job_name": f"Samples Upload: {samples_file.name}"}), - redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") - return redirect(url_for( - "species.populations.samples.upload_status", - species_id=species_id, - population_id=population_id, - job_id=the_job["jobid"])) - - -@samplesbp.route("<int:species_id>/populations/<int:population_id>/" - "upload-samples/status/<uuid:job_id>", - methods=["GET"]) -@require_login -@with_population(species_redirect_uri="species.populations.samples.index", - redirect_uri="species.populations.samples.select_population") -def upload_status(species: dict, population: dict, job_id: uuid.UUID, **kwargs):# pylint: disable=[unused-argument] - """Check on the status of a samples upload job.""" - job = with_redis_connection(lambda rconn: jobs.job( - rconn, jobs.jobsnamespace(), job_id)) - if job: - status = job["status"] - if status == "success": - return render_template("samples/upload-success.html", - job=job, - species=species, - population=population,) - - if status == "error": - return redirect(url_for( - "species.populations.samples.upload_failure", job_id=job_id)) - - error_filename = Path(jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) - if error_filename.exists(): - stat = os.stat(error_filename) - if stat.st_size > 0: - return redirect(url_for( - "samples.upload_failure", job_id=job_id)) - - return render_template("samples/upload-progress.html", - species=species, - population=population, - job=job) # maybe also handle this? - - return render_template("no_such_job.html", - job_id=job_id, - species=species, - population=population), 400 - -@samplesbp.route("/upload/failure/<uuid:job_id>", methods=["GET"]) -@require_login -def upload_failure(job_id: uuid.UUID): - """Display the errors of the samples upload failure.""" - job = with_redis_connection(lambda rconn: jobs.job( - rconn, jobs.jobsnamespace(), job_id)) - if not bool(job): - return render_template("no_such_job.html", job_id=job_id), 400 - - error_filename = Path(jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) - if error_filename.exists(): - stat = os.stat(error_filename) - if stat.st_size > 0: - return render_template("worker_failure.html", job_id=job_id) - - return render_template("samples/upload-failure.html", job=job) + extra_meta={ + "job_name": f"Samples Upload: {samples_file.name}" + }, + external_id=session.logged_in_user_id()), + _jobs_db, + Path(f"{app.config['UPLOAD_FOLDER']}/job_errors").absolute(), + loglevel=logging.getLevelName( + app.logger.getEffectiveLevel()).lower()) + return redirect( + url_for("background-jobs.job_status", job_id=job["job_id"])) |