path: root/uploader/samples/views.py

"""Code regarding samples"""
import os
import sys
import uuid
from pathlib import Path

from redis import Redis
from flask import (flash,
                   request,
                   url_for,
                   redirect,
                   Blueprint,
                   current_app as app)

from uploader import jobs
from uploader.files import save_file
from uploader.ui import make_template_renderer
from uploader.authorisation import require_login
from uploader.input_validation import is_integer_input
from uploader.datautils import safe_int, order_by_family, enumerate_sequence
from uploader.population.models import population_by_id, populations_by_species
from uploader.db_utils import (with_db_connection,
                               database_connection,
                               with_redis_connection)
from uploader.species.models import (all_species,
                                     species_by_id,
                                     order_species_by_family)
from uploader.population.models import(save_population,
                                       population_by_id,
                                       populations_by_species,
                                       population_by_species_and_id)

from .models import samples_by_species_and_population

samplesbp = Blueprint("samples", __name__)
render_template = make_template_renderer("samples")

@samplesbp.route("/samples", methods=["GET"])
@require_login
def index():
    """Direct entry-point for uploading/handling the samples."""
    with database_connection(app.config["SQL_URI"]) as conn:
        if not bool(request.args.get("species_id")):
            return render_template(
                "samples/index.html",
                species=order_species_by_family(all_species(conn)),
                activelink="samples")
        species = species_by_id(conn, request.args.get("species_id"))
        if not bool(species):
            flash("No such species!", "alert-danger")
            return redirect(url_for("species.populations.samples.index"))
        return redirect(url_for("species.populations.samples.select_population",
                                species_id=species["SpeciesId"]))


@samplesbp.route("<int:species_id>/samples/select-population", methods=["GET"])
@require_login
def select_population(species_id: int):
    """Select the population to use for the samples."""
    with database_connection(app.config["SQL_URI"]) as conn:
        species = species_by_id(conn, species_id)
        if not bool(species):
            flash("Invalid species!", "alert-danger")
            return redirect(url_for("species.populations.samples.index"))

        if not bool(request.args.get("population_id")):
            return render_template("samples/select-population.html",
                                   species=species,
                                   populations=order_by_family(
                                       populations_by_species(
                                           conn,
                                           species_id),
                                       order_key="FamilyOrder"),
                                   activelink="samples")

        population = population_by_id(conn, request.args.get("population_id"))
        if not bool(population):
            flash("Population not found!", "alert-danger")
            return redirect(url_for(
                "species.populations.samples.select_population",
                species_id=species_id))

        return redirect(url_for("species.populations.samples.list_samples",
                                species_id=species_id,
                                population_id=population["Id"]))

@samplesbp.route("<int:species_id>/populations/<int:population_id>/samples")
@require_login
def list_samples(species_id: int, population_id: int):
    """
    List the samples in a particular population and give the ability to upload
    new ones.
    """
    with database_connection(app.config["SQL_URI"]) as conn:
        species = species_by_id(conn, species_id)
        if not bool(species):
            flash("Invalid species!", "alert-danger")
            return redirect(url_for("species.populations.samples.index"))

        population = population_by_id(conn, population_id)
        if not bool(population):
            flash("Population not found!", "alert-danger")
            return redirect(url_for(
                "species.populations.samples.select_population",
                species_id=species_id))

        all_samples = enumerate_sequence(samples_by_species_and_population(
            conn, species_id, population_id))
        total_samples = len(all_samples)
        offset = max(safe_int(request.args.get("from") or 0), 0)
        count = int(request.args.get("count") or 20)
        return render_template("samples/list-samples.html",
                               species=species,
                               population=population,
                               samples=all_samples[offset:offset+count],
                               offset=offset,
                               count=count,
                               total_samples=total_samples,
                               activelink="list-samples")


def build_sample_upload_job(# pylint: disable=[too-many-arguments]
        speciesid: int,
        populationid: int,
        samplesfile: Path,
        separator: str,
        firstlineheading: bool,
        quotechar: str):
    """Define the async command to run the actual samples data upload."""
    return [
        sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"],
        str(speciesid), str(populationid), str(samplesfile.absolute()),
        separator, f"--redisuri={app.config['REDIS_URL']}",
        f"--quotechar={quotechar}"
    ] + (["--firstlineheading"] if firstlineheading else [])


@samplesbp.route("<int:species_id>/populations/<int:population_id>/upload-samples",
               methods=["GET", "POST"])
@require_login
def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements]
    """Upload the samples."""
    samples_uploads_page = redirect(url_for(
        "species.populations.samples.upload_samples",
        species_id=species_id,
        population_id=population_id))
    if not is_integer_input(species_id):
        flash("You did not provide a valid species. Please select one to "
              "continue.",
              "alert-danger")
        return redirect(url_for("expression-data.samples.select_species"))
    species = with_db_connection(lambda conn: species_by_id(conn, species_id))
    if not bool(species):
        flash("Species with given ID was not found.", "alert-danger")
        return redirect(url_for("expression-data.samples.select_species"))

    if not is_integer_input(population_id):
        flash("You did not provide a valid population. Please select one "
              "to continue.",
              "alert-danger")
        return redirect(url_for("species.populations.samples.select_population",
                                species_id=species_id),
                        code=307)
    population = with_db_connection(
        lambda conn: population_by_id(conn, int(population_id)))
    if not bool(population):
        flash("Invalid grouping/population!", "alert-error")
        return redirect(url_for("species.populations.samples.select_population",
                                species_id=species_id),
                        code=307)

    if request.method == "GET" or request.files.get("samples_file") is None:
        return render_template("samples/upload-samples.html",
                               species=species,
                               population=population)

    try:
        samples_file = save_file(request.files["samples_file"],
                                 Path(app.config["UPLOAD_FOLDER"]))
    except AssertionError:
        flash("You need to provide a file with the samples data.",
              "alert-error")
        return samples_uploads_page

    firstlineheading = (request.form.get("first_line_heading") == "on")

    separator = request.form.get("separator", ",")
    if separator == "other":
        separator = request.form.get("other_separator", ",")
    if not bool(separator):
        flash("You need to provide a separator character.", "alert-error")
        return samples_uploads_page

    quotechar = (request.form.get("field_delimiter", '"') or '"')

    redisuri = app.config["REDIS_URL"]
    with Redis.from_url(redisuri, decode_responses=True) as rconn:
        #TODO: Add a QC step here — what do we check?
        # 1. Does any sample in the uploaded file exist within the database?
        #    If yes, what is/are its/their species and population?
        # 2. If yes 1. above, provide error with notes on which species and
        #    populations already own the samples.
        the_job = jobs.launch_job(
            jobs.initialise_job(
                rconn,
                jobs.jobsnamespace(),
                str(uuid.uuid4()),
                build_sample_upload_job(
                    species["SpeciesId"],
                    population["InbredSetId"],
                    samples_file,
                    separator,
                    firstlineheading,
                    quotechar),
                "samples_upload",
                app.config["JOBS_TTL_SECONDS"],
                {"job_name": f"Samples Upload: {samples_file.name}"}),
            redisuri,
            f"{app.config['UPLOAD_FOLDER']}/job_errors")
        return redirect(url_for(
            "species.populations.samples.upload_status",
            species_id=species_id,
            population_id=population_id,
            job_id=the_job["jobid"]))


@samplesbp.route("<int:species_id>/populations/<int:population_id>/"
                 "upload-samples/status/<uuid:job_id>",
                 methods=["GET"])
@require_login
def upload_status(species_id: int, population_id: int, job_id: uuid.UUID):
    """Check on the status of a samples upload job."""
    with database_connection(app.config["SQL_URI"]) as conn:
        species = species_by_id(conn, species_id)
        if not bool(species):
            flash("You must provide a valid species.", "alert-danger")
            return redirect(url_for("species.populations.samples.index"))

        population = population_by_species_and_id(
            conn, species_id, population_id)
        if not bool(population):
            flash("You must provide a valid population.", "alert-danger")
            return redirect(url_for(
                "species.populations.samples.select_population",
                species_id=species_id))

        job = with_redis_connection(lambda rconn: jobs.job(
            rconn, jobs.jobsnamespace(), job_id))
        if job:
            status = job["status"]
            if status == "success":
                return render_template("samples/upload-success.html",
                                       job=job,
                                       species=species,
                                       population=population,)

            if status == "error":
                return redirect(url_for(
                    "species.populations.samples.upload_failure", job_id=job_id))

            error_filename = Path(jobs.error_filename(
                job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors"))
            if error_filename.exists():
                stat = os.stat(error_filename)
                if stat.st_size > 0:
                    return redirect(url_for(
                        "samples.upload_failure", job_id=job_id))

            return render_template("samples/upload-progress.html",
                                   species=species,
                                   population=population,
                                   job=job) # maybe also handle this?

        return render_template("no_such_job.html",
                               job_id=job_id,
                               species=species,
                               population=population), 400

@samplesbp.route("/upload/failure/<uuid:job_id>", methods=["GET"])
@require_login
def upload_failure(job_id: uuid.UUID):
    """Display the errors of the samples upload failure."""
    job = with_redis_connection(lambda rconn: jobs.job(
        rconn, jobs.jobsnamespace(), job_id))
    if not bool(job):
        return render_template("no_such_job.html", job_id=job_id), 400

    error_filename = Path(jobs.error_filename(
        job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors"))
    if error_filename.exists():
        stat = os.stat(error_filename)
        if stat.st_size > 0:
            return render_template("worker_failure.html", job_id=job_id)

    return render_template("samples/upload-failure.html", job=job)