diff options
| -rw-r--r-- | scripts/insert_samples.py | 16 | ||||
| -rw-r--r-- | uploader/background_jobs.py | 46 | ||||
| -rw-r--r-- | uploader/samples/views.py | 140 | ||||
| -rw-r--r-- | uploader/session.py | 5 | ||||
| -rw-r--r-- | uploader/templates/background-jobs/default-success-page.html | 17 | ||||
| -rw-r--r-- | uploader/templates/background-jobs/job-summary.html | 14 | ||||
| -rw-r--r-- | uploader/templates/base.html | 2 |
7 files changed, 61 insertions, 179 deletions
diff --git a/scripts/insert_samples.py b/scripts/insert_samples.py index fc029f9..96ae8e2 100644 --- a/scripts/insert_samples.py +++ b/scripts/insert_samples.py @@ -6,10 +6,10 @@ import argparse import traceback import MySQLdb as mdb -from redis import Redis + from gn_libs.mysqldb import database_connection -from uploader.check_connections import check_db, check_redis +from uploader.check_connections import check_db from uploader.species.models import species_by_id from uploader.population.models import population_by_id from uploader.samples.models import ( @@ -35,7 +35,6 @@ class SeparatorAction(argparse.Action): setattr(namespace, self.dest, (chr(9) if values == "\\t" else values)) def insert_samples(conn: mdb.Connection,# pylint: disable=[too-many-arguments, too-many-positional-arguments] - rconn: Redis,# pylint: disable=[unused-argument] speciesid: int, populationid: int, samplesfile: pathlib.Path, @@ -119,11 +118,6 @@ if __name__ == "__main__": help=("The character used to delimit (surround?) the value in " "each column.")) - # == Script-specific extras == - parser.add_argument("--redisuri", - help="URL to initialise connection to redis", - default="redis:///") - args = parser.parse_args() return args @@ -132,17 +126,13 @@ if __name__ == "__main__": status_code = 1 # Exit with an Exception args = cli_args() check_db(args.databaseuri) - check_redis(args.redisuri) if not args.samplesfile.exists(): logging.error("File not found: '%s'.", args.samplesfile) return 2 - with (Redis.from_url(args.redisuri, decode_responses=True) as rconn, - database_connection(args.databaseuri) as dbconn): - + with database_connection(args.databaseuri) as dbconn: try: status_code = insert_samples(dbconn, - rconn, args.speciesid, args.populationid, args.samplesfile, diff --git a/uploader/background_jobs.py b/uploader/background_jobs.py index 61e57f7..a71dd44 100644 --- a/uploader/background_jobs.py +++ b/uploader/background_jobs.py @@ -25,14 +25,27 @@ background_jobs_bp = Blueprint("background-jobs", __name__) HandlerType = Callable[[dict], Response] -def __default_error_handler__(job: dict) -> Response: - return redirect(url_for("background-jobs.job_summary", job_id=job["job_id"])) +def make_datetime_formatter(dtformat: str = "%A, %d %B %Y at %H:%M %Z") -> Callable[[str], str]: + """Make a datetime formatter with the provided `dtformat`""" + def __formatter__(val: str) -> str: + dt = datetime.datetime.fromisoformat(val) + return dt.strftime(dtformat.strip()) + + return __formatter__ + +__default_datetime_formatter__ = make_datetime_formatter() + + +def __default_handler__(_job): + return render_template("background-jobs/job-summary.html", + job=_job, + display_datetime=__default_datetime_formatter__) def register_handlers( job_type: str, success_handler: HandlerType, # pylint: disable=[redefined-outer-name] - error_handler: HandlerType = __default_error_handler__ + error_handler: HandlerType = __default_handler__ # pylint: disable=[redefined-outer-name] ) -> str: """Register success and error handlers for each job type.""" @@ -64,7 +77,7 @@ def register_job_handlers(job: dict): try: _error_handler = __load_handler__(metadata["error_handler"]) except Exception as _exc:# pylint: disable=[broad-exception-caught] - _error_handler = __default_error_handler__ + _error_handler = __default_handler__ register_handlers( metadata["job-type"], _success_handler, _error_handler) @@ -80,11 +93,7 @@ def handler(job: dict, handler_type: str) -> HandlerType: if bool(_handler): return _handler(job) - def __default_success_handler__(_job): - return render_template( - "background-jobs/default-success-page.html", job=_job) - - return __default_success_handler__ + return __default_handler__(job) error_handler = partial(handler, handler_type="error") @@ -109,7 +118,7 @@ def job_status(job_id: uuid.UUID): return render_template("background-jobs/job-status.html", job=job, - display_datetime=make_datetime_formatter()) + display_datetime=__default_datetime_formatter__) except JobNotFound as _jnf: return render_template("jobs/job-not-found.html", job_id=job_id) @@ -126,15 +135,6 @@ def job_error(job_id: uuid.UUID): return render_template("jobs/job-not-found.html", job_id=job_id) -def make_datetime_formatter(dtformat: str = "") -> Callable[[str], str]: - """Make a datetime formatter with the provided `dtformat`""" - def __formatter__(val: str) -> str: - dt = datetime.datetime.fromisoformat(val) - return dt.strftime(dtformat.strip() or "%A, %d %B %Y at %H:%M %Z") - - return __formatter__ - - @background_jobs_bp.route("/list") @require_login def list_jobs(): @@ -144,7 +144,7 @@ def list_jobs(): "background-jobs/list-jobs.html", jobs=jobs.jobs_by_external_id( conn, session.user_details()["user_id"]), - display_datetime=make_datetime_formatter()) + display_datetime=__default_datetime_formatter__) @background_jobs_bp.route("/summary/<uuid:job_id>") @@ -159,7 +159,7 @@ def job_summary(job_id: uuid.UUID): if status in ("completed", "error", "stopped"): return render_template("background-jobs/job-summary.html", job=job, - display_datetime=make_datetime_formatter()) + display_datetime=__default_datetime_formatter__) return redirect(url_for( "background-jobs.job_status", job_id=job["job_id"])) except JobNotFound as _jnf: @@ -182,7 +182,7 @@ def delete_single(job_id: uuid.UUID): if request.method == "GET": return render_template("background-jobs/delete-job.html", job=job, - display_datetime=make_datetime_formatter()) + display_datetime=__default_datetime_formatter__) if request.form["btn-confirm-delete"] == "delete": jobs.delete_job(conn, job_id) @@ -211,7 +211,7 @@ def stop_job(job_id: uuid.UUID): if request.method == "GET": return render_template("background-jobs/stop-job.html", job=job, - display_datetime=make_datetime_formatter()) + display_datetime=__default_datetime_formatter__) if request.form["btn-confirm-stop"] == "stop": jobs.kill_job(conn, job_id) diff --git a/uploader/samples/views.py b/uploader/samples/views.py index f8baf7e..f318bf0 100644 --- a/uploader/samples/views.py +++ b/uploader/samples/views.py @@ -2,16 +2,19 @@ import os import sys import uuid +import logging from pathlib import Path -from redis import Redis from flask import (flash, request, redirect, Blueprint, current_app as app) -from uploader import jobs +from gn_libs import sqlite3 +from gn_libs import jobs as jobs + +from uploader import session from uploader.files import save_file from uploader.flask_extensions import url_for from uploader.ui import make_template_renderer @@ -96,22 +99,6 @@ def list_samples(species: dict, population: dict, **kwargs):# pylint: disable=[u activelink="list-samples") -def build_sample_upload_job(# pylint: disable=[too-many-arguments, too-many-positional-arguments] - speciesid: int, - populationid: int, - samplesfile: Path, - separator: str, - firstlineheading: bool, - quotechar: str): - """Define the async command to run the actual samples data upload.""" - return [ - sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"], - str(speciesid), str(populationid), str(samplesfile.absolute()), - separator, f"--redisuri={app.config['REDIS_URL']}", - f"--quotechar={quotechar}" - ] + (["--firstlineheading"] if firstlineheading else []) - - @samplesbp.route("<int:species_id>/populations/<int:population_id>/upload-samples", methods=["GET", "POST"]) @require_login @@ -170,102 +157,29 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma quotechar = (request.form.get("field_delimiter", '"') or '"') - redisuri = app.config["REDIS_URL"] - with Redis.from_url(redisuri, decode_responses=True) as rconn: - #T0DO: Add a QC step here — what do we check? - # 1. Does any sample in the uploaded file exist within the database? - # If yes, what is/are its/their species and population? - # 2. If yes 1. above, provide error with notes on which species and - # populations already own the samples. - the_job = jobs.launch_job( + _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + with sqlite3.connection(_jobs_db) as conn: + job = jobs.launch_job( jobs.initialise_job( - rconn, - jobs.jobsnamespace(), + conn, str(uuid.uuid4()), - build_sample_upload_job( - species["SpeciesId"], - population["InbredSetId"], - samples_file, + [ + sys.executable, "-m", "scripts.insert_samples", + app.config["SQL_URI"], + str(species["SpeciesId"]), + str(population["InbredSetId"]), + str(samples_file.absolute()), separator, - firstlineheading, - quotechar), + f"--quotechar={quotechar}" + ] + (["--firstlineheading"] if firstlineheading else []), "samples_upload", - app.config["JOBS_TTL_SECONDS"], - {"job_name": f"Samples Upload: {samples_file.name}"}), - redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") - return redirect(url_for( - "species.populations.samples.upload_status", - species_id=species_id, - population_id=population_id, - job_id=the_job["jobid"])) - - -@samplesbp.route("<int:species_id>/populations/<int:population_id>/" - "upload-samples/status/<uuid:job_id>", - methods=["GET"]) -@require_login -@with_population(species_redirect_uri="species.populations.samples.index", - redirect_uri="species.populations.samples.select_population") -def upload_status(species: dict, population: dict, job_id: uuid.UUID, **kwargs):# pylint: disable=[unused-argument] - """Check on the status of a samples upload job.""" - job = with_redis_connection(lambda rconn: jobs.job( - rconn, jobs.jobsnamespace(), job_id)) - if job: - status = job["status"] - if status == "success": - return render_template("samples/upload-success.html", - job=job, - species=species, - population=population,) - - if status == "error": - return redirect(url_for( - "species.populations.samples.upload_failure", - species_id=species["SpeciesId"], - population_id=population["Id"], - job_id=job_id)) - - error_filename = Path(jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) - if error_filename.exists(): - stat = os.stat(error_filename) - if stat.st_size > 0: - return redirect(url_for( - "samples.upload_failure", job_id=job_id)) - - return render_template("samples/upload-progress.html", - species=species, - population=population, - job=job) # maybe also handle this? - - return render_template("no_such_job.html", - job_id=job_id, - species=species, - population=population), 400 - - -@samplesbp.route("<int:species_id>/populations/<int:population_id>/" - "upload-samples/failure/<uuid:job_id>", - methods=["GET"]) -@require_login -@with_population(species_redirect_uri="species.populations.samples.index", - redirect_uri="species.populations.samples.select_population") -def upload_failure(species: dict, population: dict, job_id: uuid.UUID, **kwargs):# pylint: disable=[unused-argument] - """Display the errors of the samples upload failure.""" - job = with_redis_connection(lambda rconn: jobs.job( - rconn, jobs.jobsnamespace(), job_id)) - if not bool(job): - return render_template("no_such_job.html", job_id=job_id), 400 - - error_filename = Path(jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) - if error_filename.exists(): - stat = os.stat(error_filename) - if stat.st_size > 0: - return render_template("worker_failure.html", job_id=job_id) - - return render_template("samples/upload-failure.html", - species=species, - population=population, - job=job) + extra_meta={ + "job_name": f"Samples Upload: {samples_file.name}" + }, + external_id=session.logged_in_user_id()), + _jobs_db, + Path(f"{app.config['UPLOAD_FOLDER']}/job_errors").absolute(), + loglevel=logging.getLevelName( + app.logger.getEffectiveLevel()).lower()) + return redirect( + url_for("background-jobs.job_status", job_id=job["job_id"])) diff --git a/uploader/session.py b/uploader/session.py index 9cb305b..1dcf8ac 100644 --- a/uploader/session.py +++ b/uploader/session.py @@ -25,8 +25,6 @@ class SessionInfo(TypedDict): session_id: UUID user: UserDetails anon_id: UUID - user_agent: str - ip_addr: str masquerade: Optional[UserDetails] auth_server_jwks: Optional[dict[str, Any]] @@ -69,9 +67,6 @@ def session_info() -> SessionInfo: "logged_in": False }, "anon_id": anon_id, - "user_agent": request.headers.get("User-Agent"), - "ip_addr": request.environ.get("HTTP_X_FORWARDED_FOR", - request.remote_addr), "masquerading": None })) diff --git a/uploader/templates/background-jobs/default-success-page.html b/uploader/templates/background-jobs/default-success-page.html deleted file mode 100644 index 5732456..0000000 --- a/uploader/templates/background-jobs/default-success-page.html +++ /dev/null @@ -1,17 +0,0 @@ -{%extends "phenotypes/base.html"%} -{%from "flash_messages.html" import flash_all_messages%} - -{%block title%}Background Jobs: Success{%endblock%} - -{%block pagetitle%}Background Jobs: Success{%endblock%} - -{%block contents%} -{{flash_all_messages()}} - -<div class="row"> - <p>Job <strong>{{job.job_id}}</strong>, - {%if job.get("metadata", {}).get("job-type")%} - of type '<em>{{job.metadata["job-type"]}}</em> - {%endif%}' completed successfully.</p> -</div> -{%endblock%} diff --git a/uploader/templates/background-jobs/job-summary.html b/uploader/templates/background-jobs/job-summary.html index fe62d5d..c2c2d6b 100644 --- a/uploader/templates/background-jobs/job-summary.html +++ b/uploader/templates/background-jobs/job-summary.html @@ -25,18 +25,13 @@ </div> <div class="row"> + {%if view_under_construction%} <div class="col"> <a href="#" class="btn btn-info not-implemented" title="Update the expiry date and time for this job.">update expiry</a> </div> - <div class="col"> - <a href="{{url_for('background-jobs.delete_single', job_id=job.job_id)}}" - class="btn btn-danger" - title="Delete this job.">delete</a> - </div> - {%if job.metadata.status in ("stopped",)%} <div class="col"> <a href="#" @@ -44,6 +39,13 @@ title="Create a new copy of this job, and run the copy.">Run Copy</a> </div> {%endif%} + {%endif%} + + <div class="col"> + <a href="{{url_for('background-jobs.delete_single', job_id=job.job_id)}}" + class="btn btn-danger" + title="Delete this job.">delete</a> + </div> </div> <div class="row"> diff --git a/uploader/templates/base.html b/uploader/templates/base.html index dd097de..ae4ecef 100644 --- a/uploader/templates/base.html +++ b/uploader/templates/base.html @@ -32,7 +32,6 @@ <nav id="header-nav"> <ul class="nav"> {%if user_logged_in()%} - {%if view_under_construction%} <li> <a href="{{url_for('background-jobs.list_jobs')}}" title="User's background jobs."> @@ -43,7 +42,6 @@ Background jobs </a> </li> - {%endif%} <li> <a href="{{url_for('oauth2.logout')}}" |