diff options
| -rw-r--r-- | qc_app/templates/base.html | 2 | ||||
| -rw-r--r-- | qc_app/templates/rqtl2/rqtl2-qc-job-error.html | 32 | ||||
| -rw-r--r-- | qc_app/templates/rqtl2/rqtl2-qc-job-status.html | 29 | ||||
| -rw-r--r-- | qc_app/templates/rqtl2/rqtl2-qc-job-success.html | 37 | ||||
| -rw-r--r-- | qc_app/upload/rqtl2.py | 63 | ||||
| -rw-r--r-- | scripts/qc_on_rqtl2_bundle.py | 42 |
6 files changed, 186 insertions, 19 deletions
diff --git a/qc_app/templates/base.html b/qc_app/templates/base.html index 3b701bc..2228356 100644 --- a/qc_app/templates/base.html +++ b/qc_app/templates/base.html @@ -6,7 +6,7 @@ <meta name="viewport" content="width=device-width, initial-scale=1.0" /> {%block extrameta%}{%endblock%} - <title>QC: {%block title%}{%endblock%}</title> + <title>GN Uploader: {%block title%}{%endblock%}</title> <link rel="stylesheet" type="text/css" href="/static/css/styles.css" /> {%block css%}{%endblock%} diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html new file mode 100644 index 0000000..5d2ebee --- /dev/null +++ b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html @@ -0,0 +1,32 @@ +{%extends "base.html"%} +{%from "cli-output.html" import cli_output%} + +{%block title%}R/qtl2 bundle: QC Job Error{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 bundle: QC job Error</h1> + +<div class="explainer"> + <p>The R/qtl2 bundle has failed some <emph>Quality Control</emph> checks.</p> + <p>We list below some of the errors that need to be fixed before the data can + be uploaded onto GeneNetwork.</p> +</div> + +<h2 class="heading">Errors</h2> + +<p><emph>list errors here by file type, I think …</emph></p> + +<h4>stdout</h4> +{{cli_output(job, "stdout")}} + +<h4>stderr</h4> +{{cli_output(job, "stderr")}} + +<h4>Log</h4> +<div class="cli-output"> + {%for msg in messages%} + {{msg}}<br /> + {%endfor%} +</div> + +{%endblock%} diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-status.html b/qc_app/templates/rqtl2/rqtl2-qc-job-status.html new file mode 100644 index 0000000..85b8864 --- /dev/null +++ b/qc_app/templates/rqtl2/rqtl2-qc-job-status.html @@ -0,0 +1,29 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_messages%} + +{%block title%}Job Status{%endblock%} + +{%block extrameta%} +<meta http-equiv="refresh" content="3"> +{%endblock%} + +{%block contents%} +<h1 class="heading">R/qtl2 bundle: QC job status</h1> + +<h2 class="heading">R/qtl2 bundle: QC Job Status</h2> + +<hr /> +<p>The job:</p> +<hr /> +{{job}} +<hr /> +<hr /> + +<h4>Log</h4> +<div class="cli-output"> + {%for msg in messages%} + {{msg}}<br /> + {%endfor%} +</div> + +{%endblock%} diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-success.html b/qc_app/templates/rqtl2/rqtl2-qc-job-success.html new file mode 100644 index 0000000..396f241 --- /dev/null +++ b/qc_app/templates/rqtl2/rqtl2-qc-job-success.html @@ -0,0 +1,37 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}R/qtl2 Bundle: Quality Control Successful{%endblock%} + +{%block contents%} +<h2 class="heading">R/qtl2 Bundle: Quality Control Successful</h2> + +<div class="explainer"> + <p>The R/qtl2 bundle you uploaded has passed <emph>all</emph> quality control + checks successfully, and is now ready for uploading into the database.</p> + <p>Click "Continue" below to proceed.</p> +</div> + +<!-- + The "action" on this form takes us to the next step, where we can + select all the other data necessary to enter the data into the database. + --> +<form id="frm-upload-rqtl2-bundle" + action="{{url_for('upload.rqtl2.select_dataset_info', + species_id=species.SpeciesId, + population_id=population.InbredSetId)}}" + method="POST" + enctype="multipart/form-data"> + {{flash_all_messages()}} + <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> + <input type="hidden" name="population_id" + value="{{population.InbredSetId}}" /> + <input type="hidden" name="rqtl2_bundle_file" + value="{{rqtl2_bundle_file}}" /> + + <fieldset> + <input type="submit" value="continue" class="btn btn-main form-col-2" /> + </fieldset> +</form> + +{%endblock%} diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py index 9d787e1..afe0c2e 100644 --- a/qc_app/upload/rqtl2.py +++ b/qc_app/upload/rqtl2.py @@ -18,8 +18,6 @@ from flask import ( current_app as app) from r_qtl import r_qtl2 -from r_qtl import r_qtl2_qc as rqc -from r_qtl.errors import InvalidFormat from qc_app import jobs from qc_app.files import save_file, fullpath @@ -135,12 +133,6 @@ class __RequestError__(Exception): #pylint: disable=[invalid-name] methods=["GET", "POST"]) def upload_rqtl2_bundle(species_id: int, population_id: int): """Allow upload of R/qtl2 bundle.""" - this_page_with_errors = redirect(url_for("upload.rqtl2.upload_rqtl2_bundle", - species_id=species_id, - population_id=population_id, - pgsrc="error"), - code=307) - with database_connection(app.config["SQL_URI"]) as conn: species = species_by_id(conn, species_id) population = population_by_species_and_id( @@ -171,17 +163,52 @@ def upload_rqtl2_bundle(species_id: int, population_id: int): if not is_zipfile(str(the_file)): raise __RequestError__("Invalid file! Expected a zip file.") + redisuri = app.config["REDIS_URL"] + with Redis.from_url(redisuri, decode_responses=True) as rconn: + jobid = str(uuid4()) + redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"] + jobs.launch_job( + jobs.initialise_job( + rconn, + jobs.jobsnamespace(), + jobid, + [sys.executable, "-m", "scripts.qc_on_rqtl2_bundle", + app.config["SQL_URI"], app.config["REDIS_URL"], + jobs.jobsnamespace(), jobid, "--redisexpiry", + str(redis_ttl_seconds)], + "rqtl2-bundle-qc-job", + redis_ttl_seconds, + {}), + redisuri, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + return redirect(url_for( + "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid)) + +@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>", + methods=["GET", "POST"]) +def rqtl2_bundle_qc_status(jobid: UUID): + """Check the status of the QC jobs.""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: try: - with ZipFile(str(the_file), "r") as zfile: - rqc.validate_bundle(zfile) - return render_template( - "rqtl2/upload-rqtl2-bundle-step-02.html", - species=species, - population=population, - rqtl2_bundle_file=the_file.name)#type: ignore[union-attr] - except (InvalidFormat, __RequestError__) as exc: - flash("".join(exc.args), "alert-error error-rqtl2") - return this_page_with_errors + thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid) + messagelistname = thejob.get("log-messagelist") + logmessages = (rconn.lrange(messagelistname, 0, -1) + if bool(messagelistname) else []) + jobstatus = thejob["status"] + if jobstatus == "error": + return render_template("rqtl2/rqtl2-qc-job-error.html", + job=thejob, + messages=logmessages) + if jobstatus == "success": + return render_template("rqtl2/rqtl2-qc-job-results.html", + job=thejob, + messages=logmessages) + + return render_template("rqtl2/rqtl2-qc-job-status.html", + job=thejob, + messages=tuple()) + except jobs.JobNotFound: + return render_template("rqtl2/no-such-job.html", jobid=jobid) def check_errors(conn, *args, **kwargs):#pylint: disable=[too-many-return-statements] """Check for select errors in the forms and return a page to redirect to.""" diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py new file mode 100644 index 0000000..5802f3a --- /dev/null +++ b/scripts/qc_on_rqtl2_bundle.py @@ -0,0 +1,42 @@ +"""Run Quality Control checks on R/qtl2 bundle.""" +import sys +from logging import getLogger, StreamHandler + +from redis import Redis + +from qc_app import jobs +from qc_app.db_utils import database_connection +from qc_app.check_connections import check_db, check_redis + +from scripts.cli_parser import init_cli_parser +from scripts.redis_logger import setup_redis_logger + +def run_qc(_args): + """Run the QC programs.""" + return 1 + +if __name__ == "__main__": + def main(): + """Enter R/qtl2 bundle QC runner.""" + args = init_cli_parser( + "qc-on-rqtl2-bundle", "Run QC on R/qtl2 bundle.").parse_args() + check_redis(args.redisuri) + check_db(args.databaseuri) + + logger = getLogger("qc-on-rqtl2-bundle") + logger.addHandler(StreamHandler(stream=sys.stderr)) + logger.setLevel("DEBUG") + + fqjobid = jobs.job_key(args.redisprefix, args.jobid) + with (database_connection(args.databaseuri) as _dbconn, + Redis.from_url(args.redisuri, decode_responses=True) as rconn): + logger.addHandler(setup_redis_logger( + rconn, fqjobid, f"{fqjobid}:log-messages", + args.redisexpiry)) + + exitcode = run_qc(args) + rconn.hset( + jobs.job_key(args.redisprefix, args.jobid), "exitcode", exitcode) + return exitcode + + sys.exit(main()) |