diff options
| -rw-r--r-- | qc_app/static/css/styles.css | 8 | ||||
| -rw-r--r-- | qc_app/templates/rqtl2/rqtl2-qc-job-error.html | 14 | ||||
| -rw-r--r-- | qc_app/upload/rqtl2.py | 13 | ||||
| -rw-r--r-- | scripts/qc_on_rqtl2_bundle.py | 40 |
4 files changed, 65 insertions, 10 deletions
diff --git a/qc_app/static/css/styles.css b/qc_app/static/css/styles.css index f6e829f..474c7f7 100644 --- a/qc_app/static/css/styles.css +++ b/qc_app/static/css/styles.css @@ -198,3 +198,11 @@ form fieldset legend { width: 90%; overflow: scroll; } + +.qc-error-display { + border-radius: 0.8em; + padding-top: 0.5em; + padding-bottom: 0.5em; + max-height: 250px; + overflow: scroll; +} diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html index 5d2ebee..f9a912c 100644 --- a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html +++ b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html @@ -12,9 +12,19 @@ be uploaded onto GeneNetwork.</p> </div> -<h2 class="heading">Errors</h2> +{%if errorsgeneric | length > 0%} +<h2 class="heading">Generic Errors ({{errorsgeneric | length}})</h3> +<div class="explainer"> + We found the following generic errors in your R/qtl2 bundle: +</div> +<ul class="alert-error qc-error-display"> + {%for msg in errorsgeneric%} + <li>{{msg}}</li> + {%endfor%} +</ul> +{%endif%} -<p><emph>list errors here by file type, I think …</emph></p> +<p><emph>list other errors here by file type, I think …</emph></p> <h4>stdout</h4> {{cli_output(job, "stdout")}} diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py index b96d9f0..c20bd8f 100644 --- a/qc_app/upload/rqtl2.py +++ b/qc_app/upload/rqtl2.py @@ -175,12 +175,15 @@ def upload_rqtl2_bundle(species_id: int, population_id: int): jobs.jobsnamespace(), jobid, [sys.executable, "-m", "scripts.qc_on_rqtl2_bundle", - app.config["SQL_URI"], app.config["REDIS_URL"], - jobs.jobsnamespace(), jobid, "--redisexpiry", - str(redis_ttl_seconds)], + app.config["SQL_URI"], app.config["REDIS_URL"], + jobs.jobsnamespace(), jobid, "--redisexpiry", + str(redis_ttl_seconds)], "rqtl2-bundle-qc-job", redis_ttl_seconds, - {}), + {"job-metadata": json.dumps({ + "speciesid": species_id, + "populationid": population_id, + "rqtl2-bundle-file": str(the_file.absolute())})}), redisuri, f"{app.config['UPLOAD_FOLDER']}/job_errors") return redirect(url_for( @@ -200,6 +203,8 @@ def rqtl2_bundle_qc_status(jobid: UUID): if jobstatus == "error": return render_template("rqtl2/rqtl2-qc-job-error.html", job=thejob, + errorsgeneric=json.loads( + thejob.get("errors-generic", "[]")), messages=logmessages) if jobstatus == "success": return render_template("rqtl2/rqtl2-qc-job-results.html", diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py index 5802f3a..8dd6b7e 100644 --- a/scripts/qc_on_rqtl2_bundle.py +++ b/scripts/qc_on_rqtl2_bundle.py @@ -1,6 +1,10 @@ """Run Quality Control checks on R/qtl2 bundle.""" import sys -from logging import getLogger, StreamHandler +import json +from typing import Sequence +from zipfile import ZipFile +from argparse import Namespace +from logging import Logger, getLogger, StreamHandler from redis import Redis @@ -8,12 +12,40 @@ from qc_app import jobs from qc_app.db_utils import database_connection from qc_app.check_connections import check_db, check_redis +from r_qtl import r_qtl2_qc as rqc +from r_qtl import fileerrors as rqfe + from scripts.cli_parser import init_cli_parser +from scripts.process_rqtl2_bundle import parse_job from scripts.redis_logger import setup_redis_logger -def run_qc(_args): +def add_to_errors(rconn: Redis, fqjobid: str, key: str, errors: Sequence[rqfe.MissingFile]): + """Add `errors` to a given list of errors""" + errs = tuple(set( + json.loads(rconn.hget(fqjobid, key) or "[]") + + [error.message for error in errors])) + rconn.hset(fqjobid, key, json.dumps(errs)) + +def run_qc(rconn: Redis, args: Namespace, logger: Logger) -> int: """Run the QC programs.""" - return 1 + fqjobid = jobs.job_key(args.redisprefix, args.jobid) + thejob = parse_job(rconn, args.redisprefix, args.jobid) + meta = thejob["job-metadata"] + with ZipFile(meta["rqtl2-bundle-file"], "r") as zfile: + missing = rqc.missing_files(zfile) + add_to_errors(rconn, fqjobid, "errors-generic", tuple( + rqfe.MissingFile( + mfile[0], mfile[1], ( + f"File '{mfile[1]}' is listed in the control file under " + f"the '{mfile[0]}' key, but it does not actually exist in " + "the bundle.")) + for mfile in missing)) + + if len(missing) > 0: + logger.error("Missing files in the bundle!") + return 1 + + return 0 if __name__ == "__main__": def main(): @@ -34,7 +66,7 @@ if __name__ == "__main__": rconn, fqjobid, f"{fqjobid}:log-messages", args.redisexpiry)) - exitcode = run_qc(args) + exitcode = run_qc(rconn, args, logger) rconn.hset( jobs.job_key(args.redisprefix, args.jobid), "exitcode", exitcode) return exitcode |