diff options
| -rw-r--r-- | qc_app/static/css/styles.css | 6 | ||||
| -rw-r--r-- | qc_app/templates/rqtl2/rqtl2-qc-job-error.html | 49 | ||||
| -rw-r--r-- | qc_app/upload/rqtl2.py | 6 | ||||
| -rw-r--r-- | scripts/qc_on_rqtl2_bundle.py | 88 |
4 files changed, 110 insertions, 39 deletions
diff --git a/qc_app/static/css/styles.css b/qc_app/static/css/styles.css index 474c7f7..79dab0e 100644 --- a/qc_app/static/css/styles.css +++ b/qc_app/static/css/styles.css @@ -134,11 +134,7 @@ form fieldset:nth-child(odd) { } @media(min-width: 1250px) { - form { - width: 65ch; - } - - .explainer { + form, .explainer, .error-table { width: 65ch; } } diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html index f9a912c..49df061 100644 --- a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html +++ b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html @@ -3,6 +3,31 @@ {%block title%}R/qtl2 bundle: QC Job Error{%endblock%} +{%macro errors_table(tableid, errors)%} +<table id="{{tableid}}" + class="error-table" + style="display: block; overflow: scroll; max-height: 250px;"> + <thead style="position: sticky; top: 0;"> + <tr> + <th>Line</th> + <th>Column</th> + <th>Value</th> + <th>Message</th> + </tr> + </thead> + <tbody> + {%for error in errors%} + <tr> + <td>{{error.line}}</td> + <td>{{error.column}}</td> + <td>{{error.value}}</td> + <td>{{error.message}}</td> + </tr> + {%endfor%} + </tbody> +</table> +{%endmacro%} + {%block contents%} <h1 class="heading">R/qtl2 bundle: QC job Error</h1> @@ -24,7 +49,29 @@ </ul> {%endif%} -<p><emph>list other errors here by file type, I think …</emph></p> +{%if errorsgeno | length > 0%} +<h2 class="heading">Geno Errors ({{errorsgeno | length}})</h3> +<div class="explainer"> + We found the following errors in the 'geno' file in your R/qtl2 bundle: +</div> +{{errors_table("tbl-errors-geno", errorsgeno[0:50])}} +{%endif%} + +{%if errorspheno | length > 0%} +<h2 class="heading">Pheno Errors ({{errorspheno | length}})</h3> +<div class="explainer"> + We found the following errors in the 'pheno' file in your R/qtl2 bundle: +</div> +{{errorspheno}} +{%endif%} + +{%if errorsphenocovar | length > 0%} +<h2 class="heading">Phenocovar Errors ({{errorsphenocovar | length}})</h3> +<div class="explainer"> + We found the following errors in the 'phenocovar' file in your R/qtl2 bundle: +</div> +{{errorsphenocovar}} +{%endif%} <h4>stdout</h4> {{cli_output(job, "stdout")}} diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py index 4b00891..c45952c 100644 --- a/qc_app/upload/rqtl2.py +++ b/qc_app/upload/rqtl2.py @@ -207,6 +207,12 @@ def rqtl2_bundle_qc_status(jobid: UUID): job=thejob, errorsgeneric=json.loads( thejob.get("errors-generic", "[]")), + errorsgeno=json.loads( + thejob.get("errors-geno", "[]")), + errorspheno=json.loads( + thejob.get("errors-pheno", "[]")), + errorsphenocovar=json.loads( + thejob.get("errors-phenocovar", "[]")), messages=logmessages) if jobstatus == "success": jobmeta = json.loads(thejob["job-metadata"]) diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py index 43f766a..02c8c3a 100644 --- a/scripts/qc_on_rqtl2_bundle.py +++ b/scripts/qc_on_rqtl2_bundle.py @@ -1,7 +1,6 @@ """Run Quality Control checks on R/qtl2 bundle.""" import sys import json -from pathlib import Path from zipfile import ZipFile from argparse import Namespace from typing import Union, Sequence @@ -9,6 +8,8 @@ from logging import Logger, getLogger, StreamHandler from redis import Redis +from quality_control.errors import InvalidValue + from qc_app import jobs from qc_app.db_utils import database_connection from qc_app.check_connections import check_db, check_redis @@ -20,38 +21,59 @@ from scripts.cli_parser import init_cli_parser from scripts.process_rqtl2_bundle import parse_job from scripts.redis_logger import setup_redis_logger -def add_to_errors(rconn: Redis, fqjobid: str, key: str, errors: Sequence[rqfe.MissingFile]): +def dict2tuple(dct: dict) -> tuple: + """Utility to convert items in dicts to pairs of tuples.""" + return tuple((key, val) for key,val in dct.items()) + +def add_to_errors(rconn: Redis, + fqjobid: str, + key: str, + errors: Sequence[Union[InvalidValue, rqfe.MissingFile]]): """Add `errors` to a given list of errors""" - errs = tuple(set( - json.loads(rconn.hget(fqjobid, key) or "[]") + - [error.message for error in errors])) + errs = tuple(dict(item) for item in set( + [dict2tuple(old) for old in + json.loads(rconn.hget(fqjobid, key) or "[]")] + + [dict2tuple({"type": type(error).__name__, **error._asdict()}) + for error in errors])) rconn.hset(fqjobid, key, json.dumps(errs)) -def qc_missing_files(rconn: Redis, fqjobid: str, - bundlefilepath: Union[str, Path]) -> tuple[ - tuple[str, str], ...]: +def qc_missing_files(rconn: Redis, + fqjobid: str, + zfile: ZipFile, + logger: Logger) -> bool: """Run QC for files listed in control file that don't exist in bundle.""" - with ZipFile(str(bundlefilepath), "r") as zfile: - missing = rqc.missing_files(zfile) - add_to_errors(rconn, fqjobid, "errors-generic", tuple( - rqfe.MissingFile( - mfile[0], mfile[1], ( - f"File '{mfile[1]}' is listed in the control file under " - f"the '{mfile[0]}' key, but it does not actually exist in " - "the bundle.")) - for mfile in missing)) - - return missing - -def qc_geno_errors(_rconn, _fqjobid, _job) -> bool: + logger.info("Checking for missing files…") + missing = rqc.missing_files(zfile) + add_to_errors(rconn, fqjobid, "errors-generic", tuple( + rqfe.MissingFile( + mfile[0], mfile[1], ( + f"File '{mfile[1]}' is listed in the control file under " + f"the '{mfile[0]}' key, but it does not actually exist in " + "the bundle.")) + for mfile in missing)) + if len(missing) > 0: + logger.error("Missing files in the bundle!") + return True + return False + +def qc_geno_errors(rconn, fqjobid, zfile, logger) -> bool: """Check for errors in `geno` file(s).""" + logger.info("Checking for geno errors…") + gerrs = tuple(rqc.geno_errors(zfile)) + add_to_errors(rconn, fqjobid, "errors-generic", tuple( + err for err in gerrs if isinstance(err, rqfe.MissingFile))) + add_to_errors(rconn, fqjobid, "errors-geno", tuple( + err for err in gerrs if not isinstance(err, rqfe.MissingFile))) + if len(gerrs) > 0: + logger.error("The 'geno' file has errors.") + return True return False -def qc_pheno_errors(_rconn, _fqjobid, _job) -> bool: +def qc_pheno_errors(_rconn, _fqjobid, _zfile, _logger) -> bool: """Check for errors in `pheno` file(s).""" return False -def qc_phenocovar_errors(_rconn, _fqjobid, _job) -> bool: +def qc_phenocovar_errors(_rconn, _fqjobid, _zfile, _logger) -> bool: """Check for errors in `phenocovar` file(s).""" return False @@ -63,16 +85,16 @@ def run_qc(rconn: Redis, thejob = parse_job(rconn, args.redisprefix, args.jobid) jobmeta = thejob["job-metadata"] - if len(qc_missing_files(rconn, fqjobid, jobmeta["rqtl2-bundle-file"])) > 0: - logger.error("Missing files in the bundle!") - return 1 - - return ( - 1 if any(( - qc_geno_errors(rconn, fqjobid, thejob), - qc_pheno_errors(rconn, fqjobid, thejob), - qc_phenocovar_errors(rconn, fqjobid, thejob))) - else 0) + with ZipFile(jobmeta["rqtl2-bundle-file"], "r") as zfile: + if qc_missing_files(rconn, fqjobid, zfile, logger): + return 1 + + return ( + 1 if any(( + qc_geno_errors(rconn, fqjobid, zfile, logger), + qc_pheno_errors(rconn, fqjobid, zfile, logger), + qc_phenocovar_errors(rconn, fqjobid, zfile, logger))) + else 0) if __name__ == "__main__": def main(): |