aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--qc_app/static/css/styles.css6
-rw-r--r--qc_app/templates/rqtl2/rqtl2-qc-job-error.html49
-rw-r--r--qc_app/upload/rqtl2.py6
-rw-r--r--scripts/qc_on_rqtl2_bundle.py88
4 files changed, 110 insertions, 39 deletions
diff --git a/qc_app/static/css/styles.css b/qc_app/static/css/styles.css
index 474c7f7..79dab0e 100644
--- a/qc_app/static/css/styles.css
+++ b/qc_app/static/css/styles.css
@@ -134,11 +134,7 @@ form fieldset:nth-child(odd) {
}
@media(min-width: 1250px) {
- form {
- width: 65ch;
- }
-
- .explainer {
+ form, .explainer, .error-table {
width: 65ch;
}
}
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
index f9a912c..49df061 100644
--- a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
+++ b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
@@ -3,6 +3,31 @@
{%block title%}R/qtl2 bundle: QC Job Error{%endblock%}
+{%macro errors_table(tableid, errors)%}
+<table id="{{tableid}}"
+ class="error-table"
+ style="display: block; overflow: scroll; max-height: 250px;">
+ <thead style="position: sticky; top: 0;">
+ <tr>
+ <th>Line</th>
+ <th>Column</th>
+ <th>Value</th>
+ <th>Message</th>
+ </tr>
+ </thead>
+ <tbody>
+ {%for error in errors%}
+ <tr>
+ <td>{{error.line}}</td>
+ <td>{{error.column}}</td>
+ <td>{{error.value}}</td>
+ <td>{{error.message}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+</table>
+{%endmacro%}
+
{%block contents%}
<h1 class="heading">R/qtl2 bundle: QC job Error</h1>
@@ -24,7 +49,29 @@
</ul>
{%endif%}
-<p><emph>list other errors here by file type, I think &hellip;</emph></p>
+{%if errorsgeno | length > 0%}
+<h2 class="heading">Geno Errors ({{errorsgeno | length}})</h3>
+<div class="explainer">
+ We found the following errors in the 'geno' file in your R/qtl2 bundle:
+</div>
+{{errors_table("tbl-errors-geno", errorsgeno[0:50])}}
+{%endif%}
+
+{%if errorspheno | length > 0%}
+<h2 class="heading">Pheno Errors ({{errorspheno | length}})</h3>
+<div class="explainer">
+ We found the following errors in the 'pheno' file in your R/qtl2 bundle:
+</div>
+{{errorspheno}}
+{%endif%}
+
+{%if errorsphenocovar | length > 0%}
+<h2 class="heading">Phenocovar Errors ({{errorsphenocovar | length}})</h3>
+<div class="explainer">
+ We found the following errors in the 'phenocovar' file in your R/qtl2 bundle:
+</div>
+{{errorsphenocovar}}
+{%endif%}
<h4>stdout</h4>
{{cli_output(job, "stdout")}}
diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py
index 4b00891..c45952c 100644
--- a/qc_app/upload/rqtl2.py
+++ b/qc_app/upload/rqtl2.py
@@ -207,6 +207,12 @@ def rqtl2_bundle_qc_status(jobid: UUID):
job=thejob,
errorsgeneric=json.loads(
thejob.get("errors-generic", "[]")),
+ errorsgeno=json.loads(
+ thejob.get("errors-geno", "[]")),
+ errorspheno=json.loads(
+ thejob.get("errors-pheno", "[]")),
+ errorsphenocovar=json.loads(
+ thejob.get("errors-phenocovar", "[]")),
messages=logmessages)
if jobstatus == "success":
jobmeta = json.loads(thejob["job-metadata"])
diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py
index 43f766a..02c8c3a 100644
--- a/scripts/qc_on_rqtl2_bundle.py
+++ b/scripts/qc_on_rqtl2_bundle.py
@@ -1,7 +1,6 @@
"""Run Quality Control checks on R/qtl2 bundle."""
import sys
import json
-from pathlib import Path
from zipfile import ZipFile
from argparse import Namespace
from typing import Union, Sequence
@@ -9,6 +8,8 @@ from logging import Logger, getLogger, StreamHandler
from redis import Redis
+from quality_control.errors import InvalidValue
+
from qc_app import jobs
from qc_app.db_utils import database_connection
from qc_app.check_connections import check_db, check_redis
@@ -20,38 +21,59 @@ from scripts.cli_parser import init_cli_parser
from scripts.process_rqtl2_bundle import parse_job
from scripts.redis_logger import setup_redis_logger
-def add_to_errors(rconn: Redis, fqjobid: str, key: str, errors: Sequence[rqfe.MissingFile]):
+def dict2tuple(dct: dict) -> tuple:
+ """Utility to convert items in dicts to pairs of tuples."""
+ return tuple((key, val) for key,val in dct.items())
+
+def add_to_errors(rconn: Redis,
+ fqjobid: str,
+ key: str,
+ errors: Sequence[Union[InvalidValue, rqfe.MissingFile]]):
"""Add `errors` to a given list of errors"""
- errs = tuple(set(
- json.loads(rconn.hget(fqjobid, key) or "[]") +
- [error.message for error in errors]))
+ errs = tuple(dict(item) for item in set(
+ [dict2tuple(old) for old in
+ json.loads(rconn.hget(fqjobid, key) or "[]")] +
+ [dict2tuple({"type": type(error).__name__, **error._asdict()})
+ for error in errors]))
rconn.hset(fqjobid, key, json.dumps(errs))
-def qc_missing_files(rconn: Redis, fqjobid: str,
- bundlefilepath: Union[str, Path]) -> tuple[
- tuple[str, str], ...]:
+def qc_missing_files(rconn: Redis,
+ fqjobid: str,
+ zfile: ZipFile,
+ logger: Logger) -> bool:
"""Run QC for files listed in control file that don't exist in bundle."""
- with ZipFile(str(bundlefilepath), "r") as zfile:
- missing = rqc.missing_files(zfile)
- add_to_errors(rconn, fqjobid, "errors-generic", tuple(
- rqfe.MissingFile(
- mfile[0], mfile[1], (
- f"File '{mfile[1]}' is listed in the control file under "
- f"the '{mfile[0]}' key, but it does not actually exist in "
- "the bundle."))
- for mfile in missing))
-
- return missing
-
-def qc_geno_errors(_rconn, _fqjobid, _job) -> bool:
+ logger.info("Checking for missing files…")
+ missing = rqc.missing_files(zfile)
+ add_to_errors(rconn, fqjobid, "errors-generic", tuple(
+ rqfe.MissingFile(
+ mfile[0], mfile[1], (
+ f"File '{mfile[1]}' is listed in the control file under "
+ f"the '{mfile[0]}' key, but it does not actually exist in "
+ "the bundle."))
+ for mfile in missing))
+ if len(missing) > 0:
+ logger.error("Missing files in the bundle!")
+ return True
+ return False
+
+def qc_geno_errors(rconn, fqjobid, zfile, logger) -> bool:
"""Check for errors in `geno` file(s)."""
+ logger.info("Checking for geno errors…")
+ gerrs = tuple(rqc.geno_errors(zfile))
+ add_to_errors(rconn, fqjobid, "errors-generic", tuple(
+ err for err in gerrs if isinstance(err, rqfe.MissingFile)))
+ add_to_errors(rconn, fqjobid, "errors-geno", tuple(
+ err for err in gerrs if not isinstance(err, rqfe.MissingFile)))
+ if len(gerrs) > 0:
+ logger.error("The 'geno' file has errors.")
+ return True
return False
-def qc_pheno_errors(_rconn, _fqjobid, _job) -> bool:
+def qc_pheno_errors(_rconn, _fqjobid, _zfile, _logger) -> bool:
"""Check for errors in `pheno` file(s)."""
return False
-def qc_phenocovar_errors(_rconn, _fqjobid, _job) -> bool:
+def qc_phenocovar_errors(_rconn, _fqjobid, _zfile, _logger) -> bool:
"""Check for errors in `phenocovar` file(s)."""
return False
@@ -63,16 +85,16 @@ def run_qc(rconn: Redis,
thejob = parse_job(rconn, args.redisprefix, args.jobid)
jobmeta = thejob["job-metadata"]
- if len(qc_missing_files(rconn, fqjobid, jobmeta["rqtl2-bundle-file"])) > 0:
- logger.error("Missing files in the bundle!")
- return 1
-
- return (
- 1 if any((
- qc_geno_errors(rconn, fqjobid, thejob),
- qc_pheno_errors(rconn, fqjobid, thejob),
- qc_phenocovar_errors(rconn, fqjobid, thejob)))
- else 0)
+ with ZipFile(jobmeta["rqtl2-bundle-file"], "r") as zfile:
+ if qc_missing_files(rconn, fqjobid, zfile, logger):
+ return 1
+
+ return (
+ 1 if any((
+ qc_geno_errors(rconn, fqjobid, zfile, logger),
+ qc_pheno_errors(rconn, fqjobid, zfile, logger),
+ qc_phenocovar_errors(rconn, fqjobid, zfile, logger)))
+ else 0)
if __name__ == "__main__":
def main():