aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--qc_app/static/css/styles.css8
-rw-r--r--qc_app/templates/rqtl2/rqtl2-qc-job-error.html14
-rw-r--r--qc_app/upload/rqtl2.py13
-rw-r--r--scripts/qc_on_rqtl2_bundle.py40
4 files changed, 65 insertions, 10 deletions
diff --git a/qc_app/static/css/styles.css b/qc_app/static/css/styles.css
index f6e829f..474c7f7 100644
--- a/qc_app/static/css/styles.css
+++ b/qc_app/static/css/styles.css
@@ -198,3 +198,11 @@ form fieldset legend {
width: 90%;
overflow: scroll;
}
+
+.qc-error-display {
+ border-radius: 0.8em;
+ padding-top: 0.5em;
+ padding-bottom: 0.5em;
+ max-height: 250px;
+ overflow: scroll;
+}
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
index 5d2ebee..f9a912c 100644
--- a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
+++ b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
@@ -12,9 +12,19 @@
be uploaded onto GeneNetwork.</p>
</div>
-<h2 class="heading">Errors</h2>
+{%if errorsgeneric | length > 0%}
+<h2 class="heading">Generic Errors ({{errorsgeneric | length}})</h3>
+<div class="explainer">
+ We found the following generic errors in your R/qtl2 bundle:
+</div>
+<ul class="alert-error qc-error-display">
+ {%for msg in errorsgeneric%}
+ <li>{{msg}}</li>
+ {%endfor%}
+</ul>
+{%endif%}
-<p><emph>list errors here by file type, I think &hellip;</emph></p>
+<p><emph>list other errors here by file type, I think &hellip;</emph></p>
<h4>stdout</h4>
{{cli_output(job, "stdout")}}
diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py
index b96d9f0..c20bd8f 100644
--- a/qc_app/upload/rqtl2.py
+++ b/qc_app/upload/rqtl2.py
@@ -175,12 +175,15 @@ def upload_rqtl2_bundle(species_id: int, population_id: int):
jobs.jobsnamespace(),
jobid,
[sys.executable, "-m", "scripts.qc_on_rqtl2_bundle",
- app.config["SQL_URI"], app.config["REDIS_URL"],
- jobs.jobsnamespace(), jobid, "--redisexpiry",
- str(redis_ttl_seconds)],
+ app.config["SQL_URI"], app.config["REDIS_URL"],
+ jobs.jobsnamespace(), jobid, "--redisexpiry",
+ str(redis_ttl_seconds)],
"rqtl2-bundle-qc-job",
redis_ttl_seconds,
- {}),
+ {"job-metadata": json.dumps({
+ "speciesid": species_id,
+ "populationid": population_id,
+ "rqtl2-bundle-file": str(the_file.absolute())})}),
redisuri,
f"{app.config['UPLOAD_FOLDER']}/job_errors")
return redirect(url_for(
@@ -200,6 +203,8 @@ def rqtl2_bundle_qc_status(jobid: UUID):
if jobstatus == "error":
return render_template("rqtl2/rqtl2-qc-job-error.html",
job=thejob,
+ errorsgeneric=json.loads(
+ thejob.get("errors-generic", "[]")),
messages=logmessages)
if jobstatus == "success":
return render_template("rqtl2/rqtl2-qc-job-results.html",
diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py
index 5802f3a..8dd6b7e 100644
--- a/scripts/qc_on_rqtl2_bundle.py
+++ b/scripts/qc_on_rqtl2_bundle.py
@@ -1,6 +1,10 @@
"""Run Quality Control checks on R/qtl2 bundle."""
import sys
-from logging import getLogger, StreamHandler
+import json
+from typing import Sequence
+from zipfile import ZipFile
+from argparse import Namespace
+from logging import Logger, getLogger, StreamHandler
from redis import Redis
@@ -8,12 +12,40 @@ from qc_app import jobs
from qc_app.db_utils import database_connection
from qc_app.check_connections import check_db, check_redis
+from r_qtl import r_qtl2_qc as rqc
+from r_qtl import fileerrors as rqfe
+
from scripts.cli_parser import init_cli_parser
+from scripts.process_rqtl2_bundle import parse_job
from scripts.redis_logger import setup_redis_logger
-def run_qc(_args):
+def add_to_errors(rconn: Redis, fqjobid: str, key: str, errors: Sequence[rqfe.MissingFile]):
+ """Add `errors` to a given list of errors"""
+ errs = tuple(set(
+ json.loads(rconn.hget(fqjobid, key) or "[]") +
+ [error.message for error in errors]))
+ rconn.hset(fqjobid, key, json.dumps(errs))
+
+def run_qc(rconn: Redis, args: Namespace, logger: Logger) -> int:
"""Run the QC programs."""
- return 1
+ fqjobid = jobs.job_key(args.redisprefix, args.jobid)
+ thejob = parse_job(rconn, args.redisprefix, args.jobid)
+ meta = thejob["job-metadata"]
+ with ZipFile(meta["rqtl2-bundle-file"], "r") as zfile:
+ missing = rqc.missing_files(zfile)
+ add_to_errors(rconn, fqjobid, "errors-generic", tuple(
+ rqfe.MissingFile(
+ mfile[0], mfile[1], (
+ f"File '{mfile[1]}' is listed in the control file under "
+ f"the '{mfile[0]}' key, but it does not actually exist in "
+ "the bundle."))
+ for mfile in missing))
+
+ if len(missing) > 0:
+ logger.error("Missing files in the bundle!")
+ return 1
+
+ return 0
if __name__ == "__main__":
def main():
@@ -34,7 +66,7 @@ if __name__ == "__main__":
rconn, fqjobid, f"{fqjobid}:log-messages",
args.redisexpiry))
- exitcode = run_qc(args)
+ exitcode = run_qc(rconn, args, logger)
rconn.hset(
jobs.job_key(args.redisprefix, args.jobid), "exitcode", exitcode)
return exitcode