aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--qc_app/templates/base.html2
-rw-r--r--qc_app/templates/rqtl2/rqtl2-qc-job-error.html32
-rw-r--r--qc_app/templates/rqtl2/rqtl2-qc-job-status.html29
-rw-r--r--qc_app/templates/rqtl2/rqtl2-qc-job-success.html37
-rw-r--r--qc_app/upload/rqtl2.py63
-rw-r--r--scripts/qc_on_rqtl2_bundle.py42
6 files changed, 186 insertions, 19 deletions
diff --git a/qc_app/templates/base.html b/qc_app/templates/base.html
index 3b701bc..2228356 100644
--- a/qc_app/templates/base.html
+++ b/qc_app/templates/base.html
@@ -6,7 +6,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
{%block extrameta%}{%endblock%}
- <title>QC: {%block title%}{%endblock%}</title>
+ <title>GN Uploader: {%block title%}{%endblock%}</title>
<link rel="stylesheet" type="text/css" href="/static/css/styles.css" />
{%block css%}{%endblock%}
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
new file mode 100644
index 0000000..5d2ebee
--- /dev/null
+++ b/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
@@ -0,0 +1,32 @@
+{%extends "base.html"%}
+{%from "cli-output.html" import cli_output%}
+
+{%block title%}R/qtl2 bundle: QC Job Error{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 bundle: QC job Error</h1>
+
+<div class="explainer">
+ <p>The R/qtl2 bundle has failed some <emph>Quality Control</emph> checks.</p>
+ <p>We list below some of the errors that need to be fixed before the data can
+ be uploaded onto GeneNetwork.</p>
+</div>
+
+<h2 class="heading">Errors</h2>
+
+<p><emph>list errors here by file type, I think &hellip;</emph></p>
+
+<h4>stdout</h4>
+{{cli_output(job, "stdout")}}
+
+<h4>stderr</h4>
+{{cli_output(job, "stderr")}}
+
+<h4>Log</h4>
+<div class="cli-output">
+ {%for msg in messages%}
+ {{msg}}<br />
+ {%endfor%}
+</div>
+
+{%endblock%}
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-status.html b/qc_app/templates/rqtl2/rqtl2-qc-job-status.html
new file mode 100644
index 0000000..85b8864
--- /dev/null
+++ b/qc_app/templates/rqtl2/rqtl2-qc-job-status.html
@@ -0,0 +1,29 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="3">
+{%endblock%}
+
+{%block contents%}
+<h1 class="heading">R/qtl2 bundle: QC job status</h1>
+
+<h2 class="heading">R/qtl2 bundle: QC Job Status</h2>
+
+<hr />
+<p>The job:</p>
+<hr />
+{{job}}
+<hr />
+<hr />
+
+<h4>Log</h4>
+<div class="cli-output">
+ {%for msg in messages%}
+ {{msg}}<br />
+ {%endfor%}
+</div>
+
+{%endblock%}
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-success.html b/qc_app/templates/rqtl2/rqtl2-qc-job-success.html
new file mode 100644
index 0000000..396f241
--- /dev/null
+++ b/qc_app/templates/rqtl2/rqtl2-qc-job-success.html
@@ -0,0 +1,37 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}R/qtl2 Bundle: Quality Control Successful{%endblock%}
+
+{%block contents%}
+<h2 class="heading">R/qtl2 Bundle: Quality Control Successful</h2>
+
+<div class="explainer">
+ <p>The R/qtl2 bundle you uploaded has passed <emph>all</emph> quality control
+ checks successfully, and is now ready for uploading into the database.</p>
+ <p>Click "Continue" below to proceed.</p>
+</div>
+
+<!--
+ The "action" on this form takes us to the next step, where we can
+ select all the other data necessary to enter the data into the database.
+ -->
+<form id="frm-upload-rqtl2-bundle"
+ action="{{url_for('upload.rqtl2.select_dataset_info',
+ species_id=species.SpeciesId,
+ population_id=population.InbredSetId)}}"
+ method="POST"
+ enctype="multipart/form-data">
+ {{flash_all_messages()}}
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id"
+ value="{{population.InbredSetId}}" />
+ <input type="hidden" name="rqtl2_bundle_file"
+ value="{{rqtl2_bundle_file}}" />
+
+ <fieldset>
+ <input type="submit" value="continue" class="btn btn-main form-col-2" />
+ </fieldset>
+</form>
+
+{%endblock%}
diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py
index 9d787e1..afe0c2e 100644
--- a/qc_app/upload/rqtl2.py
+++ b/qc_app/upload/rqtl2.py
@@ -18,8 +18,6 @@ from flask import (
current_app as app)
from r_qtl import r_qtl2
-from r_qtl import r_qtl2_qc as rqc
-from r_qtl.errors import InvalidFormat
from qc_app import jobs
from qc_app.files import save_file, fullpath
@@ -135,12 +133,6 @@ class __RequestError__(Exception): #pylint: disable=[invalid-name]
methods=["GET", "POST"])
def upload_rqtl2_bundle(species_id: int, population_id: int):
"""Allow upload of R/qtl2 bundle."""
- this_page_with_errors = redirect(url_for("upload.rqtl2.upload_rqtl2_bundle",
- species_id=species_id,
- population_id=population_id,
- pgsrc="error"),
- code=307)
-
with database_connection(app.config["SQL_URI"]) as conn:
species = species_by_id(conn, species_id)
population = population_by_species_and_id(
@@ -171,17 +163,52 @@ def upload_rqtl2_bundle(species_id: int, population_id: int):
if not is_zipfile(str(the_file)):
raise __RequestError__("Invalid file! Expected a zip file.")
+ redisuri = app.config["REDIS_URL"]
+ with Redis.from_url(redisuri, decode_responses=True) as rconn:
+ jobid = str(uuid4())
+ redis_ttl_seconds = app.config["JOBS_TTL_SECONDS"]
+ jobs.launch_job(
+ jobs.initialise_job(
+ rconn,
+ jobs.jobsnamespace(),
+ jobid,
+ [sys.executable, "-m", "scripts.qc_on_rqtl2_bundle",
+ app.config["SQL_URI"], app.config["REDIS_URL"],
+ jobs.jobsnamespace(), jobid, "--redisexpiry",
+ str(redis_ttl_seconds)],
+ "rqtl2-bundle-qc-job",
+ redis_ttl_seconds,
+ {}),
+ redisuri,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors")
+ return redirect(url_for(
+ "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid))
+
+@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>",
+ methods=["GET", "POST"])
+def rqtl2_bundle_qc_status(jobid: UUID):
+ """Check the status of the QC jobs."""
+ with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
try:
- with ZipFile(str(the_file), "r") as zfile:
- rqc.validate_bundle(zfile)
- return render_template(
- "rqtl2/upload-rqtl2-bundle-step-02.html",
- species=species,
- population=population,
- rqtl2_bundle_file=the_file.name)#type: ignore[union-attr]
- except (InvalidFormat, __RequestError__) as exc:
- flash("".join(exc.args), "alert-error error-rqtl2")
- return this_page_with_errors
+ thejob = jobs.job(rconn, jobs.jobsnamespace(), jobid)
+ messagelistname = thejob.get("log-messagelist")
+ logmessages = (rconn.lrange(messagelistname, 0, -1)
+ if bool(messagelistname) else [])
+ jobstatus = thejob["status"]
+ if jobstatus == "error":
+ return render_template("rqtl2/rqtl2-qc-job-error.html",
+ job=thejob,
+ messages=logmessages)
+ if jobstatus == "success":
+ return render_template("rqtl2/rqtl2-qc-job-results.html",
+ job=thejob,
+ messages=logmessages)
+
+ return render_template("rqtl2/rqtl2-qc-job-status.html",
+ job=thejob,
+ messages=tuple())
+ except jobs.JobNotFound:
+ return render_template("rqtl2/no-such-job.html", jobid=jobid)
def check_errors(conn, *args, **kwargs):#pylint: disable=[too-many-return-statements]
"""Check for select errors in the forms and return a page to redirect to."""
diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py
new file mode 100644
index 0000000..5802f3a
--- /dev/null
+++ b/scripts/qc_on_rqtl2_bundle.py
@@ -0,0 +1,42 @@
+"""Run Quality Control checks on R/qtl2 bundle."""
+import sys
+from logging import getLogger, StreamHandler
+
+from redis import Redis
+
+from qc_app import jobs
+from qc_app.db_utils import database_connection
+from qc_app.check_connections import check_db, check_redis
+
+from scripts.cli_parser import init_cli_parser
+from scripts.redis_logger import setup_redis_logger
+
+def run_qc(_args):
+ """Run the QC programs."""
+ return 1
+
+if __name__ == "__main__":
+ def main():
+ """Enter R/qtl2 bundle QC runner."""
+ args = init_cli_parser(
+ "qc-on-rqtl2-bundle", "Run QC on R/qtl2 bundle.").parse_args()
+ check_redis(args.redisuri)
+ check_db(args.databaseuri)
+
+ logger = getLogger("qc-on-rqtl2-bundle")
+ logger.addHandler(StreamHandler(stream=sys.stderr))
+ logger.setLevel("DEBUG")
+
+ fqjobid = jobs.job_key(args.redisprefix, args.jobid)
+ with (database_connection(args.databaseuri) as _dbconn,
+ Redis.from_url(args.redisuri, decode_responses=True) as rconn):
+ logger.addHandler(setup_redis_logger(
+ rconn, fqjobid, f"{fqjobid}:log-messages",
+ args.redisexpiry))
+
+ exitcode = run_qc(args)
+ rconn.hset(
+ jobs.job_key(args.redisprefix, args.jobid), "exitcode", exitcode)
+ return exitcode
+
+ sys.exit(main())