aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--qc_app/parse.py11
-rw-r--r--qc_app/templates/errors_display.html43
-rw-r--r--qc_app/templates/job_progress.html5
-rw-r--r--qc_app/templates/parse_results.html41
-rw-r--r--scripts/worker.py23
5 files changed, 73 insertions, 50 deletions
diff --git a/qc_app/parse.py b/qc_app/parse.py
index 578ab70..ec9962b 100644
--- a/qc_app/parse.py
+++ b/qc_app/parse.py
@@ -20,6 +20,7 @@ from quality_control.parsing import (
strain_names)
parsebp = Blueprint("parse", __name__)
+isinvalidvalue = lambda item: isinstance(item, InvalidValue)
@parsebp.route("/parse", methods=["GET"])
def parse():
@@ -72,19 +73,23 @@ def parse_status(job_id: str):
progress = float(job["percent"])
status = job["status"]
filename = job.get("filename", "uploaded file")
+ errors = jsonpickle.decode(
+ job.get("errors", jsonpickle.encode(tuple())))
if status == "success":
return redirect(url_for("parse.results", job_id=job_id))
if status == "parse-error":
return redirect(url_for("parse.fail", job_id=job_id))
+ app.jinja_env.globals.update(isinvalidvalue=isinvalidvalue)
return render_template(
"job_progress.html",
job_id = job_id,
job_status = status,
progress = progress,
message = job.get("message", ""),
- job_name = f"Parsing '{filename}'")
+ job_name = f"Parsing '{filename}'",
+ errors=errors)
return render_template("no_such_job.html", job_id=job_id)
@@ -97,11 +102,11 @@ def results(job_id: str):
if job:
filename = job["filename"]
errors = jsonpickle.decode(job["errors"])
+ app.jinja_env.globals.update(isinvalidvalue=isinvalidvalue)
return render_template(
"parse_results.html",
errors=errors,
- job_name = f"Parsing '{filename}'",
- isinvalidvalue=lambda item: isinstance(item, InvalidValue))
+ job_name = f"Parsing '{filename}'")
return render_template("no_such_job.html", job_id=job_id)
diff --git a/qc_app/templates/errors_display.html b/qc_app/templates/errors_display.html
new file mode 100644
index 0000000..0c9a212
--- /dev/null
+++ b/qc_app/templates/errors_display.html
@@ -0,0 +1,43 @@
+{%macro errors_display(errors, no_error_msg, error_message)%}
+
+{%if errors | length == 0 %}
+<span class="alert-success">{{no_error_msg}}</span>
+{%else %}
+<p class="alert-error">{{error_message}}</p>
+
+<table class="reports-table">
+ <thead>
+ <tr>
+ <th>line number</th>
+ <th>column(s)</th>
+ <th>error</th>
+ <th>error message</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for error in errors%}
+ <tr>
+ <td>{{error["line"]}}</td>
+ <td>
+ {%if isinvalidvalue(error):%}
+ {{error.column}}
+ {%else: %}
+ {{error.columns}}
+ {%endif %}
+ </td>
+ <td>
+ {%if isinvalidvalue(error):%}
+ Invalid Value
+ {%else: %}
+ Duplicate Header
+ {%endif %}
+ </td>
+ <td>{{error["message"]}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+</table>
+{%endif%}
+
+{%endmacro%}
diff --git a/qc_app/templates/job_progress.html b/qc_app/templates/job_progress.html
index 8a9256f..accdc1c 100644
--- a/qc_app/templates/job_progress.html
+++ b/qc_app/templates/job_progress.html
@@ -1,4 +1,5 @@
{%extends "base.html"%}
+{%from "errors_display.html" import errors_display%}
{%block extrameta%}
<meta http-equiv="refresh" content="5">
@@ -16,4 +17,8 @@
<progress id="job_{{job_id}}" value="{{progress/100}}">{{progress}}</progress>
<span>{{"%.2f" | format(progress)}}%</span>
+<div>
+ {{errors_display(errors, "No errors found so far", "We have found the following errors so far")}}
+</div>
+
{%endblock%}
diff --git a/qc_app/templates/parse_results.html b/qc_app/templates/parse_results.html
index a750bb5..8d39359 100644
--- a/qc_app/templates/parse_results.html
+++ b/qc_app/templates/parse_results.html
@@ -1,48 +1,11 @@
{%extends "base.html"%}
+{%from "errors_display.html" import errors_display%}
{%block title%}Parse Results{%endblock%}
{%block contents%}
<h1 class="heading">{{job_name}}: parse results</h2>
-{%if errors | length == 0 %}
-<span class="alert-success">No errors found in the file</span>
-{%else %}
-<p class="alert-error">We found the following errors</p>
-
-<table class="reports-table">
- <thead>
- <tr>
- <th>line number</th>
- <th>column(s)</th>
- <th>error</th>
- <th>error message</th>
- </tr>
- </thead>
-
- <tbody>
- {%for error in errors%}
- <tr>
- <td>{{error["line"]}}</td>
- <td>
- {%if isinvalidvalue(error):%}
- {{error.column}}
- {%else: %}
- {{error.columns}}
- {%endif %}
- </td>
- <td>
- {%if isinvalidvalue(error):%}
- Invalid Value
- {%else: %}
- Duplicate Header
- {%endif %}
- </td>
- <td>{{error["message"]}}</td>
- </tr>
- {%endfor%}
- </tbody>
-</table>
-{%endif%}
+{{errors_display(errors, "No errors found in the file", "We found the following errors")}}
{%endblock%}
diff --git a/scripts/worker.py b/scripts/worker.py
index f4d5b6b..6fab9f9 100644
--- a/scripts/worker.py
+++ b/scripts/worker.py
@@ -48,6 +48,12 @@ def process_cli_arguments():
return cli_args_valid(parser.parse_args())
+def stream_error(redis_conn, job_id, error):
+ errors = jsonpickle.decode(
+ redis_conn.hget(job_id, key="errors") or jsonpickle.encode(tuple()))
+ redis_conn.hset(
+ job_id, key="errors", value=jsonpickle.encode(errors + (error,)))
+
def main():
args = process_cli_arguments()
if args is None:
@@ -71,15 +77,16 @@ def main():
redis_conn.hset(
name=args.job_id, key="message", value="Collecting errors")
- if count > 0:
- errors = take(
- collect_errors(filepath, filetype, strains, progress_indicator),
- count)
- else:
- errors = collect_errors(filepath, filetype, strains, progress_indicator)
+ error_count = 0
+ for error in collect_errors(
+ filepath, filetype, strains, progress_indicator):
+ stream_error(redis_conn, args.job_id, error)
+
+ if count > 0:
+ error_count = error_count + 1
+ if error_count >= count:
+ break
- redis_conn.hset(
- name=args.job_id, key="errors", value=jsonpickle.encode(errors))
redis_conn.hset(name=args.job_id, key="status", value="success")
return 0