Add an error display to the progress status report

Enable the progress status page to show all the errors found at any
point during the processing of the file.

5 files changed, 73 insertions, 50 deletions

diff --git a/qc_app/parse.py b/qc_app/parse.py
index 578ab70..ec9962b 100644
--- a/qc_app/parse.py
+++ b/qc_app/parse.py
@@ -20,6 +20,7 @@ from quality_control.parsing import (
     strain_names)
 
 parsebp = Blueprint("parse", __name__)
+isinvalidvalue = lambda item: isinstance(item, InvalidValue)
 
 @parsebp.route("/parse", methods=["GET"])
 def parse():
@@ -72,19 +73,23 @@ def parse_status(job_id: str):
         progress = float(job["percent"])
         status = job["status"]
         filename = job.get("filename", "uploaded file")
+        errors = jsonpickle.decode(
+            job.get("errors", jsonpickle.encode(tuple())))
         if status == "success":
             return redirect(url_for("parse.results", job_id=job_id))
 
         if status == "parse-error":
             return redirect(url_for("parse.fail", job_id=job_id))
 
+        app.jinja_env.globals.update(isinvalidvalue=isinvalidvalue)
         return render_template(
             "job_progress.html",
             job_id = job_id,
             job_status = status,
             progress = progress,
             message = job.get("message", ""),
-            job_name = f"Parsing '{filename}'")
+            job_name = f"Parsing '{filename}'",
+            errors=errors)
 
     return render_template("no_such_job.html", job_id=job_id)
 
@@ -97,11 +102,11 @@ def results(job_id: str):
     if job:
         filename = job["filename"]
         errors = jsonpickle.decode(job["errors"])
+        app.jinja_env.globals.update(isinvalidvalue=isinvalidvalue)
         return render_template(
             "parse_results.html",
             errors=errors,
-            job_name = f"Parsing '{filename}'",
-            isinvalidvalue=lambda item: isinstance(item, InvalidValue))
+            job_name = f"Parsing '{filename}'")
 
     return render_template("no_such_job.html", job_id=job_id)
 
diff --git a/qc_app/templates/errors_display.html b/qc_app/templates/errors_display.html
new file mode 100644
index 0000000..0c9a212
--- /dev/null
+++ b/qc_app/templates/errors_display.html
@@ -0,0 +1,43 @@
+{%macro errors_display(errors, no_error_msg, error_message)%}
+
+{%if errors | length == 0 %}
+<span class="alert-success">{{no_error_msg}}</span>
+{%else %}
+<p class="alert-error">{{error_message}}</p>
+
+<table class="reports-table">
+  <thead>
+    <tr>
+      <th>line number</th>
+      <th>column(s)</th>
+      <th>error</th>
+      <th>error message</th>
+    </tr>
+  </thead>
+
+  <tbody>
+    {%for error in errors%}
+    <tr>
+      <td>{{error["line"]}}</td>
+      <td>
+	{%if isinvalidvalue(error):%}
+	{{error.column}}
+	{%else: %}
+	{{error.columns}}
+	{%endif %}
+      </td>
+      <td>
+	{%if isinvalidvalue(error):%}
+	Invalid Value
+	{%else: %}
+	Duplicate Header
+	{%endif %}
+      </td>
+      <td>{{error["message"]}}</td>
+    </tr>
+    {%endfor%}
+  </tbody>
+</table>
+{%endif%}
+
+{%endmacro%}
diff --git a/qc_app/templates/job_progress.html b/qc_app/templates/job_progress.html
index 8a9256f..accdc1c 100644
--- a/qc_app/templates/job_progress.html
+++ b/qc_app/templates/job_progress.html
@@ -1,4 +1,5 @@
 {%extends "base.html"%}
+{%from "errors_display.html" import errors_display%}
 
 {%block extrameta%}
 <meta http-equiv="refresh" content="5">
@@ -16,4 +17,8 @@
 <progress id="job_{{job_id}}" value="{{progress/100}}">{{progress}}</progress>
 <span>{{"%.2f" | format(progress)}}%</span>
 
+<div>
+  {{errors_display(errors, "No errors found so far", "We have found the following errors so far")}}
+</div>
+
 {%endblock%}
diff --git a/qc_app/templates/parse_results.html b/qc_app/templates/parse_results.html
index a750bb5..8d39359 100644
--- a/qc_app/templates/parse_results.html
+++ b/qc_app/templates/parse_results.html
@@ -1,48 +1,11 @@
 {%extends "base.html"%}
+{%from "errors_display.html" import errors_display%}
 
 {%block title%}Parse Results{%endblock%}
 
 {%block contents%}
 <h1 class="heading">{{job_name}}: parse results</h2>
 
-{%if errors | length == 0 %}
-<span class="alert-success">No errors found in the file</span>
-{%else %}
-<p class="alert-error">We found the following errors</p>
-
-<table class="reports-table">
-  <thead>
-    <tr>
-      <th>line number</th>
-      <th>column(s)</th>
-      <th>error</th>
-      <th>error message</th>
-    </tr>
-  </thead>
-
-  <tbody>
-    {%for error in errors%}
-    <tr>
-      <td>{{error["line"]}}</td>
-      <td>
-	{%if isinvalidvalue(error):%}
-	{{error.column}}
-	{%else: %}
-	{{error.columns}}
-	{%endif %}
-      </td>
-      <td>
-	{%if isinvalidvalue(error):%}
-	Invalid Value
-	{%else: %}
-	Duplicate Header
-	{%endif %}
-      </td>
-      <td>{{error["message"]}}</td>
-    </tr>
-    {%endfor%}
-  </tbody>
-</table>
-{%endif%}
+{{errors_display(errors, "No errors found in the file", "We found the following errors")}}
 
 {%endblock%}
diff --git a/scripts/worker.py b/scripts/worker.py
index f4d5b6b..6fab9f9 100644
--- a/scripts/worker.py
+++ b/scripts/worker.py
@@ -48,6 +48,12 @@ def process_cli_arguments():
 
     return cli_args_valid(parser.parse_args())
 
+def stream_error(redis_conn, job_id, error):
+    errors = jsonpickle.decode(
+        redis_conn.hget(job_id, key="errors") or jsonpickle.encode(tuple()))
+    redis_conn.hset(
+        job_id, key="errors", value=jsonpickle.encode(errors + (error,)))
+
 def main():
     args = process_cli_arguments()
     if args is None:
@@ -71,15 +77,16 @@ def main():
         redis_conn.hset(
             name=args.job_id, key="message", value="Collecting errors")
 
-        if count > 0:
-            errors = take(
-                collect_errors(filepath, filetype, strains, progress_indicator),
-                count)
-        else:
-            errors = collect_errors(filepath, filetype, strains, progress_indicator)
+        error_count = 0
+        for error in collect_errors(
+                filepath, filetype, strains, progress_indicator):
+            stream_error(redis_conn, args.job_id, error)
+
+            if count > 0:
+                error_count = error_count + 1
+                if error_count >= count:
+                    break
 
-        redis_conn.hset(
-            name=args.job_id, key="errors", value=jsonpickle.encode(errors))
         redis_conn.hset(name=args.job_id, key="status", value="success")
 
     return 0