diff options
| -rw-r--r-- | qc_app/parse.py | 11 | ||||
| -rw-r--r-- | qc_app/templates/errors_display.html | 43 | ||||
| -rw-r--r-- | qc_app/templates/job_progress.html | 5 | ||||
| -rw-r--r-- | qc_app/templates/parse_results.html | 41 | ||||
| -rw-r--r-- | scripts/worker.py | 23 |
5 files changed, 73 insertions, 50 deletions
diff --git a/qc_app/parse.py b/qc_app/parse.py index 578ab70..ec9962b 100644 --- a/qc_app/parse.py +++ b/qc_app/parse.py @@ -20,6 +20,7 @@ from quality_control.parsing import ( strain_names) parsebp = Blueprint("parse", __name__) +isinvalidvalue = lambda item: isinstance(item, InvalidValue) @parsebp.route("/parse", methods=["GET"]) def parse(): @@ -72,19 +73,23 @@ def parse_status(job_id: str): progress = float(job["percent"]) status = job["status"] filename = job.get("filename", "uploaded file") + errors = jsonpickle.decode( + job.get("errors", jsonpickle.encode(tuple()))) if status == "success": return redirect(url_for("parse.results", job_id=job_id)) if status == "parse-error": return redirect(url_for("parse.fail", job_id=job_id)) + app.jinja_env.globals.update(isinvalidvalue=isinvalidvalue) return render_template( "job_progress.html", job_id = job_id, job_status = status, progress = progress, message = job.get("message", ""), - job_name = f"Parsing '{filename}'") + job_name = f"Parsing '{filename}'", + errors=errors) return render_template("no_such_job.html", job_id=job_id) @@ -97,11 +102,11 @@ def results(job_id: str): if job: filename = job["filename"] errors = jsonpickle.decode(job["errors"]) + app.jinja_env.globals.update(isinvalidvalue=isinvalidvalue) return render_template( "parse_results.html", errors=errors, - job_name = f"Parsing '{filename}'", - isinvalidvalue=lambda item: isinstance(item, InvalidValue)) + job_name = f"Parsing '{filename}'") return render_template("no_such_job.html", job_id=job_id) diff --git a/qc_app/templates/errors_display.html b/qc_app/templates/errors_display.html new file mode 100644 index 0000000..0c9a212 --- /dev/null +++ b/qc_app/templates/errors_display.html @@ -0,0 +1,43 @@ +{%macro errors_display(errors, no_error_msg, error_message)%} + +{%if errors | length == 0 %} +<span class="alert-success">{{no_error_msg}}</span> +{%else %} +<p class="alert-error">{{error_message}}</p> + +<table class="reports-table"> + <thead> + <tr> + <th>line number</th> + <th>column(s)</th> + <th>error</th> + <th>error message</th> + </tr> + </thead> + + <tbody> + {%for error in errors%} + <tr> + <td>{{error["line"]}}</td> + <td> + {%if isinvalidvalue(error):%} + {{error.column}} + {%else: %} + {{error.columns}} + {%endif %} + </td> + <td> + {%if isinvalidvalue(error):%} + Invalid Value + {%else: %} + Duplicate Header + {%endif %} + </td> + <td>{{error["message"]}}</td> + </tr> + {%endfor%} + </tbody> +</table> +{%endif%} + +{%endmacro%} diff --git a/qc_app/templates/job_progress.html b/qc_app/templates/job_progress.html index 8a9256f..accdc1c 100644 --- a/qc_app/templates/job_progress.html +++ b/qc_app/templates/job_progress.html @@ -1,4 +1,5 @@ {%extends "base.html"%} +{%from "errors_display.html" import errors_display%} {%block extrameta%} <meta http-equiv="refresh" content="5"> @@ -16,4 +17,8 @@ <progress id="job_{{job_id}}" value="{{progress/100}}">{{progress}}</progress> <span>{{"%.2f" | format(progress)}}%</span> +<div> + {{errors_display(errors, "No errors found so far", "We have found the following errors so far")}} +</div> + {%endblock%} diff --git a/qc_app/templates/parse_results.html b/qc_app/templates/parse_results.html index a750bb5..8d39359 100644 --- a/qc_app/templates/parse_results.html +++ b/qc_app/templates/parse_results.html @@ -1,48 +1,11 @@ {%extends "base.html"%} +{%from "errors_display.html" import errors_display%} {%block title%}Parse Results{%endblock%} {%block contents%} <h1 class="heading">{{job_name}}: parse results</h2> -{%if errors | length == 0 %} -<span class="alert-success">No errors found in the file</span> -{%else %} -<p class="alert-error">We found the following errors</p> - -<table class="reports-table"> - <thead> - <tr> - <th>line number</th> - <th>column(s)</th> - <th>error</th> - <th>error message</th> - </tr> - </thead> - - <tbody> - {%for error in errors%} - <tr> - <td>{{error["line"]}}</td> - <td> - {%if isinvalidvalue(error):%} - {{error.column}} - {%else: %} - {{error.columns}} - {%endif %} - </td> - <td> - {%if isinvalidvalue(error):%} - Invalid Value - {%else: %} - Duplicate Header - {%endif %} - </td> - <td>{{error["message"]}}</td> - </tr> - {%endfor%} - </tbody> -</table> -{%endif%} +{{errors_display(errors, "No errors found in the file", "We found the following errors")}} {%endblock%} diff --git a/scripts/worker.py b/scripts/worker.py index f4d5b6b..6fab9f9 100644 --- a/scripts/worker.py +++ b/scripts/worker.py @@ -48,6 +48,12 @@ def process_cli_arguments(): return cli_args_valid(parser.parse_args()) +def stream_error(redis_conn, job_id, error): + errors = jsonpickle.decode( + redis_conn.hget(job_id, key="errors") or jsonpickle.encode(tuple())) + redis_conn.hset( + job_id, key="errors", value=jsonpickle.encode(errors + (error,))) + def main(): args = process_cli_arguments() if args is None: @@ -71,15 +77,16 @@ def main(): redis_conn.hset( name=args.job_id, key="message", value="Collecting errors") - if count > 0: - errors = take( - collect_errors(filepath, filetype, strains, progress_indicator), - count) - else: - errors = collect_errors(filepath, filetype, strains, progress_indicator) + error_count = 0 + for error in collect_errors( + filepath, filetype, strains, progress_indicator): + stream_error(redis_conn, args.job_id, error) + + if count > 0: + error_count = error_count + 1 + if error_count >= count: + break - redis_conn.hset( - name=args.job_id, key="errors", value=jsonpickle.encode(errors)) redis_conn.hset(name=args.job_id, key="status", value="success") return 0 |