diff options
Diffstat (limited to 'scripts/worker.py')
| -rw-r--r-- | scripts/worker.py | 23 |
1 files changed, 15 insertions, 8 deletions
diff --git a/scripts/worker.py b/scripts/worker.py index f4d5b6b..6fab9f9 100644 --- a/scripts/worker.py +++ b/scripts/worker.py @@ -48,6 +48,12 @@ def process_cli_arguments(): return cli_args_valid(parser.parse_args()) +def stream_error(redis_conn, job_id, error): + errors = jsonpickle.decode( + redis_conn.hget(job_id, key="errors") or jsonpickle.encode(tuple())) + redis_conn.hset( + job_id, key="errors", value=jsonpickle.encode(errors + (error,))) + def main(): args = process_cli_arguments() if args is None: @@ -71,15 +77,16 @@ def main(): redis_conn.hset( name=args.job_id, key="message", value="Collecting errors") - if count > 0: - errors = take( - collect_errors(filepath, filetype, strains, progress_indicator), - count) - else: - errors = collect_errors(filepath, filetype, strains, progress_indicator) + error_count = 0 + for error in collect_errors( + filepath, filetype, strains, progress_indicator): + stream_error(redis_conn, args.job_id, error) + + if count > 0: + error_count = error_count + 1 + if error_count >= count: + break - redis_conn.hset( - name=args.job_id, key="errors", value=jsonpickle.encode(errors)) redis_conn.hset(name=args.job_id, key="status", value="success") return 0 |