From e6895f5bac672d2e1d2a04fe8118fa55c3a40b91 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Tue, 26 Apr 2022 09:43:18 +0300
Subject: Queue file parsing jobs

Enable the queuing of file parsing jobs, since the files could be
really large and take a long time to parse and present results.

* etc/default_config.py: Add default config for redis server
* manifest.scm: Add redis, and rq as dependencies
* qc_app/__init__.py
* qc_app/jobs.py: module to hold utilities for management of the jobs
* qc_app/parse.py: Enqueue the job - extract file-parsing code to
  callable function
* qc_app/templates/base.html: Enable addition of extra meta tags
* qc_app/templates/job_progress.html: template to display job progress
* qc_app/templates/no_such_job.html: template to indicate when a job
  id is invalid
* quality_control/parsing.py: Add the total size parsed so far
---
 quality_control/parsing.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'quality_control')

diff --git a/quality_control/parsing.py b/quality_control/parsing.py
index eb4c989..a4edb0f 100644
--- a/quality_control/parsing.py
+++ b/quality_control/parsing.py
@@ -66,8 +66,10 @@ def parse_file(filepath: str, filetype: FileType, strains: list):
                     seek_pos = seek_pos + len(line)
                     continue
 
-                yield LINE_PARSERS[filetype](
-                    tuple(field.strip() for field in line.split("\t")))
+                yield (
+                    LINE_PARSERS[filetype](
+                        tuple(field.strip() for field in line.split("\t"))),
+                    seek_pos + len(line))
                 seek_pos = seek_pos + len(line)
     except (DuplicateHeader, InvalidCellValue, InvalidHeaderValue) as err:
         raise ParseError({
-- 
cgit v1.2.3