aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--qc_app/parse.py9
-rw-r--r--quality_control/parsing.py37
-rw-r--r--scripts/qc.py5
-rw-r--r--tests/conftest.py4
4 files changed, 22 insertions, 33 deletions
diff --git a/qc_app/parse.py b/qc_app/parse.py
index 795cc01..baad9a6 100644
--- a/qc_app/parse.py
+++ b/qc_app/parse.py
@@ -19,8 +19,7 @@ from quality_control.parsing import (
FileType,
parse_file,
strain_names,
- parse_errors,
- parse_strains)
+ parse_errors)
parsebp = Blueprint("parse", __name__)
@@ -34,8 +33,7 @@ def queued_parse(
try:
job_meta = jobs.update_meta(
dbconn, job_id, status = "in-progress", progress = 0)
- parsed = parse_file(
- filepath, filetype, strain_names(parse_strains(strainsfile)))
+ parsed = parse_file(filepath, filetype, strain_names(strainsfile))
for line, curr_size in parsed:
job_meta = jobs.update_meta(
dbconn, job_id,
@@ -174,11 +172,10 @@ def queued_collect_errors(
dbconn = sqlite3.connect(dbpath)
job_meta = jobs.retrieve_meta(dbconn, job.get_id())
for error in parse_errors(
- filepath, filetype, strain_names(parse_strains(strainsfile)),
+ filepath, filetype, strain_names(strainsfile),
seek_pos):
count = count + 1
progress = ((error["position"] / job_meta["filesize"]) * 100)
- print(f"CURRENT PROGRESS: {progress}")
job_meta = jobs.update_meta(
dbconn, job_id, message = f"Collected {count} errors",
progress = progress)
diff --git a/quality_control/parsing.py b/quality_control/parsing.py
index 9fe88f1..436c90c 100644
--- a/quality_control/parsing.py
+++ b/quality_control/parsing.py
@@ -16,21 +16,6 @@ class FileType(Enum):
AVERAGE = 1
STANDARD_ERROR = 2
-def parse_strains(filepath):
- """Parse the strains file"""
- with open(filepath, encoding="utf8") as strains_file:
- reader = csv.DictReader(
- strains_file,
- fieldnames=[
- header.strip() for header
- in strains_file.readline().split("\t")],
- delimiter="\t")
- for row in reader:
- yield {
- key: (value if value != "\\N" else None)
- for key, value in row.items()
- }
-
def __parse_header(line, strains):
return valid_header(
set(strains),
@@ -47,13 +32,21 @@ LINE_PARSERS = {
FileType.STANDARD_ERROR: __parse_standard_error_line
}
-def strain_names(strains):
- """Retrieve a complete list of the names of the strains"""
- def __extract_strain_names(acc, strain):
- return acc + tuple(
- item for item in (strain["Name"], strain["Name2"])
- if (item is not None and item != ""))
- return reduce(__extract_strain_names, strains, tuple())
+def strain_names(filepath):
+ """Retrieve the strains names from given file"""
+ strains = set()
+ with open(filepath, encoding="utf8") as strains_file:
+ for idx, line in enumerate(strains_file.readlines()):
+ if idx > 0:
+ parts = line.split()
+ for name in (parts[1], parts[2]):
+ strains.add(name.strip())
+ if len(parts) >= 6:
+ alias = parts[5].strip()
+ if alias != "" and alias not in ("P", "\\N"):
+ strains.add(alias)
+
+ return strains
def parse_file(filepath: str, filetype: FileType, strains: list):
"""Parse the given file"""
diff --git a/scripts/qc.py b/scripts/qc.py
index 09758cb..9937e5b 100644
--- a/scripts/qc.py
+++ b/scripts/qc.py
@@ -10,8 +10,7 @@ from quality_control.parsing import (
FileType,
parse_file,
strain_names,
- parse_errors,
- parse_strains)
+ parse_errors)
def is_file_mime(filepath, mimetype):
@@ -93,7 +92,7 @@ def main():
if args.verbose:
print(f"Parsing the strain names from '{args.strainsfile}'")
- strains = strain_names(parse_strains(os.path.realpath(args.strainsfile)))
+ strains = strain_names(os.path.realpath(args.strainsfile))
filepath = os.path.realpath(args.filepath)
if args.verbose:
diff --git a/tests/conftest.py b/tests/conftest.py
index f79166d..6ef5374 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2,9 +2,9 @@
import pytest
-from quality_control.parsing import strain_names, parse_strains
+from quality_control.parsing import strain_names
@pytest.fixture(scope="session")
def strains():
"""Parse the strains once every test session"""
- return strain_names(parse_strains("strains.csv"))
+ return strain_names("etc/strains.csv")