From b1483d974d30d162e12557f55e856ec7d79bad2e Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Tue, 20 Feb 2024 05:16:45 +0300 Subject: Read each file separately Provide the function 'read_file_data' in the 'r_qtl.r_qtl2' module to read each file in the bundle separately. The function 'file_data' in the 'r_qtl.r_qtl2' module reads *ALL* the files of a particular type (e.g. geno files) and returns a single generator object with the data from *ALL* the files. This does not render itself very useful for error checking. We needed a way to check for errors, and report them for each and every file in the bundle, for easier tracking and fixing. --- r_qtl/r_qtl2.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'r_qtl') diff --git a/r_qtl/r_qtl2.py b/r_qtl/r_qtl2.py index 93b8c8e..8c17362 100644 --- a/r_qtl/r_qtl2.py +++ b/r_qtl/r_qtl2.py @@ -5,7 +5,7 @@ import json from pathlib import Path from zipfile import ZipFile from functools import reduce, partial -from typing import Iterator, Iterable, Callable, Optional +from typing import Union, Iterator, Iterable, Callable, Optional import yaml @@ -302,7 +302,7 @@ def raw_file_data(zipfilepath: Union[str, Path], zfile.open(memberfilename) as innerfile): wrappedfile = io.TextIOWrapper(innerfile) for line in wrappedfile: - yield line.strip() + yield line def strip_comments(rawdata: Iterator[str], commentchar) -> Iterator[str]: """Remove comments from raw text.""" @@ -334,3 +334,23 @@ def read_control_file(zipfilepath: Union[str, Path]) -> dict: if bool(cdata.get(ftype)) } } + + +def read_file_data( + zipfilepath: Union[str, Path], + memberfilename: str, + processfile: Callable[[Iterator[str]], Iterator[str]] = lambda itr: itr, + processline: Callable[[str], str] = lambda line: line, + processfield: Callable[ + [Optional[str]], Optional[str]] = lambda val: val) -> Iterator[ + tuple[Optional[str], ...]]: + """Read a single file from the bundle processing each field.""" + cdata = read_control_file(zipfilepath) + return ( + tuple(processfield(field.strip()) + for field in processline(row.strip()).split(cdata["sep"])) + for row in + processfile( + strip_comments( + raw_file_data(zipfilepath, memberfilename), + cdata["comment.char"]))) -- cgit v1.2.3