diff options
| author | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-02 10:08:14 -0500 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-02 10:08:14 -0500 |
| commit | 49260fefb471e3c005a1c900b111a6df1cd53fdb (patch) | |
| tree | fb264c2f9b5d0d687a8f2c4758273d3a07d4fe65 /uploader/csvprocessing.py | |
| parent | 8555c7d6a78519ee0d95482d9cd52f67ebab3ee0 (diff) | |
| download | gn-uploader-49260fefb471e3c005a1c900b111a6df1cd53fdb.tar.gz | |
Move CSV-file specific processing code to new module.
Diffstat (limited to 'uploader/csvprocessing.py')
| -rw-r--r-- | uploader/csvprocessing.py | 73 |
1 files changed, 72 insertions, 1 deletions
diff --git a/uploader/csvprocessing.py b/uploader/csvprocessing.py index 3b46515..b135c6f 100644 --- a/uploader/csvprocessing.py +++ b/uploader/csvprocessing.py @@ -1,5 +1,6 @@ """Utilities for processing CSV files.""" -from typing import Callable +from pathlib import Path +from typing import Union, Callable, Iterator def is_blank_line(line: str) -> bool: @@ -29,3 +30,73 @@ def make_comment_line_checker(comment_char: str) -> Callable[[str], bool]: return line.startswith(comment_char) return _is_comment_line_ + + +def read_text_file(filepath: Union[str, Path]) -> Iterator[str]: + """Read the raw text from a text file. + + :param filepath: Path to the plain text file to read. + :type filepath: str | pathlib.Path + :rtype Iterator[str] + """ + with Path(filepath).open("r", encoding="utf8") as _file: + yield from _file + + +def read_csv_file( + filepath: Path, + separator: str, + is_skippable_line: Callable[[str], bool] +) -> Iterator[tuple[str, ...]]: + """Read a file as a csv file. This does not process the N/A values. + + :param filepath: Path to the file to read as a CSV file. + :type filepath: pathlib.Path + :param separator: The fields' separator in the CSV file. + :type separator: str + :param is_skippable_line: Function to determine whether or not to skip a + particular line when parsing, e.g. comment lines. + :type is_skippable_line: typing.Callable[[str], bool] + :rtype typing.Iterator[tuple[str, ...]] + """ + for line in read_text_file(filepath): + if is_skippable_line(line): + continue + yield tuple(field.strip() for field in line.split(separator)) + + +def read_csv_file_headers( + filepath: Path, + transposed: bool, + separator: str, + is_skippable_line: Callable[[str], bool] +) -> tuple[str, ...]: + """Read the 'true' headers of a CSV file. + + :param filepath: Path to the CSV file to read. + :type filepath: pathlib.Path + :parama transposed: Is the file transposed or not? + :type transposed: bool + :param separator: The fields' separator in the CSV file. + :type separator: str + :param comment_char: The character which, when it begins a line, indicates + that that line is a comment. + :type comment_char: str + :param is_skippable_line: Function to determine whether or not to skip a + particular line when parsing, e.g. comment lines. + :type is_skippable_line: typing.Callable[[str], bool] + :rtype tuple[str, ...] + """ + headers: tuple[str, ...] = tuple() + for line in read_text_file(filepath): + if is_skippable_line(line): + continue + + row = tuple(field.strip() for field in line.split(separator)) + if not transposed: + return row + + headers = headers + (row[0],) + continue + + return headers |
