about summary refs log tree commit diff
path: root/uploader/csvprocessing.py
blob: b135c6f491a26d1d8bff58b5fc0466c59ebfeacd (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
"""Utilities for processing CSV files."""
from pathlib import Path
from typing import Union, Callable, Iterator


def is_blank_line(line: str) -> bool:
    """Is the given line a blank line?

    :param line: The line to check for blankness.
    :type line: str
    :rtype bool
    """
    return line.strip() == ""


def make_comment_line_checker(comment_char: str) -> Callable[[str], bool]:
    """Make function that checks whether a line is a comment line.

    :param comment_char: The character indicating the line is a comment.
    :type comment_char: str
    :rtype Callable[[str], bool]
    """
    def _is_comment_line_(line: str) -> bool:
        """Check whether the line is a comment.

        :param line: The line to check.
        :type line: str
        :rtype bool
        """
        return line.startswith(comment_char)

    return _is_comment_line_


def read_text_file(filepath: Union[str, Path]) -> Iterator[str]:
    """Read the raw text from a text file.

    :param filepath: Path to the plain text file to read.
    :type filepath: str | pathlib.Path
    :rtype Iterator[str]
    """
    with Path(filepath).open("r", encoding="utf8") as _file:
        yield from _file


def read_csv_file(
        filepath: Path,
        separator: str,
        is_skippable_line: Callable[[str], bool]
) -> Iterator[tuple[str, ...]]:
    """Read a file as a csv file. This does not process the N/A values.

    :param filepath: Path to the file to read as a CSV file.
    :type filepath: pathlib.Path
    :param separator: The fields' separator in the CSV file.
    :type separator: str
    :param is_skippable_line: Function to determine whether or not to skip a
           particular line when parsing, e.g. comment lines.
    :type is_skippable_line: typing.Callable[[str], bool]
    :rtype typing.Iterator[tuple[str, ...]]
    """
    for line in read_text_file(filepath):
        if is_skippable_line(line):
            continue
        yield tuple(field.strip() for field in line.split(separator))


def read_csv_file_headers(
        filepath: Path,
        transposed: bool,
        separator: str,
        is_skippable_line: Callable[[str], bool]
) -> tuple[str, ...]:
    """Read the 'true' headers of a CSV file.

    :param filepath: Path to the CSV file to read.
    :type filepath: pathlib.Path
    :parama transposed: Is the file transposed or not?
    :type transposed: bool
    :param separator: The fields' separator in the CSV file.
    :type separator: str
    :param comment_char: The character which, when it begins a line, indicates
           that that line is a comment.
    :type comment_char: str
    :param is_skippable_line: Function to determine whether or not to skip a
           particular line when parsing, e.g. comment lines.
    :type is_skippable_line: typing.Callable[[str], bool]
    :rtype tuple[str, ...]
    """
    headers: tuple[str, ...] = tuple()
    for line in read_text_file(filepath):
        if is_skippable_line(line):
            continue

        row = tuple(field.strip() for field in line.split(separator))
        if not transposed:
            return row

        headers = headers + (row[0],)
        continue

    return headers