about summary refs log tree commit diff
path: root/uploader/genotypes/qc.py
blob: 66b6242c3b7f170fcf2f952d0fbe4ce93bad186f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
"""Quality-control checks for genotype files."""
import warnings
from typing import Any, Mapping, Sequence

from uploader.csvprocessing import (is_blank_line, make_comment_line_checker,
                                    read_csv_file_headers, read_text_file)

SPECIAL_FIELD_KEYS = ("markers", "chromosome", "cm", "mb")
SORT_WARNING = (
    "Marker ordering is a placeholder: records are sorted by chromosome, then "
    "Mb, then cM. The appropriate ordering has not been established yet."
)


def allowed_symbols(stage02: Mapping[str, Any]) -> set[str]:
    """Return the genotype symbols allowed by the Stage 02 encoding."""
    _encoding = stage02.get("geno_encoding") or {}
    _symbols = {_encoding.get(key, "") for key in ("mat", "pat", "het")}
    _symbols.update(stage02.get("missing_symbols") or [])
    return {symbol for symbol in _symbols if symbol}


def sort_markers(
        markers: Sequence[Mapping[str, Any]]
) -> list[Mapping[str, Any]]:
    """Sort marker records by chromosome, then cM, then Mb.

    TODO: replace this placeholder ordering once the appropriate ordering has
    been established.
    """
    warnings.warn(SORT_WARNING, stacklevel=2)
    return sorted(
        markers,
        key=lambda marker: (
            str(marker.get("chromosome", "")),
            _position(marker.get("mb")),
            _position(marker.get("cm"))))


def check_file(filepath, config: Mapping[str, Any], stage02: Mapping[str, Any]) -> dict:
    """Check the genotype values in a normalised file against the encoding."""
    _allowed = allowed_symbols(stage02)
    _is_skippable = _make_is_skippable(config)
    _splitter = _make_splitter(config)
    headers = read_csv_file_headers(
        filepath, False, config["separator"], _is_skippable)
    errors, checked = _check_untransposed(
        filepath, config, headers, _allowed, _is_skippable, _splitter)
    return {"values_checked": checked, "errors": errors}


def _position(value: Any) -> tuple[int, float]:
    try:
        return (0, float(value))
    except (TypeError, ValueError):
        return (1, 0.0)


def _make_splitter(config: Mapping[str, Any]):
    def __splitter__(line: str) -> tuple[str, ...]:
        return tuple(field.strip() for field in line.split(config["separator"]))
    return __splitter__


def _make_is_skippable(config: Mapping[str, Any]):
    _is_comment = make_comment_line_checker(config["comment.char"])

    def __is_skippable__(line: str) -> bool:
        return (is_blank_line(line) or _is_comment(line)
                or line.startswith("@"))
    return __is_skippable__


def _error(lineno: int, row: str, column: str, value: str, message: str) -> dict:
    return {
        "line": lineno, "row": row, "column": column,
        "value": value, "message": message}


def _check_value(# pylint: disable=[too-many-arguments, too-many-positional-arguments]
        lineno: int,
        row: str,
        column: str,
        value: str,
        allowed: set[str],
        errors: list[dict]
):
    if value == "":
        errors.append(_error(
            lineno, row, column, value, "Empty genotype value."))
    elif value not in allowed:
        errors.append(_error(
            lineno, row, column, value,
            f"Unknown genotype symbol '{value}'."))


def _check_untransposed(# pylint: disable=[too-many-locals, too-many-arguments, too-many-positional-arguments]
        filepath, config, headers, allowed, is_skippable, splitter
) -> tuple[list[dict], int]:
    _special = {config[key] for key in SPECIAL_FIELD_KEYS}
    _sample_columns = tuple(
        (idx, header) for idx, header in enumerate(headers)
        if header not in _special)
    try:
        _marker_idx = headers.index(config["markers"])
    except ValueError:
        _marker_idx = 0
    try:
        _chrom_idx = headers.index(config["chromosome"])
    except ValueError:
        _chrom_idx = None
    _seen_chrom_markers: dict[str, set[str]] = {}
    errors: list[dict] = []
    checked = 0
    lineno = 0
    for line in read_text_file(filepath):
        lineno += 1
        line = line.strip()
        if is_skippable(line):
            continue
        fields = splitter(line)
        if fields == headers:
            continue
        if len(fields) != len(headers):
            errors.append(_error(
                lineno,
                fields[_marker_idx] if _marker_idx < len(fields) else "",
                "-", "", (
                    f"Columns mismatch: {len(headers)} headers but "
                    f"{len(fields)} field values.")))
            continue
        marker = fields[_marker_idx] if _marker_idx < len(fields) else ""
        if _chrom_idx is not None and _chrom_idx < len(fields):
            chromosome = fields[_chrom_idx]
            chrom_markers = _seen_chrom_markers.setdefault(chromosome, set())
            if marker and chromosome:
                if marker in chrom_markers:
                    errors.append(_error(
                        lineno, marker, config["markers"], marker,
                        (f"Duplicate marker '{marker}' on chromosome "
                         f"'{chromosome}'.")))
                else:
                    chrom_markers.add(marker)
        for idx, header in _sample_columns:
            checked += 1
            _check_value(lineno, marker, header, fields[idx], allowed, errors)
    return errors, checked