about summary refs log tree commit diff
path: root/tests/uploader/scripts
diff options
context:
space:
mode:
authorFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-11-04 12:38:45 -0600
committerFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-11-04 13:28:10 -0600
commitb301c6d61b99c5ad42e97a1c8c714c466253c41b (patch)
tree71223a4d12e258cb7c53f52c539709695d00a58a /tests/uploader/scripts
parent27dcfcdb7a588c44c7a267584628b2ddf719da21 (diff)
downloadgn-uploader-b301c6d61b99c5ad42e97a1c8c714c466253c41b.tar.gz
Re-organise the tests.
Diffstat (limited to 'tests/uploader/scripts')
-rw-r--r--tests/uploader/scripts/genotypes/__init__.py0
-rw-r--r--tests/uploader/scripts/genotypes/test_merge_chunk_stats.py106
-rw-r--r--tests/uploader/scripts/genotypes/test_multiprocessing.py91
-rw-r--r--tests/uploader/scripts/genotypes/test_preprocess_csv_collect_info.py141
4 files changed, 338 insertions, 0 deletions
diff --git a/tests/uploader/scripts/genotypes/__init__.py b/tests/uploader/scripts/genotypes/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/uploader/scripts/genotypes/__init__.py
diff --git a/tests/uploader/scripts/genotypes/test_merge_chunk_stats.py b/tests/uploader/scripts/genotypes/test_merge_chunk_stats.py
new file mode 100644
index 0000000..96b19bd
--- /dev/null
+++ b/tests/uploader/scripts/genotypes/test_merge_chunk_stats.py
@@ -0,0 +1,106 @@
+"""Tests for merging statistics accumulated from consecutive chunks."""
+import pytest
+
+from uploader.scripts.genotypes.preprocess_csv_collect_info import (
+    _ChunkStats,
+    _merge_chunk_stats,
+    _merge_all_chunk_stats)
+
+from uploader.scripts.genotypes.types import LineError
+
+
+def _line_error(message: str) -> LineError:
+    """Build a simple line error for the tests."""
+    return LineError(row_header="m1", column_header="-", message=message,
+                     line=("m1", "1", "A"))
+
+
+@pytest.mark.unit_test
+def test_merge_chunk_stats_sums_counters():
+    """
+    GIVEN: two sets of chunk statistics
+    WHEN: they are merged
+    THEN: all the line counters are summed
+    """
+    left = _ChunkStats(total_lines=3, data_lines=2, comment_lines=1)
+    right = _ChunkStats(total_lines=2, data_lines=1, blank_lines=1)
+    merged = _merge_chunk_stats(left, right, transposed=False)
+    assert merged.total_lines == 5
+    assert merged.data_lines == 3
+    assert merged.comment_lines == 1
+    assert merged.blank_lines == 1
+
+
+@pytest.mark.unit_test
+def test_merge_chunk_stats_unions_symbols_and_orders_errors():
+    """
+    GIVEN: two sets of chunk statistics with symbols, errors and dot-geno fields
+    WHEN: they are merged
+    THEN: symbols are unioned, errors keep file order and later fields win
+    """
+    left = _ChunkStats(geno_symbols={"A"},
+                       errors=(_line_error("first"),),
+                       dot_geno_fields={"name": "a", "mat": "a"})
+    right = _ChunkStats(geno_symbols={"B"},
+                        errors=(_line_error("second"),),
+                        dot_geno_fields={"name": "b"})
+    merged = _merge_chunk_stats(left, right, transposed=False)
+    assert merged.geno_symbols == {"A", "B"}
+    assert [error.message for error in merged.errors] == ["first", "second"]
+    assert merged.dot_geno_fields == {"name": "b", "mat": "a"}
+
+
+@pytest.mark.unit_test
+def test_merge_chunk_stats_concatenates_untransposed_markers():
+    """
+    GIVEN: two sets of chunk statistics from an untransposed file
+    WHEN: they are merged
+    THEN: the marker names are concatenated in file order
+    """
+    left = _ChunkStats(markers=("m1",))
+    right = _ChunkStats(markers=("m2", "m3"))
+    merged = _merge_chunk_stats(left, right, transposed=False)
+    assert merged.markers == ("m1", "m2", "m3")
+
+
+@pytest.mark.unit_test
+def test_merge_chunk_stats_keeps_last_transposed_markers():
+    """
+    GIVEN: two sets of chunk statistics from a transposed file
+    WHEN: the later chunk saw a markers line
+    THEN: the later chunk's markers are used
+    """
+    left = _ChunkStats(markers=("m1",), markers_seen=True)
+    right = _ChunkStats(markers=("m2", "m3"), markers_seen=True)
+    merged = _merge_chunk_stats(left, right, transposed=True)
+    assert merged.markers == ("m2", "m3")
+    assert merged.markers_seen is True
+
+
+@pytest.mark.unit_test
+def test_merge_chunk_stats_falls_back_to_earlier_transposed_markers():
+    """
+    GIVEN: two sets of chunk statistics from a transposed file
+    WHEN: the later chunk saw no markers line
+    THEN: the earlier chunk's markers are retained
+    """
+    left = _ChunkStats(markers=("m1",), markers_seen=True)
+    right = _ChunkStats()
+    merged = _merge_chunk_stats(left, right, transposed=True)
+    assert merged.markers == ("m1",)
+    assert merged.markers_seen is True
+
+
+@pytest.mark.unit_test
+def test_merge_all_chunk_stats_folds_in_order():
+    """
+    GIVEN: a sequence of chunk statistics
+    WHEN: they are all merged
+    THEN: the result reflects every chunk in order
+    """
+    chunks = (_ChunkStats(total_lines=1, markers=("m1",)),
+              _ChunkStats(total_lines=2, markers=("m2",)),
+              _ChunkStats(total_lines=3, markers=("m3",)))
+    merged = _merge_all_chunk_stats(chunks, transposed=False)
+    assert merged.total_lines == 6
+    assert merged.markers == ("m1", "m2", "m3")
diff --git a/tests/uploader/scripts/genotypes/test_multiprocessing.py b/tests/uploader/scripts/genotypes/test_multiprocessing.py
new file mode 100644
index 0000000..bea3955
--- /dev/null
+++ b/tests/uploader/scripts/genotypes/test_multiprocessing.py
@@ -0,0 +1,91 @@
+"""Tests for the multi-processed file statistics collection."""
+import pytest
+
+from uploader.scripts.genotypes.preprocess_csv_collect_info import (
+    _read_chunks,
+    file_stats_singlethread,
+    file_stats_multiprocessing)
+
+from tests.uploader.scripts.genotypes.test_preprocess_csv_collect_info import (
+    UNTRANSPOSED_CONFIG,
+    UNTRANSPOSED_HEADERS,
+    UNTRANSPOSED_LINES,
+    TRANSPOSED_CONFIG,
+    TRANSPOSED_HEADERS,
+    TRANSPOSED_LINES)
+
+
+@pytest.mark.unit_test
+@pytest.mark.parametrize("chunk_size", [1, 2, 5, 1000])
+def test_multiprocessing_matches_singlethread_for_untransposed_files(chunk_size):
+    """
+    GIVEN: an untransposed genotype file
+    WHEN: statistics are computed with multiple processes and various chunk sizes
+    THEN: the result matches the single-threaded statistics
+    """
+    expected = file_stats_singlethread(UNTRANSPOSED_CONFIG,
+                                       UNTRANSPOSED_HEADERS,
+                                       list(UNTRANSPOSED_LINES))
+    result = file_stats_multiprocessing(UNTRANSPOSED_CONFIG,
+                                        UNTRANSPOSED_HEADERS,
+                                        list(UNTRANSPOSED_LINES),
+                                        chunk_size=chunk_size)
+    assert result == expected
+
+
+@pytest.mark.unit_test
+@pytest.mark.parametrize("chunk_size", [1, 3, 1000])
+def test_multiprocessing_matches_singlethread_for_transposed_files(chunk_size):
+    """
+    GIVEN: a transposed genotype file
+    WHEN: statistics are computed with multiple processes and various chunk sizes
+    THEN: the result matches the single-threaded statistics
+    """
+    expected = file_stats_singlethread(TRANSPOSED_CONFIG,
+                                       TRANSPOSED_HEADERS,
+                                       list(TRANSPOSED_LINES))
+    result = file_stats_multiprocessing(TRANSPOSED_CONFIG,
+                                        TRANSPOSED_HEADERS,
+                                        list(TRANSPOSED_LINES),
+                                        chunk_size=chunk_size)
+    assert result == expected
+
+
+@pytest.mark.unit_test
+def test_multiprocessing_matches_singlethread_with_line_errors():
+    """
+    GIVEN: a file containing a line with a column mismatch
+    WHEN: statistics are computed with multiple processes
+    THEN: the errors are reported in the same order as the single-threaded run
+    """
+    lines = list(UNTRANSPOSED_LINES) + ["m3,2,x,y\n"]
+    expected = file_stats_singlethread(UNTRANSPOSED_CONFIG,
+                                       UNTRANSPOSED_HEADERS,
+                                       list(lines))
+    result = file_stats_multiprocessing(UNTRANSPOSED_CONFIG,
+                                        UNTRANSPOSED_HEADERS,
+                                        list(lines),
+                                        chunk_size=2)
+    assert result == expected
+    assert len(result.errors) > 0
+
+
+@pytest.mark.unit_test
+def test_read_chunks_groups_lines_including_partial_final_chunk():
+    """
+    GIVEN: an iterable of lines and a chunk size
+    WHEN: the lines are grouped into chunks
+    THEN: every chunk has the requested size except possibly the last one
+    """
+    chunks = list(_read_chunks(iter(["a", "b", "c", "d", "e"]), 2))
+    assert chunks == [["a", "b"], ["c", "d"], ["e"]]
+
+
+@pytest.mark.unit_test
+def test_read_chunks_yields_nothing_for_empty_input():
+    """
+    GIVEN: an empty iterable of lines
+    WHEN: the lines are grouped into chunks
+    THEN: no chunks are produced
+    """
+    assert list(_read_chunks(iter([]), 3)) == []
diff --git a/tests/uploader/scripts/genotypes/test_preprocess_csv_collect_info.py b/tests/uploader/scripts/genotypes/test_preprocess_csv_collect_info.py
new file mode 100644
index 0000000..a45927c
--- /dev/null
+++ b/tests/uploader/scripts/genotypes/test_preprocess_csv_collect_info.py
@@ -0,0 +1,141 @@
+"""Tests for the per-line genotype file statistics collection."""
+import pytest
+
+from uploader.scripts.genotypes.preprocess_csv_collect_info import (
+    _accumulate,
+    _ChunkStats,
+    file_stats_singlethread)
+
+from uploader.scripts.genotypes.types import DotGenoFields
+
+
+UNTRANSPOSED_CONFIG = {
+    "markers": "Locus",
+    "chromosome": "Chr",
+    "cm": "cM",
+    "mb": "Mb",
+    "separator": ",",
+    "comment.char": "#",
+    "na_strings": ["-", "NA", "N/A"],
+    "transposed": False,
+}
+
+TRANSPOSED_CONFIG = {**UNTRANSPOSED_CONFIG, "transposed": True}
+
+UNTRANSPOSED_HEADERS = ("Locus", "Chr", "cM", "Mb", "s1", "s2")
+
+UNTRANSPOSED_LINES = (
+    "Locus,Chr,cM,Mb,s1,s2\n",
+    "m1,1,0.0,1.0,A,T\n",
+    "m2,1,1.0,2.0,C,G\n",
+    "# a comment\n",
+    "\n",
+    "@name:test\n",
+    "@mat:A\n",
+    "@pat:B\n",
+    "@het:H\n",
+)
+
+TRANSPOSED_HEADERS = ("Locus", "Chr", "cM", "Mb", "s1", "s2")
+
+TRANSPOSED_LINES = (
+    "Locus,m1,m2,m3\n",
+    "Chr,1,1,2\n",
+    "cM,0,1,2\n",
+    "Mb,0,1,2\n",
+    "s1,A,T,C\n",
+    "s2,G,C,A\n",
+)
+
+
+@pytest.mark.unit_test
+def test_accumulate_counts_each_kind_of_line():
+    """
+    GIVEN: a mixture of data, comment, blank and dot-geno lines
+    WHEN: the lines are accumulated
+    THEN: each type of line is counted separately
+    """
+    stats = _accumulate(UNTRANSPOSED_HEADERS,
+                        UNTRANSPOSED_CONFIG,
+                        list(UNTRANSPOSED_LINES))
+    assert isinstance(stats, _ChunkStats)
+    assert stats.total_lines == 9
+    assert stats.data_lines == 2
+    assert stats.comment_lines == 1
+    assert stats.blank_lines == 1
+    assert stats.dot_geno_lines == 4
+
+
+@pytest.mark.unit_test
+def test_accumulate_collects_symbols_markers_and_dot_geno_fields():
+    """
+    GIVEN: genotype data with sample columns and dot-geno directives
+    WHEN: the lines are accumulated
+    THEN: the genotype symbols, marker names and dot-geno fields are collected
+    """
+    stats = _accumulate(UNTRANSPOSED_HEADERS,
+                        UNTRANSPOSED_CONFIG,
+                        list(UNTRANSPOSED_LINES))
+    assert stats.geno_symbols == {"A", "T", "C", "G"}
+    assert stats.markers == ("m1", "m2")
+    assert stats.markers_seen is False
+    assert stats.dot_geno_fields == {
+        "name": "test", "mat": "A", "pat": "B", "het": "H"}
+    assert stats.errors == ()
+
+
+@pytest.mark.unit_test
+def test_accumulate_handles_transposed_files():
+    """
+    GIVEN: a transposed genotype file
+    WHEN: the lines are accumulated
+    THEN: the markers are taken from the markers line and no errors reported
+    """
+    stats = _accumulate(TRANSPOSED_HEADERS,
+                        TRANSPOSED_CONFIG,
+                        list(TRANSPOSED_LINES))
+    assert stats.total_lines == 6
+    assert stats.data_lines == 6
+    assert stats.geno_symbols == {"A", "T", "C", "G"}
+    assert stats.markers == ("m1", "m2", "m3")
+    assert stats.markers_seen is True
+    assert stats.errors == ()
+
+
+@pytest.mark.unit_test
+def test_singlethread_reports_expected_file_stats():
+    """
+    GIVEN: an untransposed genotype file with dot-geno directives
+    WHEN: statistics are computed with a single thread
+    THEN: the resulting file statistics mirror the accumulated statistics
+    """
+    stats = file_stats_singlethread(UNTRANSPOSED_CONFIG,
+                                    UNTRANSPOSED_HEADERS,
+                                    list(UNTRANSPOSED_LINES))
+    assert stats.total_lines == 9
+    assert stats.data_lines == 2
+    assert stats.comment_lines == 1
+    assert stats.blank_lines == 1
+    assert stats.dot_geno_lines == 4
+    assert set(stats.geno_symbols) == {"A", "T", "C", "G"}
+    assert stats.samples_list == ("s1", "s2")
+    assert stats.markers_list == ("m1", "m2")
+    assert stats.dot_geno_fields == DotGenoFields(
+        name="test", mat="A", pat="B", het="H")
+    assert stats.errors == ()
+
+
+@pytest.mark.unit_test
+def test_singlethread_reports_missing_headers_before_line_errors():
+    """
+    GIVEN: headers missing the genetic/physical distance columns
+    WHEN: statistics are computed with a single thread
+    THEN: the header errors are reported ahead of any line errors
+    """
+    config = {**UNTRANSPOSED_CONFIG}
+    headers = ("Locus", "Chr", "s1")
+    lines = ("m1,1,A\n",)
+    stats = file_stats_singlethread(config, headers, list(lines))
+    assert len(stats.errors) == 2
+    assert all(error.column_header == "-" for error in stats.errors)
+    assert stats.markers_list == ("m1",)