diff options
| author | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-11-04 12:38:45 -0600 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-11-04 13:28:10 -0600 |
| commit | b301c6d61b99c5ad42e97a1c8c714c466253c41b (patch) | |
| tree | 71223a4d12e258cb7c53f52c539709695d00a58a /tests/uploader/scripts | |
| parent | 27dcfcdb7a588c44c7a267584628b2ddf719da21 (diff) | |
| download | gn-uploader-b301c6d61b99c5ad42e97a1c8c714c466253c41b.tar.gz | |
Re-organise the tests.
Diffstat (limited to 'tests/uploader/scripts')
4 files changed, 338 insertions, 0 deletions
diff --git a/tests/uploader/scripts/genotypes/__init__.py b/tests/uploader/scripts/genotypes/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/uploader/scripts/genotypes/__init__.py diff --git a/tests/uploader/scripts/genotypes/test_merge_chunk_stats.py b/tests/uploader/scripts/genotypes/test_merge_chunk_stats.py new file mode 100644 index 0000000..96b19bd --- /dev/null +++ b/tests/uploader/scripts/genotypes/test_merge_chunk_stats.py @@ -0,0 +1,106 @@ +"""Tests for merging statistics accumulated from consecutive chunks.""" +import pytest + +from uploader.scripts.genotypes.preprocess_csv_collect_info import ( + _ChunkStats, + _merge_chunk_stats, + _merge_all_chunk_stats) + +from uploader.scripts.genotypes.types import LineError + + +def _line_error(message: str) -> LineError: + """Build a simple line error for the tests.""" + return LineError(row_header="m1", column_header="-", message=message, + line=("m1", "1", "A")) + + +@pytest.mark.unit_test +def test_merge_chunk_stats_sums_counters(): + """ + GIVEN: two sets of chunk statistics + WHEN: they are merged + THEN: all the line counters are summed + """ + left = _ChunkStats(total_lines=3, data_lines=2, comment_lines=1) + right = _ChunkStats(total_lines=2, data_lines=1, blank_lines=1) + merged = _merge_chunk_stats(left, right, transposed=False) + assert merged.total_lines == 5 + assert merged.data_lines == 3 + assert merged.comment_lines == 1 + assert merged.blank_lines == 1 + + +@pytest.mark.unit_test +def test_merge_chunk_stats_unions_symbols_and_orders_errors(): + """ + GIVEN: two sets of chunk statistics with symbols, errors and dot-geno fields + WHEN: they are merged + THEN: symbols are unioned, errors keep file order and later fields win + """ + left = _ChunkStats(geno_symbols={"A"}, + errors=(_line_error("first"),), + dot_geno_fields={"name": "a", "mat": "a"}) + right = _ChunkStats(geno_symbols={"B"}, + errors=(_line_error("second"),), + dot_geno_fields={"name": "b"}) + merged = _merge_chunk_stats(left, right, transposed=False) + assert merged.geno_symbols == {"A", "B"} + assert [error.message for error in merged.errors] == ["first", "second"] + assert merged.dot_geno_fields == {"name": "b", "mat": "a"} + + +@pytest.mark.unit_test +def test_merge_chunk_stats_concatenates_untransposed_markers(): + """ + GIVEN: two sets of chunk statistics from an untransposed file + WHEN: they are merged + THEN: the marker names are concatenated in file order + """ + left = _ChunkStats(markers=("m1",)) + right = _ChunkStats(markers=("m2", "m3")) + merged = _merge_chunk_stats(left, right, transposed=False) + assert merged.markers == ("m1", "m2", "m3") + + +@pytest.mark.unit_test +def test_merge_chunk_stats_keeps_last_transposed_markers(): + """ + GIVEN: two sets of chunk statistics from a transposed file + WHEN: the later chunk saw a markers line + THEN: the later chunk's markers are used + """ + left = _ChunkStats(markers=("m1",), markers_seen=True) + right = _ChunkStats(markers=("m2", "m3"), markers_seen=True) + merged = _merge_chunk_stats(left, right, transposed=True) + assert merged.markers == ("m2", "m3") + assert merged.markers_seen is True + + +@pytest.mark.unit_test +def test_merge_chunk_stats_falls_back_to_earlier_transposed_markers(): + """ + GIVEN: two sets of chunk statistics from a transposed file + WHEN: the later chunk saw no markers line + THEN: the earlier chunk's markers are retained + """ + left = _ChunkStats(markers=("m1",), markers_seen=True) + right = _ChunkStats() + merged = _merge_chunk_stats(left, right, transposed=True) + assert merged.markers == ("m1",) + assert merged.markers_seen is True + + +@pytest.mark.unit_test +def test_merge_all_chunk_stats_folds_in_order(): + """ + GIVEN: a sequence of chunk statistics + WHEN: they are all merged + THEN: the result reflects every chunk in order + """ + chunks = (_ChunkStats(total_lines=1, markers=("m1",)), + _ChunkStats(total_lines=2, markers=("m2",)), + _ChunkStats(total_lines=3, markers=("m3",))) + merged = _merge_all_chunk_stats(chunks, transposed=False) + assert merged.total_lines == 6 + assert merged.markers == ("m1", "m2", "m3") diff --git a/tests/uploader/scripts/genotypes/test_multiprocessing.py b/tests/uploader/scripts/genotypes/test_multiprocessing.py new file mode 100644 index 0000000..bea3955 --- /dev/null +++ b/tests/uploader/scripts/genotypes/test_multiprocessing.py @@ -0,0 +1,91 @@ +"""Tests for the multi-processed file statistics collection.""" +import pytest + +from uploader.scripts.genotypes.preprocess_csv_collect_info import ( + _read_chunks, + file_stats_singlethread, + file_stats_multiprocessing) + +from tests.uploader.scripts.genotypes.test_preprocess_csv_collect_info import ( + UNTRANSPOSED_CONFIG, + UNTRANSPOSED_HEADERS, + UNTRANSPOSED_LINES, + TRANSPOSED_CONFIG, + TRANSPOSED_HEADERS, + TRANSPOSED_LINES) + + +@pytest.mark.unit_test +@pytest.mark.parametrize("chunk_size", [1, 2, 5, 1000]) +def test_multiprocessing_matches_singlethread_for_untransposed_files(chunk_size): + """ + GIVEN: an untransposed genotype file + WHEN: statistics are computed with multiple processes and various chunk sizes + THEN: the result matches the single-threaded statistics + """ + expected = file_stats_singlethread(UNTRANSPOSED_CONFIG, + UNTRANSPOSED_HEADERS, + list(UNTRANSPOSED_LINES)) + result = file_stats_multiprocessing(UNTRANSPOSED_CONFIG, + UNTRANSPOSED_HEADERS, + list(UNTRANSPOSED_LINES), + chunk_size=chunk_size) + assert result == expected + + +@pytest.mark.unit_test +@pytest.mark.parametrize("chunk_size", [1, 3, 1000]) +def test_multiprocessing_matches_singlethread_for_transposed_files(chunk_size): + """ + GIVEN: a transposed genotype file + WHEN: statistics are computed with multiple processes and various chunk sizes + THEN: the result matches the single-threaded statistics + """ + expected = file_stats_singlethread(TRANSPOSED_CONFIG, + TRANSPOSED_HEADERS, + list(TRANSPOSED_LINES)) + result = file_stats_multiprocessing(TRANSPOSED_CONFIG, + TRANSPOSED_HEADERS, + list(TRANSPOSED_LINES), + chunk_size=chunk_size) + assert result == expected + + +@pytest.mark.unit_test +def test_multiprocessing_matches_singlethread_with_line_errors(): + """ + GIVEN: a file containing a line with a column mismatch + WHEN: statistics are computed with multiple processes + THEN: the errors are reported in the same order as the single-threaded run + """ + lines = list(UNTRANSPOSED_LINES) + ["m3,2,x,y\n"] + expected = file_stats_singlethread(UNTRANSPOSED_CONFIG, + UNTRANSPOSED_HEADERS, + list(lines)) + result = file_stats_multiprocessing(UNTRANSPOSED_CONFIG, + UNTRANSPOSED_HEADERS, + list(lines), + chunk_size=2) + assert result == expected + assert len(result.errors) > 0 + + +@pytest.mark.unit_test +def test_read_chunks_groups_lines_including_partial_final_chunk(): + """ + GIVEN: an iterable of lines and a chunk size + WHEN: the lines are grouped into chunks + THEN: every chunk has the requested size except possibly the last one + """ + chunks = list(_read_chunks(iter(["a", "b", "c", "d", "e"]), 2)) + assert chunks == [["a", "b"], ["c", "d"], ["e"]] + + +@pytest.mark.unit_test +def test_read_chunks_yields_nothing_for_empty_input(): + """ + GIVEN: an empty iterable of lines + WHEN: the lines are grouped into chunks + THEN: no chunks are produced + """ + assert list(_read_chunks(iter([]), 3)) == [] diff --git a/tests/uploader/scripts/genotypes/test_preprocess_csv_collect_info.py b/tests/uploader/scripts/genotypes/test_preprocess_csv_collect_info.py new file mode 100644 index 0000000..a45927c --- /dev/null +++ b/tests/uploader/scripts/genotypes/test_preprocess_csv_collect_info.py @@ -0,0 +1,141 @@ +"""Tests for the per-line genotype file statistics collection.""" +import pytest + +from uploader.scripts.genotypes.preprocess_csv_collect_info import ( + _accumulate, + _ChunkStats, + file_stats_singlethread) + +from uploader.scripts.genotypes.types import DotGenoFields + + +UNTRANSPOSED_CONFIG = { + "markers": "Locus", + "chromosome": "Chr", + "cm": "cM", + "mb": "Mb", + "separator": ",", + "comment.char": "#", + "na_strings": ["-", "NA", "N/A"], + "transposed": False, +} + +TRANSPOSED_CONFIG = {**UNTRANSPOSED_CONFIG, "transposed": True} + +UNTRANSPOSED_HEADERS = ("Locus", "Chr", "cM", "Mb", "s1", "s2") + +UNTRANSPOSED_LINES = ( + "Locus,Chr,cM,Mb,s1,s2\n", + "m1,1,0.0,1.0,A,T\n", + "m2,1,1.0,2.0,C,G\n", + "# a comment\n", + "\n", + "@name:test\n", + "@mat:A\n", + "@pat:B\n", + "@het:H\n", +) + +TRANSPOSED_HEADERS = ("Locus", "Chr", "cM", "Mb", "s1", "s2") + +TRANSPOSED_LINES = ( + "Locus,m1,m2,m3\n", + "Chr,1,1,2\n", + "cM,0,1,2\n", + "Mb,0,1,2\n", + "s1,A,T,C\n", + "s2,G,C,A\n", +) + + +@pytest.mark.unit_test +def test_accumulate_counts_each_kind_of_line(): + """ + GIVEN: a mixture of data, comment, blank and dot-geno lines + WHEN: the lines are accumulated + THEN: each type of line is counted separately + """ + stats = _accumulate(UNTRANSPOSED_HEADERS, + UNTRANSPOSED_CONFIG, + list(UNTRANSPOSED_LINES)) + assert isinstance(stats, _ChunkStats) + assert stats.total_lines == 9 + assert stats.data_lines == 2 + assert stats.comment_lines == 1 + assert stats.blank_lines == 1 + assert stats.dot_geno_lines == 4 + + +@pytest.mark.unit_test +def test_accumulate_collects_symbols_markers_and_dot_geno_fields(): + """ + GIVEN: genotype data with sample columns and dot-geno directives + WHEN: the lines are accumulated + THEN: the genotype symbols, marker names and dot-geno fields are collected + """ + stats = _accumulate(UNTRANSPOSED_HEADERS, + UNTRANSPOSED_CONFIG, + list(UNTRANSPOSED_LINES)) + assert stats.geno_symbols == {"A", "T", "C", "G"} + assert stats.markers == ("m1", "m2") + assert stats.markers_seen is False + assert stats.dot_geno_fields == { + "name": "test", "mat": "A", "pat": "B", "het": "H"} + assert stats.errors == () + + +@pytest.mark.unit_test +def test_accumulate_handles_transposed_files(): + """ + GIVEN: a transposed genotype file + WHEN: the lines are accumulated + THEN: the markers are taken from the markers line and no errors reported + """ + stats = _accumulate(TRANSPOSED_HEADERS, + TRANSPOSED_CONFIG, + list(TRANSPOSED_LINES)) + assert stats.total_lines == 6 + assert stats.data_lines == 6 + assert stats.geno_symbols == {"A", "T", "C", "G"} + assert stats.markers == ("m1", "m2", "m3") + assert stats.markers_seen is True + assert stats.errors == () + + +@pytest.mark.unit_test +def test_singlethread_reports_expected_file_stats(): + """ + GIVEN: an untransposed genotype file with dot-geno directives + WHEN: statistics are computed with a single thread + THEN: the resulting file statistics mirror the accumulated statistics + """ + stats = file_stats_singlethread(UNTRANSPOSED_CONFIG, + UNTRANSPOSED_HEADERS, + list(UNTRANSPOSED_LINES)) + assert stats.total_lines == 9 + assert stats.data_lines == 2 + assert stats.comment_lines == 1 + assert stats.blank_lines == 1 + assert stats.dot_geno_lines == 4 + assert set(stats.geno_symbols) == {"A", "T", "C", "G"} + assert stats.samples_list == ("s1", "s2") + assert stats.markers_list == ("m1", "m2") + assert stats.dot_geno_fields == DotGenoFields( + name="test", mat="A", pat="B", het="H") + assert stats.errors == () + + +@pytest.mark.unit_test +def test_singlethread_reports_missing_headers_before_line_errors(): + """ + GIVEN: headers missing the genetic/physical distance columns + WHEN: statistics are computed with a single thread + THEN: the header errors are reported ahead of any line errors + """ + config = {**UNTRANSPOSED_CONFIG} + headers = ("Locus", "Chr", "s1") + lines = ("m1,1,A\n",) + stats = file_stats_singlethread(config, headers, list(lines)) + assert len(stats.errors) == 2 + assert all(error.column_header == "-" for error in stats.errors) + assert stats.markers_list == ("m1",) |
