diff options
| author | pi coding agent <pi@localhost> | 2026-10-15 20:44:47 +0000 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-15 16:00:32 -0500 |
| commit | 930dfee072eeba7482fbd2828efd8fa3922b138f (patch) | |
| tree | 3f620daff3b1f263b79fba4fea00da6334ab3c95 /scripts | |
| parent | 43fc51a9191306ddb48d94518f6d85987d0fb3e6 (diff) | |
| download | gn-uploader-930dfee072eeba7482fbd2828efd8fa3922b138f.tar.gz | |
Implement multi-processed file statistics collection
Implement `file_stats_multiprocessing`, which streams the file lines into bounded chunks, processes each chunk in a worker process and merges the per-chunk statistics in file order so that the result matches `file_stats_singlethread`. The worker (`_process_chunk`) is a module-level function that only depends on picklable data (headers, configuration and the chunk of lines), so it works with any multiprocessing start method. Also fix the file-size threshold check in `compute_statistics`: it compared the size in megabytes against a threshold expressed in bytes, making the multi-processed branch effectively unreachable. Compare bytes to bytes instead.
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/genotypes/preprocess_csv_collect_info.py | 49 |
1 files changed, 48 insertions, 1 deletions
diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py index 79fc1fa..0fb1b96 100644 --- a/scripts/genotypes/preprocess_csv_collect_info.py +++ b/scripts/genotypes/preprocess_csv_collect_info.py @@ -3,6 +3,7 @@ import os import sys import json import logging +import multiprocessing as mproc from pathlib import Path from dataclasses import asdict, dataclass, field, fields from typing import Union, Iterable, Iterator, Optional, Callable @@ -31,6 +32,7 @@ from .types import (FileStats, logger = logging.getLogger(__name__) FILE_SIZE_THRESHOLD = 100 * 1024 * 1024 # 100MB +MULTIPROCESSING_CHUNK_SIZE = 10_000 # lines def filecontents(path: Path) -> Iterator[str]: @@ -289,6 +291,51 @@ def _merge_all_chunk_stats( return merged +def _read_chunks( + lines: Iterator[str], + chunk_size: int +) -> Iterator[list[str]]: + """Group the lines into chunks of at most ``chunk_size`` lines.""" + chunk: list[str] = [] + for line in lines: + chunk.append(line) + if len(chunk) >= chunk_size: + yield chunk + chunk = [] + if chunk: + yield chunk + + +def _process_chunk(args: tuple[tuple[str, ...], dict, list[str]]) -> _ChunkStats: + """Process a single chunk of lines. + + This is a module-level function so that it can be pickled and run in a + separate process. + """ + headers, fileconfigs, chunk = args + return _accumulate(headers, fileconfigs, chunk) + + +def file_stats_multiprocessing( + fileconfigs: dict, + headers: tuple[str, ...], + lines: Iterator[str], + chunk_size: int = MULTIPROCESSING_CHUNK_SIZE +) -> FileStats: + """File stats from the lines in the file using multiple processes.""" + logger.debug("THE FILE HEADERS: %s", headers) + with mproc.Pool(max(1, mproc.cpu_count() - 1)) as pool: + chunks = ((headers, fileconfigs, chunk) + for chunk in _read_chunks(lines, chunk_size)) + merged = _merge_all_chunk_stats( + pool.imap(_process_chunk, chunks), + fileconfigs["transposed"]) + return _as_file_stats( + merged, + identify_columns(headers, fileconfigs).samples, + validate_headers(headers, fileconfigs)) + + def _as_file_stats( stats: _ChunkStats, samples: tuple[str, ...], @@ -321,7 +368,7 @@ def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats: basic_stats = os.stat(filepath) lines = read_text_file(filepath) return (file_stats_multiprocessing(fileconfigs, headers, lines) - if (basic_stats.st_size/(1024*1024)) >= FILE_SIZE_THRESHOLD + if basic_stats.st_size >= FILE_SIZE_THRESHOLD else file_stats_singlethread(fileconfigs, headers, lines)) |
