about summary refs log tree commit diff
diff options
context:
space:
mode:
authorpi coding agent <pi@localhost>2026-10-15 20:44:47 +0000
committerFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-10-15 16:00:32 -0500
commit930dfee072eeba7482fbd2828efd8fa3922b138f (patch)
tree3f620daff3b1f263b79fba4fea00da6334ab3c95
parent43fc51a9191306ddb48d94518f6d85987d0fb3e6 (diff)
downloadgn-uploader-930dfee072eeba7482fbd2828efd8fa3922b138f.tar.gz
Implement multi-processed file statistics collection
Implement `file_stats_multiprocessing`, which streams the file lines into
bounded chunks, processes each chunk in a worker process and merges the
per-chunk statistics in file order so that the result matches
`file_stats_singlethread`.

The worker (`_process_chunk`) is a module-level function that only depends on
picklable data (headers, configuration and the chunk of lines), so it works
with any multiprocessing start method.

Also fix the file-size threshold check in `compute_statistics`: it compared
the size in megabytes against a threshold expressed in bytes, making the
multi-processed branch effectively unreachable. Compare bytes to bytes
instead.
-rw-r--r--scripts/genotypes/preprocess_csv_collect_info.py49
1 files changed, 48 insertions, 1 deletions
diff --git a/scripts/genotypes/preprocess_csv_collect_info.py b/scripts/genotypes/preprocess_csv_collect_info.py
index 79fc1fa..0fb1b96 100644
--- a/scripts/genotypes/preprocess_csv_collect_info.py
+++ b/scripts/genotypes/preprocess_csv_collect_info.py
@@ -3,6 +3,7 @@ import os
 import sys
 import json
 import logging
+import multiprocessing as mproc
 from pathlib import Path
 from dataclasses import asdict, dataclass, field, fields
 from typing import Union, Iterable, Iterator, Optional, Callable
@@ -31,6 +32,7 @@ from .types import (FileStats,
 logger = logging.getLogger(__name__)
 
 FILE_SIZE_THRESHOLD = 100 * 1024 * 1024 # 100MB
+MULTIPROCESSING_CHUNK_SIZE = 10_000 # lines
 
 
 def filecontents(path: Path) -> Iterator[str]:
@@ -289,6 +291,51 @@ def _merge_all_chunk_stats(
     return merged
 
 
+def _read_chunks(
+        lines: Iterator[str],
+        chunk_size: int
+) -> Iterator[list[str]]:
+    """Group the lines into chunks of at most ``chunk_size`` lines."""
+    chunk: list[str] = []
+    for line in lines:
+        chunk.append(line)
+        if len(chunk) >= chunk_size:
+            yield chunk
+            chunk = []
+    if chunk:
+        yield chunk
+
+
+def _process_chunk(args: tuple[tuple[str, ...], dict, list[str]]) -> _ChunkStats:
+    """Process a single chunk of lines.
+
+    This is a module-level function so that it can be pickled and run in a
+    separate process.
+    """
+    headers, fileconfigs, chunk = args
+    return _accumulate(headers, fileconfigs, chunk)
+
+
+def file_stats_multiprocessing(
+        fileconfigs: dict,
+        headers: tuple[str, ...],
+        lines: Iterator[str],
+        chunk_size: int = MULTIPROCESSING_CHUNK_SIZE
+) -> FileStats:
+    """File stats from the lines in the file using multiple processes."""
+    logger.debug("THE FILE HEADERS: %s", headers)
+    with mproc.Pool(max(1, mproc.cpu_count() - 1)) as pool:
+        chunks = ((headers, fileconfigs, chunk)
+                  for chunk in _read_chunks(lines, chunk_size))
+        merged = _merge_all_chunk_stats(
+            pool.imap(_process_chunk, chunks),
+            fileconfigs["transposed"])
+    return _as_file_stats(
+        merged,
+        identify_columns(headers, fileconfigs).samples,
+        validate_headers(headers, fileconfigs))
+
+
 def _as_file_stats(
         stats: _ChunkStats,
         samples: tuple[str, ...],
@@ -321,7 +368,7 @@ def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats:
     basic_stats = os.stat(filepath)
     lines = read_text_file(filepath)
     return (file_stats_multiprocessing(fileconfigs, headers, lines)
-            if (basic_stats.st_size/(1024*1024)) >= FILE_SIZE_THRESHOLD
+            if basic_stats.st_size >= FILE_SIZE_THRESHOLD
             else file_stats_singlethread(fileconfigs, headers, lines))