about summary refs log tree commit diff
path: root/uploader/scripts
diff options
context:
space:
mode:
authorpi coding agent <pi@localhost>2026-11-04 19:10:02 +0000
committerFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-11-04 13:28:11 -0600
commitfb9e85d123ca7d0853c4405d021e80936233fc17 (patch)
treeb2001ec158acce25ca0508dc1c2a657b21e8c5b2 /uploader/scripts
parent718777873f56170cabf06801076788e66ff60d45 (diff)
downloadgn-uploader-fb9e85d123ca7d0853c4405d021e80936233fc17.tar.gz
Collect comment lines into the file description
Add a `description` field to `FileStats`, built by cleaning comment lines
(strip the configured comment character and surrounding whitespace) and joining
them with newlines. Detect comments using the configured comment character
rather than a hard-coded '#', and preserve comment order when merging chunks.
Diffstat (limited to 'uploader/scripts')
-rw-r--r--uploader/scripts/genotypes/preprocess_csv_collect_info.py14
-rw-r--r--uploader/scripts/genotypes/types.py1
2 files changed, 11 insertions, 4 deletions
diff --git a/uploader/scripts/genotypes/preprocess_csv_collect_info.py b/uploader/scripts/genotypes/preprocess_csv_collect_info.py
index 870694b..e1978bd 100644
--- a/uploader/scripts/genotypes/preprocess_csv_collect_info.py
+++ b/uploader/scripts/genotypes/preprocess_csv_collect_info.py
@@ -203,6 +203,7 @@ class _ChunkStats:# pylint: disable=[too-many-instance-attributes]
     markers: tuple = field(default_factory=tuple)
     markers_seen: bool = False
     errors: tuple = field(default_factory=tuple)
+    comments: tuple = field(default_factory=tuple)
 
 
 def _accumulate(
@@ -212,6 +213,7 @@ def _accumulate(
 ) -> _ChunkStats:
     """Accumulate statistics from the provided lines."""
     is_skippable_line = _make_is_skippable_line(fileconfigs, headers)
+    is_comment_line = make_comment_line_checker(fileconfigs["comment.char"])
     line_splitter = _make_line_splitter(fileconfigs["separator"])
     stats = _ChunkStats()
     for line in (ln.strip() for ln in lines):
@@ -237,13 +239,15 @@ def _accumulate(
                     stats.markers = stats.markers + (str(line_dets.marker),)
             # TOD0: do more processing
             continue # end of line processing
-        if line.startswith("@"): #handle special geno formats
+        if is_dot_geno_line(line): #handle special geno formats
             stats.dot_geno_lines += 1
             stats.dot_geno_fields = {
                 **stats.dot_geno_fields,
                 **parse_dot_geno_line(line)}
-        if line.startswith("#"): #handle special geno formats
+        if is_comment_line(line): #handle special geno formats
             stats.comment_lines += 1
+            stats.comments = stats.comments + (
+                line[len(fileconfigs["comment.char"]):].strip(),)
         if line == "": # Blank line
             stats.blank_lines += 1
     return stats
@@ -276,7 +280,8 @@ def _merge_chunk_stats(
         geno_symbols=left.geno_symbols | right.geno_symbols,
         markers=markers,
         markers_seen=left.markers_seen or right.markers_seen,
-        errors=left.errors + right.errors)
+        errors=left.errors + right.errors,
+        comments=left.comments + right.comments)
 
 
 def _merge_all_chunk_stats(
@@ -351,7 +356,8 @@ def _as_file_stats(
         geno_symbols=tuple(sorted(stats.geno_symbols)),
         samples_list=tuple(samples),
         markers_list=stats.markers,
-        errors=tuple(header_errors) + tuple(stats.errors))
+        errors=tuple(header_errors) + tuple(stats.errors),
+        description="\n".join(stats.comments))
 
 
 def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats:
diff --git a/uploader/scripts/genotypes/types.py b/uploader/scripts/genotypes/types.py
index d765fe5..4403f3d 100644
--- a/uploader/scripts/genotypes/types.py
+++ b/uploader/scripts/genotypes/types.py
@@ -67,3 +67,4 @@ class FileStats:# pylint: disable=[too-many-instance-attributes]
     samples_list: Sequence[str] = field(default_factory=tuple)
     markers_list: Sequence[str] = field(default_factory=tuple)
     errors: Sequence[LineError] = field(default_factory=tuple)
+    description: str = ""