diff options
| author | pi coding agent <pi@localhost> | 2026-11-04 19:10:02 +0000 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-11-04 13:28:11 -0600 |
| commit | fb9e85d123ca7d0853c4405d021e80936233fc17 (patch) | |
| tree | b2001ec158acce25ca0508dc1c2a657b21e8c5b2 /uploader/scripts | |
| parent | 718777873f56170cabf06801076788e66ff60d45 (diff) | |
| download | gn-uploader-fb9e85d123ca7d0853c4405d021e80936233fc17.tar.gz | |
Collect comment lines into the file description
Add a `description` field to `FileStats`, built by cleaning comment lines (strip the configured comment character and surrounding whitespace) and joining them with newlines. Detect comments using the configured comment character rather than a hard-coded '#', and preserve comment order when merging chunks.
Diffstat (limited to 'uploader/scripts')
| -rw-r--r-- | uploader/scripts/genotypes/preprocess_csv_collect_info.py | 14 | ||||
| -rw-r--r-- | uploader/scripts/genotypes/types.py | 1 |
2 files changed, 11 insertions, 4 deletions
diff --git a/uploader/scripts/genotypes/preprocess_csv_collect_info.py b/uploader/scripts/genotypes/preprocess_csv_collect_info.py index 870694b..e1978bd 100644 --- a/uploader/scripts/genotypes/preprocess_csv_collect_info.py +++ b/uploader/scripts/genotypes/preprocess_csv_collect_info.py @@ -203,6 +203,7 @@ class _ChunkStats:# pylint: disable=[too-many-instance-attributes] markers: tuple = field(default_factory=tuple) markers_seen: bool = False errors: tuple = field(default_factory=tuple) + comments: tuple = field(default_factory=tuple) def _accumulate( @@ -212,6 +213,7 @@ def _accumulate( ) -> _ChunkStats: """Accumulate statistics from the provided lines.""" is_skippable_line = _make_is_skippable_line(fileconfigs, headers) + is_comment_line = make_comment_line_checker(fileconfigs["comment.char"]) line_splitter = _make_line_splitter(fileconfigs["separator"]) stats = _ChunkStats() for line in (ln.strip() for ln in lines): @@ -237,13 +239,15 @@ def _accumulate( stats.markers = stats.markers + (str(line_dets.marker),) # TOD0: do more processing continue # end of line processing - if line.startswith("@"): #handle special geno formats + if is_dot_geno_line(line): #handle special geno formats stats.dot_geno_lines += 1 stats.dot_geno_fields = { **stats.dot_geno_fields, **parse_dot_geno_line(line)} - if line.startswith("#"): #handle special geno formats + if is_comment_line(line): #handle special geno formats stats.comment_lines += 1 + stats.comments = stats.comments + ( + line[len(fileconfigs["comment.char"]):].strip(),) if line == "": # Blank line stats.blank_lines += 1 return stats @@ -276,7 +280,8 @@ def _merge_chunk_stats( geno_symbols=left.geno_symbols | right.geno_symbols, markers=markers, markers_seen=left.markers_seen or right.markers_seen, - errors=left.errors + right.errors) + errors=left.errors + right.errors, + comments=left.comments + right.comments) def _merge_all_chunk_stats( @@ -351,7 +356,8 @@ def _as_file_stats( geno_symbols=tuple(sorted(stats.geno_symbols)), samples_list=tuple(samples), markers_list=stats.markers, - errors=tuple(header_errors) + tuple(stats.errors)) + errors=tuple(header_errors) + tuple(stats.errors), + description="\n".join(stats.comments)) def compute_statistics(filepath: Path, fileconfigs: dict) -> FileStats: diff --git a/uploader/scripts/genotypes/types.py b/uploader/scripts/genotypes/types.py index d765fe5..4403f3d 100644 --- a/uploader/scripts/genotypes/types.py +++ b/uploader/scripts/genotypes/types.py @@ -67,3 +67,4 @@ class FileStats:# pylint: disable=[too-many-instance-attributes] samples_list: Sequence[str] = field(default_factory=tuple) markers_list: Sequence[str] = field(default_factory=tuple) errors: Sequence[LineError] = field(default_factory=tuple) + description: str = "" |
