about summary refs log tree commit diff
path: root/scripts
diff options
context:
space:
mode:
authorFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-10-02 10:09:31 -0500
committerFrederick Muriuki Muriithi <fredmanglis@gmail.com>2026-10-02 10:09:31 -0500
commit9652c118028ea30c423c3c08deb031c5fe2df76e (patch)
tree44a3d79d0c7711edc4d32f15af19cea69925b65a /scripts
parent49260fefb471e3c005a1c900b111a6df1cd53fdb (diff)
downloadgn-uploader-9652c118028ea30c423c3c08deb031c5fe2df76e.tar.gz
Update calls to CSV processing functions.
- Update the imports
- Pass in function rather than comment-char value
Diffstat (limited to 'scripts')
-rw-r--r--scripts/load_phenotypes_to_db.py31
-rw-r--r--scripts/rqtl2/phenotypes_qc.py12
2 files changed, 28 insertions, 15 deletions
diff --git a/scripts/load_phenotypes_to_db.py b/scripts/load_phenotypes_to_db.py
index 72cd641..f906e07 100644
--- a/scripts/load_phenotypes_to_db.py
+++ b/scripts/load_phenotypes_to_db.py
@@ -19,6 +19,7 @@ from gn_libs import jobs, mysqldb, sqlite3, monadic_requests as mrequests
 from r_qtl import r_qtl2 as rqtl2
 from uploader.species.models import species_by_id
 from uploader.population.models import population_by_species_and_id
+from uploader.csvprocessing import read_csv_file, read_csv_file_headers
 from uploader.samples.models import samples_by_species_and_population
 from uploader.phenotypes.models import (
     dataset_by_id,
@@ -53,6 +54,9 @@ def save_phenotypes(
     if len(phenofiles) <= 0:
         return tuple()
 
+    def _is_skippable_line_(line):
+        return line.startswith(control_data["comment.char"])
+
     if control_data["phenocovar_transposed"]:
         logger.info("Undoing transposition of the files rows and columns.")
         phenofiles = tuple(
@@ -62,10 +66,10 @@ def save_phenotypes(
                 build_line_joiner(control_data))
             for _file in phenofiles)
 
-    _headers = rqtl2.read_csv_file_headers(phenofiles[0],
-                                           control_data["phenocovar_transposed"],
-                                           control_data["sep"],
-                                           control_data["comment.char"])
+    _headers = read_csv_file_headers(phenofiles[0],
+                                     control_data["phenocovar_transposed"],
+                                     control_data["sep"],
+                                     _is_skippable_line_)
     return create_new_phenotypes(
         conn,
         population_id,
@@ -73,9 +77,9 @@ def save_phenotypes(
         (dict(zip(_headers,
                   __replace_na_strings__(line, control_data["na.strings"])))
          for filecontent
-         in (rqtl2.read_csv_file(path,
-                                 separator=control_data["sep"],
-                                 comment_char=control_data["comment.char"])
+         in (read_csv_file(path,
+                           separator=control_data["sep"],
+                           is_skippable_line=_is_skippable_line_)
              for path in phenofiles)
          for idx, line in enumerate(filecontent)
          if idx != 0))
@@ -105,15 +109,18 @@ def __build_dataitems__(
         dataidmap,
         pheno_name2id
 ):
-    _headers = rqtl2.read_csv_file_headers(
+    def _is_skippable_line_(line):
+        return line.startswith(control_data["comment.char"])
+
+    _headers = read_csv_file_headers(
         phenofiles[0],
         False, # Any transposed files have been un-transposed by this point
         control_data["sep"],
-        control_data["comment.char"])
+        _is_skippable_line_)
     _filescontents = (
-        rqtl2.read_csv_file(path,
-                            separator=control_data["sep"],
-                            comment_char=control_data["comment.char"])
+        read_csv_file(path,
+                      separator=control_data["sep"],
+                      is_skippable_line=_is_skippable_line_)
         for path in phenofiles)
     _linescontents = (
         __row_to_dataitems__(
diff --git a/scripts/rqtl2/phenotypes_qc.py b/scripts/rqtl2/phenotypes_qc.py
index e65bbcc..943cd68 100644
--- a/scripts/rqtl2/phenotypes_qc.py
+++ b/scripts/rqtl2/phenotypes_qc.py
@@ -26,6 +26,7 @@ from functional_tools import chain
 
 from quality_control.checks import decimal_places_pattern
 
+from uploader.csvprocessing import read_csv_file
 from uploader.files import sha256_digest_over_file
 from uploader.samples.models import samples_by_species_and_population
 
@@ -185,7 +186,8 @@ def qc_phenocovar_file(
             f"{fqkey}:logs") as _logger,
           Redis.from_url(redisuri, decode_responses=True) as rconn):
         print("Running QC on file: ", filepath.name)
-        _csvfile = rqtl2.read_csv_file(filepath, separator, comment_char)
+        _csvfile = read_csv_file(
+            filepath, separator, lambda line: line.startswith(comment_char))
         _headings = tuple(heading.lower() for heading in next(_csvfile))
         _errors: tuple[InvalidValue, ...] = tuple()
         save_error = partial(
@@ -296,7 +298,8 @@ def qc_pheno_file(# pylint: disable=[too-many-locals, too-many-arguments, too-ma
         print("Running QC on file: ", filepath.name)
         save_error = partial(
             push_error, rconn, file_fqkey(fqkey, "errors", filepath))
-        _csvfile = rqtl2.read_csv_file(filepath, separator, comment_char)
+        _csvfile = read_csv_file(
+            filepath, separator, lambda line: line.startswith(comment_char))
         _headings: tuple[str, ...] = tuple(
             # select lowercase for comparison purposes
             heading.lower() for heading in next(_csvfile))
@@ -362,7 +365,10 @@ def phenotype_names(filepath: Path,
                     comment_char: str) -> tuple[str, ...]:
     """Read phenotype names from `phenocovar` file."""
     return reduce(lambda tpl, line: tpl + (line[0],),#type: ignore[arg-type, return-value]
-                  rqtl2.read_csv_file(filepath, separator, comment_char),
+                  read_csv_file(
+                      filepath,
+                      separator,
+                      lambda line: line.startswith(comment_char)),
                   tuple())[1:]
 
 def fullyqualifiedkey(