diff options
| author | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-02 10:09:31 -0500 |
|---|---|---|
| committer | Frederick Muriuki Muriithi <fredmanglis@gmail.com> | 2026-10-02 10:09:31 -0500 |
| commit | 9652c118028ea30c423c3c08deb031c5fe2df76e (patch) | |
| tree | 44a3d79d0c7711edc4d32f15af19cea69925b65a /scripts | |
| parent | 49260fefb471e3c005a1c900b111a6df1cd53fdb (diff) | |
| download | gn-uploader-9652c118028ea30c423c3c08deb031c5fe2df76e.tar.gz | |
Update calls to CSV processing functions.
- Update the imports - Pass in function rather than comment-char value
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/load_phenotypes_to_db.py | 31 | ||||
| -rw-r--r-- | scripts/rqtl2/phenotypes_qc.py | 12 |
2 files changed, 28 insertions, 15 deletions
diff --git a/scripts/load_phenotypes_to_db.py b/scripts/load_phenotypes_to_db.py index 72cd641..f906e07 100644 --- a/scripts/load_phenotypes_to_db.py +++ b/scripts/load_phenotypes_to_db.py @@ -19,6 +19,7 @@ from gn_libs import jobs, mysqldb, sqlite3, monadic_requests as mrequests from r_qtl import r_qtl2 as rqtl2 from uploader.species.models import species_by_id from uploader.population.models import population_by_species_and_id +from uploader.csvprocessing import read_csv_file, read_csv_file_headers from uploader.samples.models import samples_by_species_and_population from uploader.phenotypes.models import ( dataset_by_id, @@ -53,6 +54,9 @@ def save_phenotypes( if len(phenofiles) <= 0: return tuple() + def _is_skippable_line_(line): + return line.startswith(control_data["comment.char"]) + if control_data["phenocovar_transposed"]: logger.info("Undoing transposition of the files rows and columns.") phenofiles = tuple( @@ -62,10 +66,10 @@ def save_phenotypes( build_line_joiner(control_data)) for _file in phenofiles) - _headers = rqtl2.read_csv_file_headers(phenofiles[0], - control_data["phenocovar_transposed"], - control_data["sep"], - control_data["comment.char"]) + _headers = read_csv_file_headers(phenofiles[0], + control_data["phenocovar_transposed"], + control_data["sep"], + _is_skippable_line_) return create_new_phenotypes( conn, population_id, @@ -73,9 +77,9 @@ def save_phenotypes( (dict(zip(_headers, __replace_na_strings__(line, control_data["na.strings"]))) for filecontent - in (rqtl2.read_csv_file(path, - separator=control_data["sep"], - comment_char=control_data["comment.char"]) + in (read_csv_file(path, + separator=control_data["sep"], + is_skippable_line=_is_skippable_line_) for path in phenofiles) for idx, line in enumerate(filecontent) if idx != 0)) @@ -105,15 +109,18 @@ def __build_dataitems__( dataidmap, pheno_name2id ): - _headers = rqtl2.read_csv_file_headers( + def _is_skippable_line_(line): + return line.startswith(control_data["comment.char"]) + + _headers = read_csv_file_headers( phenofiles[0], False, # Any transposed files have been un-transposed by this point control_data["sep"], - control_data["comment.char"]) + _is_skippable_line_) _filescontents = ( - rqtl2.read_csv_file(path, - separator=control_data["sep"], - comment_char=control_data["comment.char"]) + read_csv_file(path, + separator=control_data["sep"], + is_skippable_line=_is_skippable_line_) for path in phenofiles) _linescontents = ( __row_to_dataitems__( diff --git a/scripts/rqtl2/phenotypes_qc.py b/scripts/rqtl2/phenotypes_qc.py index e65bbcc..943cd68 100644 --- a/scripts/rqtl2/phenotypes_qc.py +++ b/scripts/rqtl2/phenotypes_qc.py @@ -26,6 +26,7 @@ from functional_tools import chain from quality_control.checks import decimal_places_pattern +from uploader.csvprocessing import read_csv_file from uploader.files import sha256_digest_over_file from uploader.samples.models import samples_by_species_and_population @@ -185,7 +186,8 @@ def qc_phenocovar_file( f"{fqkey}:logs") as _logger, Redis.from_url(redisuri, decode_responses=True) as rconn): print("Running QC on file: ", filepath.name) - _csvfile = rqtl2.read_csv_file(filepath, separator, comment_char) + _csvfile = read_csv_file( + filepath, separator, lambda line: line.startswith(comment_char)) _headings = tuple(heading.lower() for heading in next(_csvfile)) _errors: tuple[InvalidValue, ...] = tuple() save_error = partial( @@ -296,7 +298,8 @@ def qc_pheno_file(# pylint: disable=[too-many-locals, too-many-arguments, too-ma print("Running QC on file: ", filepath.name) save_error = partial( push_error, rconn, file_fqkey(fqkey, "errors", filepath)) - _csvfile = rqtl2.read_csv_file(filepath, separator, comment_char) + _csvfile = read_csv_file( + filepath, separator, lambda line: line.startswith(comment_char)) _headings: tuple[str, ...] = tuple( # select lowercase for comparison purposes heading.lower() for heading in next(_csvfile)) @@ -362,7 +365,10 @@ def phenotype_names(filepath: Path, comment_char: str) -> tuple[str, ...]: """Read phenotype names from `phenocovar` file.""" return reduce(lambda tpl, line: tpl + (line[0],),#type: ignore[arg-type, return-value] - rqtl2.read_csv_file(filepath, separator, comment_char), + read_csv_file( + filepath, + separator, + lambda line: line.startswith(comment_char)), tuple())[1:] def fullyqualifiedkey( |
