diff options
| -rw-r--r-- | scripts/rqtl2/phenotypes_qc.py | 119 |
1 files changed, 71 insertions, 48 deletions
diff --git a/scripts/rqtl2/phenotypes_qc.py b/scripts/rqtl2/phenotypes_qc.py index 4c02578..d8bf3c2 100644 --- a/scripts/rqtl2/phenotypes_qc.py +++ b/scripts/rqtl2/phenotypes_qc.py @@ -150,7 +150,7 @@ def redis_logger( def qc_phenocovar_file( - filename: Path, + filepath: Path, redisuri, fqkey: str, separator: str, @@ -159,20 +159,20 @@ def qc_phenocovar_file( with redis_logger( redisuri, f"{__MODULE__}.qc_phenocovar_file", - filename.name, + filepath.name, fqkey) as logger: - logger.info("Running QC on file: %s", filename.name) - _csvfile = rqtl2.read_csv_file(filename, separator, comment_char) + logger.info("Running QC on file: %s", filepath.name) + _csvfile = rqtl2.read_csv_file(filepath, separator, comment_char) _headings = tuple(heading.lower() for heading in next(_csvfile)) _errors: tuple[InvalidValue, ...] = tuple() for heading in ("description", "units"): if heading not in _headings: _errors = (InvalidValue( - filename.name, + filepath.name, "header row", "-", "-", - (f"File {filename.name} is missing the {heading} heading " + (f"File {filepath.name} is missing the {heading} heading " "in the header line.")),) def collect_errors(errors_and_linecount, line): @@ -180,16 +180,16 @@ def qc_phenocovar_file( logger.info("Testing record '%s'", line[0]) if len(line) != len(_headings): _errs = _errs + (InvalidValue( - filename.name, + filepath.name, line[0], "-", "-", - (f"Record {_lc} in file {filename.name} has a different " + (f"Record {_lc} in file {filepath.name} has a different " "number of columns than the number of headings")),) _line = dict(zip(_headings, line)) if not bool(_line["description"]): _errs = _errs + ( - InvalidValue(filename.name, + InvalidValue(filepath.name, _line[_headings[0]], "description", _line["description"], @@ -198,7 +198,7 @@ def qc_phenocovar_file( return _errs, _lc+1 return { - filename.name: dict(zip( + filepath.name: dict(zip( ("errors", "linecount"), reduce(collect_errors, _csvfile, (_errors, 1)))) } @@ -242,6 +242,8 @@ def integer_error( def qc_pheno_file(# pylint: disable=[too-many-arguments] filepath: Path, + redisuri: str, + fqkey: str, samples: tuple[str, ...], phenonames: tuple[str, ...], separator: str, @@ -250,49 +252,55 @@ def qc_pheno_file(# pylint: disable=[too-many-arguments] error_fn: Callable = decimal_points_error ): """Run QC/QA on a `pheno` file.""" - _csvfile = rqtl2.read_csv_file(filepath, separator, comment_char) - _headings: tuple[str, ...] = tuple( - heading.lower() for heading in next(_csvfile)) - _errors: tuple[InvalidValue, ...] = tuple() - - _absent = tuple(pheno for pheno in _headings[1:] if pheno not in phenonames) - if len(_absent) > 0: - _errors = _errors + (InvalidValue( - filepath.name, - "header row", - "-", - ", ".join(_absent), - (f"The phenotype names ({', '.join(samples)}) do not exist in any " - "of the provided phenocovar files.")),) - - def collect_errors(errors_and_linecount, line): - _errs, _lc = errors_and_linecount - if line[0] not in samples: - _errs = _errs + (InvalidValue( + with redis_logger( + redisuri, + f"{__MODULE__}.qc_pheno_file", filepath.name, - line[0], - _headings[0], - line[0], - (f"The sample named '{line[0]}' does not exist in the database. " - "You will need to upload that first.")),) - - for field, value in zip(_headings[1:], line[1:]): - if value in na_strings: - continue - _err = error_fn( + fqkey) as logger: + logger.info("Running QC on file: %s", filepath.name) + _csvfile = rqtl2.read_csv_file(filepath, separator, comment_char) + _headings: tuple[str, ...] = tuple( + heading.lower() for heading in next(_csvfile)) + _errors: tuple[InvalidValue, ...] = tuple() + + _absent = tuple(pheno for pheno in _headings[1:] if pheno not in phenonames) + if len(_absent) > 0: + _errors = _errors + (InvalidValue( filepath.name, + "header row", + "-", + ", ".join(_absent), + (f"The phenotype names ({', '.join(samples)}) do not exist in any " + "of the provided phenocovar files.")),) + + def collect_errors(errors_and_linecount, line): + _errs, _lc = errors_and_linecount + if line[0] not in samples: + _errs = _errs + (InvalidValue( + filepath.name, + line[0], + _headings[0], line[0], - field, - value) - _errs = _errs + ((_err,) if bool(_err) else tuple()) + (f"The sample named '{line[0]}' does not exist in the database. " + "You will need to upload that first.")),) + + for field, value in zip(_headings[1:], line[1:]): + if value in na_strings: + continue + _err = error_fn( + filepath.name, + line[0], + field, + value) + _errs = _errs + ((_err,) if bool(_err) else tuple()) - return _errs, _lc+1 + return _errs, _lc+1 - return { - filepath.name: dict(zip( - ("errors", "linecount"), - reduce(collect_errors, _csvfile, (_errors, 1)))) - } + return { + filepath.name: dict(zip( + ("errors", "linecount"), + reduce(collect_errors, _csvfile, (_errors, 1)))) + } def phenotype_names(filepath: Path, @@ -384,6 +392,11 @@ def run_qc(# pylint: disable=[too-many-locals] logger.debug("Check for errors in 'pheno' file(s).") _pheno_qc_res = merge_dicts(*pool.starmap(qc_pheno_file, tuple(( extractiondir.joinpath(_file), + args.redisuri, + chain( + "pheno", + fullyqualifiedkey(args.jobid), + fullyqualifiedkey(args.redisprefix)), samples, phenonames, cdata["sep"], @@ -398,6 +411,11 @@ def run_qc(# pylint: disable=[too-many-locals] logger.debug("Check for errors in 'phenose' file(s).") _phenose_qc_res = merge_dicts(*pool.starmap(qc_pheno_file, tuple(( extractiondir.joinpath(_file), + args.redisuri, + chain( + "phenose", + fullyqualifiedkey(args.jobid), + fullyqualifiedkey(args.redisprefix)), samples, phenonames, cdata["sep"], @@ -409,6 +427,11 @@ def run_qc(# pylint: disable=[too-many-locals] logger.debug("Check for errors in 'phenonum' file(s).") _phenonum_qc_res = merge_dicts(*pool.starmap(qc_pheno_file, tuple(( extractiondir.joinpath(_file), + args.redisuri, + chain( + "phenonum", + fullyqualifiedkey(args.jobid), + fullyqualifiedkey(args.redisprefix)), samples, phenonames, cdata["sep"], |