diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/compute_phenotype_means.py | 2 | ||||
| -rw-r--r-- | scripts/load_phenotypes_to_db.py | 17 | ||||
| -rw-r--r-- | scripts/qc_on_rqtl2_bundle.py | 9 | ||||
| -rw-r--r-- | scripts/rqtl2/install_genotypes.py | 6 | ||||
| -rw-r--r-- | scripts/rqtl2/install_phenos.py | 7 | ||||
| -rw-r--r-- | scripts/rqtl2/phenotypes_qc.py | 3 | ||||
| -rw-r--r-- | scripts/run_qtlreaper.py | 27 |
7 files changed, 38 insertions, 33 deletions
diff --git a/scripts/compute_phenotype_means.py b/scripts/compute_phenotype_means.py index ef2fabc..6d39ace 100644 --- a/scripts/compute_phenotype_means.py +++ b/scripts/compute_phenotype_means.py @@ -51,7 +51,7 @@ def run(args) -> int: T = TypeVar("T") -def comma_separated_list(val: str, itemstype: T = str) -> tuple[T, ...]: +def comma_separated_list(val: str, itemstype: type = str) -> tuple[T, ...]: """Convert val into a list of items of type 'itemstype'.""" return tuple(itemstype(item.strip()) for item in val.split(",")) diff --git a/scripts/load_phenotypes_to_db.py b/scripts/load_phenotypes_to_db.py index e449b82..e303bb3 100644 --- a/scripts/load_phenotypes_to_db.py +++ b/scripts/load_phenotypes_to_db.py @@ -6,9 +6,9 @@ import time import logging import argparse import datetime -from typing import Any from pathlib import Path from zipfile import ZipFile +from typing import Any, Iterable from urllib.parse import urljoin from functools import reduce, partial @@ -55,7 +55,7 @@ def save_phenotypes( if control_data["phenocovar_transposed"]: logger.info("Undoing transposition of the files rows and columns.") - phenofiles = ( + phenofiles = tuple( rqtl2.transpose_csv_with_rename( _file, build_line_splitter(control_data), @@ -86,7 +86,7 @@ def __row_to_dataitems__( dataidmap: dict, pheno_name2id: dict[str, int], samples: dict -) -> tuple[dict, ...]: +) -> Iterable[dict]: samplename = sample_row["id"] return ({ @@ -134,7 +134,7 @@ def save_numeric_data(# pylint: disable=[too-many-positional-arguments,too-many- conn: mysqldb.Connection, dataidmap: dict, pheno_name2id: dict[str, int], - samples: tuple[dict, ...], + samples: dict, control_data: dict, filesdir: Path, filetype: str, @@ -311,7 +311,9 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments ).either(__handle_error__, __handle_success__) -def load_data(conn: mysqldb.Connection, job: dict) -> int:#pylint: disable=[too-many-locals] +def load_data(# pylint: disable=[too-many-locals] + conn: mysqldb.Connection, job: dict +) -> tuple[dict, dict, dict, tuple[int, ...]]: """Load the data attached in the given job.""" _job_metadata = job["metadata"] # Steps @@ -365,9 +367,8 @@ def load_data(conn: mysqldb.Connection, job: dict) -> int:#pylint: disable=[too- "publication_id": row["publication_id"], "data_id": row["data_id"] },))) - dataidmap, pheno_name2id, _xrefs = reduce(__build_phenos_maps__, - _phenos, - ({},{}, tuple())) + dataidmap, pheno_name2id, _xrefs = reduce(# type: ignore[var-annotated] + __build_phenos_maps__, _phenos, ({},{}, tuple())) # 3. a. Fetch the strain names and IDS: create name->ID map samples = { row["Name"]: row diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py index 0207938..4e6ef00 100644 --- a/scripts/qc_on_rqtl2_bundle.py +++ b/scripts/qc_on_rqtl2_bundle.py @@ -40,7 +40,7 @@ def add_to_errors(rconn: Redis, """Add `errors` to a given list of errors""" errs = tuple(dict(item) for item in set( [dict2tuple(old) for old in - json.loads(rconn.hget(fqjobid, key) or "[]")] + + json.loads(rconn.hget(fqjobid, key) or "[]")] +# type: ignore[arg-type] [dict2tuple({"type": type(error).__name__, **error._asdict()}) for error in errors])) rconn.hset(fqjobid, key, json.dumps(errs)) @@ -83,7 +83,8 @@ def retrieve_errors_with_progress(rconn: Redis,#pylint: disable=[too-many-locals count = 0 checked = 0 cdata = rqtl2.control_data(zfile) - rconn.hset(fqjobid, f"{filetype}-filesize", compute_filesize(zfile, filetype)) + rconn.hset( + fqjobid, f"{filetype}-filesize", str(compute_filesize(zfile, filetype))) def __update_processed__(value): nonlocal checked checked = checked + len(value) @@ -104,7 +105,7 @@ def retrieve_errors_with_progress(rconn: Redis,#pylint: disable=[too-many-locals yield error __update_processed__(value) - rconn.hset(fqjobid, f"{filetype}-linecount", count) + rconn.hset(fqjobid, f"{filetype}-linecount", count)# type: ignore[arg-type] except rqe.MissingFileException: fname = cdata.get(filetype) yield rqfe.MissingFile(filetype, fname, ( @@ -295,7 +296,7 @@ def run_qc(rconn: Redis, return 1 def __fetch_errors__(rkey: str) -> tuple: - return tuple(json.loads(rconn.hget(fqjobid, rkey) or "[]")) + return tuple(json.loads(rconn.hget(fqjobid, rkey) or "[]")) # type: ignore[arg-type] return (1 if any(( bool(__fetch_errors__(key)) diff --git a/scripts/rqtl2/install_genotypes.py b/scripts/rqtl2/install_genotypes.py index 8762655..5e6abb0 100644 --- a/scripts/rqtl2/install_genotypes.py +++ b/scripts/rqtl2/install_genotypes.py @@ -20,7 +20,7 @@ from scripts.rqtl2.entry import build_main from scripts.rqtl2.cli_parser import add_common_arguments from scripts.cli_parser import init_cli_parser, add_global_data_arguments -__MODULE__ = "scripts.rqtl2.install_genotypes" +logger = getLogger(__name__) def insert_markers( dbconn: mdb.Connection, @@ -191,7 +191,7 @@ def install_genotypes(#pylint: disable=[too-many-locals] dbconn: mdb.Connection, fullyqualifiedjobid: str,#pylint: disable=[unused-argument] args: argparse.Namespace, - logger: Logger = getLogger(__name__) + logger: Logger = logger # pylint: disable=[redefined-outer-name] ) -> int: """Load any existing genotypes into the database.""" (speciesid, populationid, datasetid, rqtl2bundle) = ( @@ -257,5 +257,5 @@ if __name__ == "__main__": return parser.parse_args() - main = build_main(cli_args(), install_genotypes, __MODULE__) + main = build_main(cli_args(), install_genotypes, logger) sys.exit(main()) diff --git a/scripts/rqtl2/install_phenos.py b/scripts/rqtl2/install_phenos.py index 9059cd6..11ac8a4 100644 --- a/scripts/rqtl2/install_phenos.py +++ b/scripts/rqtl2/install_phenos.py @@ -19,7 +19,7 @@ from r_qtl import r_qtl2_qc as rqc from functional_tools import take -__MODULE__ = "scripts.rqtl2.install_phenos" +logger = getLogger(__name__) def insert_probesets(dbconn: mdb.Connection, platformid: int, @@ -101,7 +101,8 @@ def install_pheno_files(#pylint: disable=[too-many-locals] dbconn: mdb.Connection, fullyqualifiedjobid: str,#pylint: disable=[unused-argument] args: argparse.Namespace, - logger: Logger = getLogger()) -> int: + logger: Logger = logger # pylint: disable=[redefined-outer-name] +) -> int: """Load data in `pheno` files and other related files into the database.""" (speciesid, platformid, datasetid, rqtl2bundle) = ( args.speciesid, args.platformid, args.datasetid, args.rqtl2bundle) @@ -159,5 +160,5 @@ if __name__ == "__main__": return parser.parse_args() - main = build_main(cli_args(), install_pheno_files, __MODULE__) + main = build_main(cli_args(), install_pheno_files, logger) sys.exit(main()) diff --git a/scripts/rqtl2/phenotypes_qc.py b/scripts/rqtl2/phenotypes_qc.py index 9f11f57..72d6c83 100644 --- a/scripts/rqtl2/phenotypes_qc.py +++ b/scripts/rqtl2/phenotypes_qc.py @@ -376,7 +376,8 @@ def run_qc(# pylint: disable=[too-many-locals] rconn: Redis, dbconn: mdb.Connection, fullyqualifiedjobid: str, - args: Namespace + args: Namespace, + logger: Logger = logger # pylint: disable=[redefined-outer-name] ) -> int: """Run quality control checks on the bundle.""" print("Beginning the quality assurance checks.") diff --git a/scripts/run_qtlreaper.py b/scripts/run_qtlreaper.py index 89cc3ec..7d58402 100644 --- a/scripts/run_qtlreaper.py +++ b/scripts/run_qtlreaper.py @@ -6,8 +6,8 @@ import secrets import logging import subprocess from pathlib import Path -from typing import Union from functools import reduce +from typing import Union, Iterator from argparse import Namespace, ArgumentParser from gn_libs import mysqldb @@ -56,7 +56,7 @@ def reconcile_samples( def generate_qtlreaper_traits_file( outdir: Path, samples: tuple[str, ...], - traits_data: dict[str, Union[int, float]], + traits_data: tuple[dict[str, Union[int, float]], ...], filename_prefix: str = "" ) -> Path: """Generate a file for use with qtlreaper that contains the traits' data.""" @@ -65,7 +65,7 @@ def generate_qtlreaper_traits_file( _dialect.quoting=0 _traitsfile = outdir.joinpath( - f"{filename_prefix}_{secrets.token_urlsafe(15)}.tsv") + f"{filename_prefix}_{secrets.token_urlsafe(15)}.tsv")#type: ignore[attr-defined] with _traitsfile.open(mode="w", encoding="utf-8") as outptr: writer = csv.DictWriter( outptr, fieldnames=("Trait",) + samples, dialect=_dialect) @@ -79,7 +79,7 @@ def generate_qtlreaper_traits_file( return _traitsfile -def parse_tsv_file(results_file: Path) -> list[dict]: +def parse_tsv_file(results_file: Path) -> Iterator[dict]: """Parse the rust-qtlreaper output into usable python objects.""" with results_file.open("r", encoding="utf-8") as readptr: _dialect = csv.unix_dialect() @@ -96,7 +96,7 @@ def __qtls_by_trait__(qtls, current): } -def save_qtl_values_to_db(conn, qtls: dict): +def save_qtl_values_to_db(conn, qtls: tuple[dict, ...]): """Save computed QTLs to the database.""" with conn.cursor() as cursor: cursor.executemany( @@ -131,11 +131,11 @@ def dispatch(args: Namespace) -> int: ", ".join(_samples_not_in_genofile)) # Fetch traits data: provided list, or all traits in db - _traitsdata = phenotypes_vector_data( + _traitsdata = tuple(phenotypes_vector_data( conn, args.species_id, args.population_id, - xref_ids=tuple(args.xref_ids)).values() + xref_ids=tuple(args.xref_ids)).values()) logger.debug("Successfully got traits data. Generating the QTLReaper's traits file…") _traitsfile = generate_qtlreaper_traits_file( args.working_dir, @@ -145,7 +145,7 @@ def dispatch(args: Namespace) -> int: logger.debug("QTLReaper's Traits file: %s", _traitsfile) _qtlreaper_main_output = args.working_dir.joinpath( - f"main-output-{secrets.token_urlsafe(15)}.tsv") + f"main-output-{secrets.token_urlsafe(15)}.tsv")#type: ignore[attr-defined] logger.debug("Main output filename: %s", _qtlreaper_main_output) with subprocess.Popen( ("qtlreaper", @@ -156,11 +156,12 @@ def dispatch(args: Namespace) -> int: while _qtlreaper.poll() is None: logger.debug("QTLReaper process running…") time.sleep(1) - results = tuple(max(qtls, key=lambda qtl: qtl["LRS"]) - for qtls in - reduce(__qtls_by_trait__, - parse_tsv_file(_qtlreaper_main_output), - {}).values()) + results = tuple(#type: ignore[var-annotated] + max(qtls, key=lambda qtl: qtl["LRS"]) + for qtls in + reduce(__qtls_by_trait__, + parse_tsv_file(_qtlreaper_main_output), + {}).values()) save_qtl_values_to_db(conn, results) logger.debug("Cleaning up temporary files.") _traitsfile.unlink() |