From 6ca397279e09d968679930f429d6341798be990e Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 22 Jan 2024 09:27:58 +0300 Subject: scripts: Pass in logger to get detailed updates. --- scripts/process_rqtl2_bundle.py | 24 ++++++++++++++---------- scripts/rqtl2/install_genotypes.py | 16 ++++++++-------- scripts/rqtl2/install_phenos.py | 24 +++++++++++++----------- 3 files changed, 35 insertions(+), 29 deletions(-) (limited to 'scripts') diff --git a/scripts/process_rqtl2_bundle.py b/scripts/process_rqtl2_bundle.py index feb5e19..7b114cf 100644 --- a/scripts/process_rqtl2_bundle.py +++ b/scripts/process_rqtl2_bundle.py @@ -2,10 +2,10 @@ import sys import uuid import json -import logging import traceback from typing import Any from pathlib import Path +from logging import Logger, getLogger, StreamHandler import MySQLdb as mdb from redis import Redis @@ -20,11 +20,6 @@ from scripts.redis_logger import setup_redis_logger from scripts.rqtl2.install_phenos import install_pheno_files from scripts.rqtl2.install_genotypes import install_genotypes -stderr_handler = logging.StreamHandler(stream=sys.stderr) -logger = logging.getLogger("process_rqtl2_bundle") -logger.setLevel("DEBUG") -logger.addHandler(stderr_handler) - def safe_json_decode(value: str) -> Any: """Safely decode the string values into JSON.""" try: @@ -54,7 +49,10 @@ def percent_completion(geno: float, pheno: float) -> float: """Compute the total completion percent.""" return 0.5 * (geno + pheno) -def process_bundle(dbconn: mdb.Connection, rconn: Redis, jobid: uuid.UUID) -> int: +def process_bundle(dbconn: mdb.Connection, + rconn: Redis, + jobid: uuid.UUID, + logger: Logger) -> int: """Process the R/qtl2 bundle.""" try: thejob = parse_job(rconn, jobid) @@ -69,7 +67,8 @@ def process_bundle(dbconn: mdb.Connection, rconn: Redis, jobid: uuid.UUID) -> in meta["speciesid"], meta["populationid"], meta["geno-dataset-id"], - Path(meta["rqtl2-bundle-file"])) + Path(meta["rqtl2-bundle-file"]), + logger) if genoexit != 0: raise Exception("Processing 'geno' file failed.") logger.debug( @@ -83,7 +82,8 @@ def process_bundle(dbconn: mdb.Connection, rconn: Redis, jobid: uuid.UUID) -> in meta["speciesid"], meta["platformid"], meta["probe-dataset-id"], - Path(meta["rqtl2-bundle-file"])) + Path(meta["rqtl2-bundle-file"]), + logger) if phenoexit != 0: raise Exception("Processing 'pheno' file failed.") logger.debug( @@ -109,13 +109,17 @@ if __name__ == "__main__": check_db(args.databaseuri) check_redis(args.redisuri) + logger = getLogger("process_rqtl2_bundle") + logger.addHandler(StreamHandler(stream=sys.stderr)) + logger.setLevel("DEBUG") + jobid = args.jobid with (database_connection(args.databaseuri) as dbconn, Redis.from_url(args.redisuri, decode_responses=True) as rconn): logger.addHandler(setup_redis_logger( rconn, jobid, f"{str(jobid)}:log-messages", args.redisexpiry)) - exitcode = process_bundle(dbconn, rconn, args.jobid) + exitcode = process_bundle(dbconn, rconn, args.jobid, logger) rconn.hset(str(args.jobid), "percent", "100") return exitcode diff --git a/scripts/rqtl2/install_genotypes.py b/scripts/rqtl2/install_genotypes.py index 733ccf9..bfe17b8 100644 --- a/scripts/rqtl2/install_genotypes.py +++ b/scripts/rqtl2/install_genotypes.py @@ -1,11 +1,11 @@ """Load genotypes from R/qtl2 bundle into the database.""" import sys -import logging import traceback from pathlib import Path from zipfile import ZipFile from functools import reduce from typing import Iterator, Optional +from logging import Logger, getLogger, StreamHandler import MySQLdb as mdb from MySQLdb.cursors import DictCursor @@ -18,10 +18,6 @@ from scripts.rqtl2.entry import build_main from scripts.rqtl2.cli_parser import add_common_arguments from scripts.cli_parser import init_cli_parser, add_global_data_arguments -stderr_handler = logging.StreamHandler(stream=sys.stderr) -logger = logging.getLogger("install_genotypes") -logger.addHandler(stderr_handler) - def insert_markers(dbconn: mdb.Connection, speciesid: int, markers: tuple[str, ...], @@ -158,11 +154,13 @@ def cross_reference_genotypes(dbconn: mdb.Connection, } for row in dataids)) return cursor.rowcount -def install_genotypes(dbconn: mdb.Connection, +def install_genotypes(#pylint: disable=[too-many-arguments, too-many-locals] + dbconn: mdb.Connection, speciesid: int, populationid: int, datasetid: int, - rqtl2bundle: Path) -> int: + rqtl2bundle: Path, + logger: Logger = getLogger()) -> int: """Load any existing genotypes into the database.""" count = 0 with ZipFile(str(rqtl2bundle.absolute()), "r") as zfile: @@ -223,6 +221,8 @@ if __name__ == "__main__": return parser.parse_args() + thelogger = getLogger("install_genotypes") + thelogger.addHandler(StreamHandler(stream=sys.stderr)) main = build_main( cli_args(), lambda dbconn, args: install_genotypes(dbconn, @@ -230,6 +230,6 @@ if __name__ == "__main__": args.populationid, args.datasetid, args.rqtl2bundle), - logger, + thelogger, "INFO") sys.exit(main()) diff --git a/scripts/rqtl2/install_phenos.py b/scripts/rqtl2/install_phenos.py index 21b5f00..a6cdacc 100644 --- a/scripts/rqtl2/install_phenos.py +++ b/scripts/rqtl2/install_phenos.py @@ -1,10 +1,10 @@ """Load pheno from R/qtl2 bundle into the database.""" import sys -import logging import traceback from pathlib import Path from zipfile import ZipFile from functools import reduce +from logging import Logger, getLogger, StreamHandler import MySQLdb as mdb from MySQLdb.cursors import DictCursor @@ -16,9 +16,6 @@ from scripts.cli_parser import init_cli_parser, add_global_data_arguments from r_qtl import r_qtl2 as rqtl2 from functional_tools import take -stderr_handler = logging.StreamHandler(stream=sys.stderr) -logger = logging.getLogger("install_phenos") -logger.addHandler(stderr_handler) def insert_probesets(dbconn: mdb.Connection, platformid: int, @@ -95,11 +92,13 @@ def cross_reference_probeset_data(dbconn: mdb.Connection, } for row in dataids)) return cursor.rowcount -def install_pheno_files(dbconn: mdb.Connection, - speciesid: int, - platformid: int, - datasetid: int, - rqtl2bundle: Path) -> int: +def install_pheno_files(#pylint: disable=[too-many-arguments, too-many-locals] + dbconn: mdb.Connection, + speciesid: int, + platformid: int, + datasetid: int, + rqtl2bundle: Path, + logger: Logger = getLogger()) -> int: """Load data in `pheno` files and other related files into the database.""" with ZipFile(str(rqtl2bundle), "r") as zfile: try: @@ -155,13 +154,16 @@ if __name__ == "__main__": return parser.parse_args() + thelogger = getLogger("install_phenos") + thelogger.addHandler(StreamHandler(stream=sys.stderr)) main = build_main( cli_args(), lambda dbconn, args: install_pheno_files(dbconn, args.speciesid, args.platformid, args.datasetid, - args.rqtl2bundle), - logger, + args.rqtl2bundle, + thelogger), + thelogger, "DEBUG") sys.exit(main()) -- cgit v1.2.3