From 6ca397279e09d968679930f429d6341798be990e Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Mon, 22 Jan 2024 09:27:58 +0300
Subject: scripts: Pass in logger to get detailed updates.
---
qc_app/templates/rqtl2/rqtl2-job-status.html | 2 --
qc_app/upload/rqtl2.py | 4 +++-
scripts/process_rqtl2_bundle.py | 24 ++++++++++++++----------
scripts/rqtl2/install_genotypes.py | 16 ++++++++--------
scripts/rqtl2/install_phenos.py | 24 +++++++++++++-----------
5 files changed, 38 insertions(+), 32 deletions(-)
diff --git a/qc_app/templates/rqtl2/rqtl2-job-status.html b/qc_app/templates/rqtl2/rqtl2-job-status.html
index a8bc640..c6fd1f2 100644
--- a/qc_app/templates/rqtl2/rqtl2-job-status.html
+++ b/qc_app/templates/rqtl2/rqtl2-job-status.html
@@ -12,8 +12,6 @@
R/qtl2 Upload: Job Status
-{{job}}
-
Log
{%for msg in messages%}
diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py
index 8a0d8b4..7805114 100644
--- a/qc_app/upload/rqtl2.py
+++ b/qc_app/upload/rqtl2.py
@@ -592,7 +592,9 @@ def rqtl2_processing_status(jobid: UUID):
if not bool(thejob):
return render_template("rqtl2/no-such-job.html", jobid=jobid)
- logmessages = rconn.lrange(thejob.get("log-messagelist"), 0, -1) or []
+ messagelistname = thejob.get("log-messagelist")
+ logmessages = (rconn.lrange(messagelistname, 0, -1)
+ if bool(messagelistname) else [])
if thejob["status"] == "error":
return render_template(
diff --git a/scripts/process_rqtl2_bundle.py b/scripts/process_rqtl2_bundle.py
index feb5e19..7b114cf 100644
--- a/scripts/process_rqtl2_bundle.py
+++ b/scripts/process_rqtl2_bundle.py
@@ -2,10 +2,10 @@
import sys
import uuid
import json
-import logging
import traceback
from typing import Any
from pathlib import Path
+from logging import Logger, getLogger, StreamHandler
import MySQLdb as mdb
from redis import Redis
@@ -20,11 +20,6 @@ from scripts.redis_logger import setup_redis_logger
from scripts.rqtl2.install_phenos import install_pheno_files
from scripts.rqtl2.install_genotypes import install_genotypes
-stderr_handler = logging.StreamHandler(stream=sys.stderr)
-logger = logging.getLogger("process_rqtl2_bundle")
-logger.setLevel("DEBUG")
-logger.addHandler(stderr_handler)
-
def safe_json_decode(value: str) -> Any:
"""Safely decode the string values into JSON."""
try:
@@ -54,7 +49,10 @@ def percent_completion(geno: float, pheno: float) -> float:
"""Compute the total completion percent."""
return 0.5 * (geno + pheno)
-def process_bundle(dbconn: mdb.Connection, rconn: Redis, jobid: uuid.UUID) -> int:
+def process_bundle(dbconn: mdb.Connection,
+ rconn: Redis,
+ jobid: uuid.UUID,
+ logger: Logger) -> int:
"""Process the R/qtl2 bundle."""
try:
thejob = parse_job(rconn, jobid)
@@ -69,7 +67,8 @@ def process_bundle(dbconn: mdb.Connection, rconn: Redis, jobid: uuid.UUID) -> in
meta["speciesid"],
meta["populationid"],
meta["geno-dataset-id"],
- Path(meta["rqtl2-bundle-file"]))
+ Path(meta["rqtl2-bundle-file"]),
+ logger)
if genoexit != 0:
raise Exception("Processing 'geno' file failed.")
logger.debug(
@@ -83,7 +82,8 @@ def process_bundle(dbconn: mdb.Connection, rconn: Redis, jobid: uuid.UUID) -> in
meta["speciesid"],
meta["platformid"],
meta["probe-dataset-id"],
- Path(meta["rqtl2-bundle-file"]))
+ Path(meta["rqtl2-bundle-file"]),
+ logger)
if phenoexit != 0:
raise Exception("Processing 'pheno' file failed.")
logger.debug(
@@ -109,13 +109,17 @@ if __name__ == "__main__":
check_db(args.databaseuri)
check_redis(args.redisuri)
+ logger = getLogger("process_rqtl2_bundle")
+ logger.addHandler(StreamHandler(stream=sys.stderr))
+ logger.setLevel("DEBUG")
+
jobid = args.jobid
with (database_connection(args.databaseuri) as dbconn,
Redis.from_url(args.redisuri, decode_responses=True) as rconn):
logger.addHandler(setup_redis_logger(
rconn, jobid, f"{str(jobid)}:log-messages", args.redisexpiry))
- exitcode = process_bundle(dbconn, rconn, args.jobid)
+ exitcode = process_bundle(dbconn, rconn, args.jobid, logger)
rconn.hset(str(args.jobid), "percent", "100")
return exitcode
diff --git a/scripts/rqtl2/install_genotypes.py b/scripts/rqtl2/install_genotypes.py
index 733ccf9..bfe17b8 100644
--- a/scripts/rqtl2/install_genotypes.py
+++ b/scripts/rqtl2/install_genotypes.py
@@ -1,11 +1,11 @@
"""Load genotypes from R/qtl2 bundle into the database."""
import sys
-import logging
import traceback
from pathlib import Path
from zipfile import ZipFile
from functools import reduce
from typing import Iterator, Optional
+from logging import Logger, getLogger, StreamHandler
import MySQLdb as mdb
from MySQLdb.cursors import DictCursor
@@ -18,10 +18,6 @@ from scripts.rqtl2.entry import build_main
from scripts.rqtl2.cli_parser import add_common_arguments
from scripts.cli_parser import init_cli_parser, add_global_data_arguments
-stderr_handler = logging.StreamHandler(stream=sys.stderr)
-logger = logging.getLogger("install_genotypes")
-logger.addHandler(stderr_handler)
-
def insert_markers(dbconn: mdb.Connection,
speciesid: int,
markers: tuple[str, ...],
@@ -158,11 +154,13 @@ def cross_reference_genotypes(dbconn: mdb.Connection,
} for row in dataids))
return cursor.rowcount
-def install_genotypes(dbconn: mdb.Connection,
+def install_genotypes(#pylint: disable=[too-many-arguments, too-many-locals]
+ dbconn: mdb.Connection,
speciesid: int,
populationid: int,
datasetid: int,
- rqtl2bundle: Path) -> int:
+ rqtl2bundle: Path,
+ logger: Logger = getLogger()) -> int:
"""Load any existing genotypes into the database."""
count = 0
with ZipFile(str(rqtl2bundle.absolute()), "r") as zfile:
@@ -223,6 +221,8 @@ if __name__ == "__main__":
return parser.parse_args()
+ thelogger = getLogger("install_genotypes")
+ thelogger.addHandler(StreamHandler(stream=sys.stderr))
main = build_main(
cli_args(),
lambda dbconn, args: install_genotypes(dbconn,
@@ -230,6 +230,6 @@ if __name__ == "__main__":
args.populationid,
args.datasetid,
args.rqtl2bundle),
- logger,
+ thelogger,
"INFO")
sys.exit(main())
diff --git a/scripts/rqtl2/install_phenos.py b/scripts/rqtl2/install_phenos.py
index 21b5f00..a6cdacc 100644
--- a/scripts/rqtl2/install_phenos.py
+++ b/scripts/rqtl2/install_phenos.py
@@ -1,10 +1,10 @@
"""Load pheno from R/qtl2 bundle into the database."""
import sys
-import logging
import traceback
from pathlib import Path
from zipfile import ZipFile
from functools import reduce
+from logging import Logger, getLogger, StreamHandler
import MySQLdb as mdb
from MySQLdb.cursors import DictCursor
@@ -16,9 +16,6 @@ from scripts.cli_parser import init_cli_parser, add_global_data_arguments
from r_qtl import r_qtl2 as rqtl2
from functional_tools import take
-stderr_handler = logging.StreamHandler(stream=sys.stderr)
-logger = logging.getLogger("install_phenos")
-logger.addHandler(stderr_handler)
def insert_probesets(dbconn: mdb.Connection,
platformid: int,
@@ -95,11 +92,13 @@ def cross_reference_probeset_data(dbconn: mdb.Connection,
} for row in dataids))
return cursor.rowcount
-def install_pheno_files(dbconn: mdb.Connection,
- speciesid: int,
- platformid: int,
- datasetid: int,
- rqtl2bundle: Path) -> int:
+def install_pheno_files(#pylint: disable=[too-many-arguments, too-many-locals]
+ dbconn: mdb.Connection,
+ speciesid: int,
+ platformid: int,
+ datasetid: int,
+ rqtl2bundle: Path,
+ logger: Logger = getLogger()) -> int:
"""Load data in `pheno` files and other related files into the database."""
with ZipFile(str(rqtl2bundle), "r") as zfile:
try:
@@ -155,13 +154,16 @@ if __name__ == "__main__":
return parser.parse_args()
+ thelogger = getLogger("install_phenos")
+ thelogger.addHandler(StreamHandler(stream=sys.stderr))
main = build_main(
cli_args(),
lambda dbconn, args: install_pheno_files(dbconn,
args.speciesid,
args.platformid,
args.datasetid,
- args.rqtl2bundle),
- logger,
+ args.rqtl2bundle,
+ thelogger),
+ thelogger,
"DEBUG")
sys.exit(main())
--
cgit v1.2.3