scripts: Pass in logger to get detailed updates.

5 files changed, 38 insertions, 32 deletions

diff --git a/qc_app/templates/rqtl2/rqtl2-job-status.html b/qc_app/templates/rqtl2/rqtl2-job-status.html
index a8bc640..c6fd1f2 100644
--- a/qc_app/templates/rqtl2/rqtl2-job-status.html
+++ b/qc_app/templates/rqtl2/rqtl2-job-status.html
@@ -12,8 +12,6 @@
 
 <h2>R/qtl2 Upload: Job Status</h2>
 
-{{job}}
-
 <h4>Log</h4>
 <div class="cli-output">
   {%for msg in messages%}
diff --git a/qc_app/upload/rqtl2.py b/qc_app/upload/rqtl2.py
index 8a0d8b4..7805114 100644
--- a/qc_app/upload/rqtl2.py
+++ b/qc_app/upload/rqtl2.py
@@ -592,7 +592,9 @@ def rqtl2_processing_status(jobid: UUID):
         if not bool(thejob):
             return render_template("rqtl2/no-such-job.html", jobid=jobid)
 
-        logmessages = rconn.lrange(thejob.get("log-messagelist"), 0, -1) or []
+        messagelistname = thejob.get("log-messagelist")
+        logmessages = (rconn.lrange(messagelistname, 0, -1)
+                       if bool(messagelistname) else [])
 
         if thejob["status"] == "error":
             return render_template(
diff --git a/scripts/process_rqtl2_bundle.py b/scripts/process_rqtl2_bundle.py
index feb5e19..7b114cf 100644
--- a/scripts/process_rqtl2_bundle.py
+++ b/scripts/process_rqtl2_bundle.py
@@ -2,10 +2,10 @@
 import sys
 import uuid
 import json
-import logging
 import traceback
 from typing import Any
 from pathlib import Path
+from logging import Logger, getLogger, StreamHandler
 
 import MySQLdb as mdb
 from redis import Redis
@@ -20,11 +20,6 @@ from scripts.redis_logger import setup_redis_logger
 from scripts.rqtl2.install_phenos import install_pheno_files
 from scripts.rqtl2.install_genotypes import install_genotypes
 
-stderr_handler = logging.StreamHandler(stream=sys.stderr)
-logger = logging.getLogger("process_rqtl2_bundle")
-logger.setLevel("DEBUG")
-logger.addHandler(stderr_handler)
-
 def safe_json_decode(value: str) -> Any:
     """Safely decode the string values into JSON."""
     try:
@@ -54,7 +49,10 @@ def percent_completion(geno: float, pheno: float) -> float:
     """Compute the total completion percent."""
     return 0.5 * (geno + pheno)
 
-def process_bundle(dbconn: mdb.Connection, rconn: Redis, jobid: uuid.UUID) -> int:
+def process_bundle(dbconn: mdb.Connection,
+                   rconn: Redis,
+                   jobid: uuid.UUID,
+                   logger: Logger) -> int:
     """Process the R/qtl2 bundle."""
     try:
         thejob = parse_job(rconn, jobid)
@@ -69,7 +67,8 @@ def process_bundle(dbconn: mdb.Connection, rconn: Redis, jobid: uuid.UUID) -> in
                 meta["speciesid"],
                 meta["populationid"],
                 meta["geno-dataset-id"],
-                Path(meta["rqtl2-bundle-file"]))
+                Path(meta["rqtl2-bundle-file"]),
+                logger)
             if genoexit != 0:
                 raise Exception("Processing 'geno' file failed.")
             logger.debug(
@@ -83,7 +82,8 @@ def process_bundle(dbconn: mdb.Connection, rconn: Redis, jobid: uuid.UUID) -> in
                 meta["speciesid"],
                 meta["platformid"],
                 meta["probe-dataset-id"],
-                Path(meta["rqtl2-bundle-file"]))
+                Path(meta["rqtl2-bundle-file"]),
+                logger)
             if phenoexit != 0:
                 raise Exception("Processing 'pheno' file failed.")
             logger.debug(
@@ -109,13 +109,17 @@ if __name__ == "__main__":
         check_db(args.databaseuri)
         check_redis(args.redisuri)
 
+        logger = getLogger("process_rqtl2_bundle")
+        logger.addHandler(StreamHandler(stream=sys.stderr))
+        logger.setLevel("DEBUG")
+
         jobid = args.jobid
         with (database_connection(args.databaseuri) as dbconn,
               Redis.from_url(args.redisuri, decode_responses=True) as rconn):
             logger.addHandler(setup_redis_logger(
                 rconn, jobid, f"{str(jobid)}:log-messages", args.redisexpiry))
 
-            exitcode = process_bundle(dbconn, rconn, args.jobid)
+            exitcode = process_bundle(dbconn, rconn, args.jobid, logger)
             rconn.hset(str(args.jobid), "percent", "100")
             return exitcode
 
diff --git a/scripts/rqtl2/install_genotypes.py b/scripts/rqtl2/install_genotypes.py
index 733ccf9..bfe17b8 100644
--- a/scripts/rqtl2/install_genotypes.py
+++ b/scripts/rqtl2/install_genotypes.py
@@ -1,11 +1,11 @@
 """Load genotypes from R/qtl2 bundle into the database."""
 import sys
-import logging
 import traceback
 from pathlib import Path
 from zipfile import ZipFile
 from functools import reduce
 from typing import Iterator, Optional
+from logging import Logger, getLogger, StreamHandler
 
 import MySQLdb as mdb
 from MySQLdb.cursors import DictCursor
@@ -18,10 +18,6 @@ from scripts.rqtl2.entry import build_main
 from scripts.rqtl2.cli_parser import add_common_arguments
 from scripts.cli_parser import init_cli_parser, add_global_data_arguments
 
-stderr_handler = logging.StreamHandler(stream=sys.stderr)
-logger = logging.getLogger("install_genotypes")
-logger.addHandler(stderr_handler)
-
 def insert_markers(dbconn: mdb.Connection,
                    speciesid: int,
                    markers: tuple[str, ...],
@@ -158,11 +154,13 @@ def cross_reference_genotypes(dbconn: mdb.Connection,
             } for row in dataids))
         return cursor.rowcount
 
-def install_genotypes(dbconn: mdb.Connection,
+def install_genotypes(#pylint: disable=[too-many-arguments, too-many-locals]
+        dbconn: mdb.Connection,
                       speciesid: int,
                       populationid: int,
                       datasetid: int,
-                      rqtl2bundle: Path) -> int:
+                      rqtl2bundle: Path,
+                      logger: Logger = getLogger()) -> int:
     """Load any existing genotypes into the database."""
     count = 0
     with ZipFile(str(rqtl2bundle.absolute()), "r") as zfile:
@@ -223,6 +221,8 @@ if __name__ == "__main__":
 
         return parser.parse_args()
 
+    thelogger = getLogger("install_genotypes")
+    thelogger.addHandler(StreamHandler(stream=sys.stderr))
     main = build_main(
         cli_args(),
         lambda dbconn, args: install_genotypes(dbconn,
@@ -230,6 +230,6 @@ if __name__ == "__main__":
                                                args.populationid,
                                                args.datasetid,
                                                args.rqtl2bundle),
-        logger,
+        thelogger,
         "INFO")
     sys.exit(main())
diff --git a/scripts/rqtl2/install_phenos.py b/scripts/rqtl2/install_phenos.py
index 21b5f00..a6cdacc 100644
--- a/scripts/rqtl2/install_phenos.py
+++ b/scripts/rqtl2/install_phenos.py
@@ -1,10 +1,10 @@
 """Load pheno from R/qtl2 bundle into the database."""
 import sys
-import logging
 import traceback
 from pathlib import Path
 from zipfile import ZipFile
 from functools import reduce
+from logging import Logger, getLogger, StreamHandler
 
 import MySQLdb as mdb
 from MySQLdb.cursors import DictCursor
@@ -16,9 +16,6 @@ from scripts.cli_parser import init_cli_parser, add_global_data_arguments
 from r_qtl import r_qtl2 as rqtl2
 
 from functional_tools import take
-stderr_handler = logging.StreamHandler(stream=sys.stderr)
-logger = logging.getLogger("install_phenos")
-logger.addHandler(stderr_handler)
 
 def insert_probesets(dbconn: mdb.Connection,
                      platformid: int,
@@ -95,11 +92,13 @@ def cross_reference_probeset_data(dbconn: mdb.Connection,
             } for row in dataids))
         return cursor.rowcount
 
-def install_pheno_files(dbconn: mdb.Connection,
-                        speciesid: int,
-                        platformid: int,
-                        datasetid: int,
-                        rqtl2bundle: Path) -> int:
+def install_pheno_files(#pylint: disable=[too-many-arguments, too-many-locals]
+        dbconn: mdb.Connection,
+        speciesid: int,
+        platformid: int,
+        datasetid: int,
+        rqtl2bundle: Path,
+        logger: Logger = getLogger()) -> int:
     """Load data in `pheno` files and other related files into the database."""
     with ZipFile(str(rqtl2bundle), "r") as zfile:
         try:
@@ -155,13 +154,16 @@ if __name__ == "__main__":
 
         return parser.parse_args()
 
+    thelogger = getLogger("install_phenos")
+    thelogger.addHandler(StreamHandler(stream=sys.stderr))
     main = build_main(
         cli_args(),
         lambda dbconn, args: install_pheno_files(dbconn,
                                                  args.speciesid,
                                                  args.platformid,
                                                  args.datasetid,
-                                                 args.rqtl2bundle),
-        logger,
+                                                 args.rqtl2bundle,
+                                                 thelogger),
+        thelogger,
         "DEBUG")
     sys.exit(main())