diff options
Diffstat (limited to 'gn_libs/jobs')
-rw-r--r-- | gn_libs/jobs/__init__.py | 1 | ||||
-rw-r--r-- | gn_libs/jobs/jobs.py | 21 | ||||
-rw-r--r-- | gn_libs/jobs/launcher.py | 6 | ||||
-rw-r--r-- | gn_libs/jobs/migrations.py | 5 |
4 files changed, 22 insertions, 11 deletions
diff --git a/gn_libs/jobs/__init__.py b/gn_libs/jobs/__init__.py index 6f400ef..d6e4ce3 100644 --- a/gn_libs/jobs/__init__.py +++ b/gn_libs/jobs/__init__.py @@ -1,3 +1,4 @@ +"""This package deals with launching and managing background/async jobs.""" from .migrations import run_migrations from .jobs import (job, launch_job, diff --git a/gn_libs/jobs/jobs.py b/gn_libs/jobs/jobs.py index 17c1ac6..8d77139 100644 --- a/gn_libs/jobs/jobs.py +++ b/gn_libs/jobs/jobs.py @@ -6,7 +6,6 @@ import shlex import logging import subprocess from pathlib import Path -from functools import reduce from functools import partial from typing import Union, Optional from datetime import datetime, timezone, timedelta @@ -87,16 +86,18 @@ def __save_job__(conn: DbConnection, the_job: dict, expiry_seconds: int) -> dict return the_job -def initialise_job( +def initialise_job(# pylint: disable=[too-many-arguments, too-many-positional-arguments] conn: DbConnection, job_id: uuid.UUID, command: list, job_type: str, - extra_meta: dict = {}, + extra_meta: Optional[dict] = None, expiry_seconds: Optional[int] = _DEFAULT_EXPIRY_SECONDS_ ) -> dict: """Initialise the job and put the details in a SQLite3 database.""" - + if extra_meta is None: + extra_meta = {} + _job = { "job_id": job_id, "command": shlex.join(command), @@ -121,7 +122,11 @@ stdout_filename = partial(output_file, stream="stdout") stderr_filename = partial(output_file, stream="stderr") -def build_environment(extras: dict[str, str] = {}): +def build_environment(extras: Optional[dict[str, str]] = None) -> dict[str, str]: + """Setup the runtime environment variables for the background script.""" + if extras is None: + extras = {} + return { **dict(os.environ), "PYTHONPATH": ":".join(sys.path), @@ -180,7 +185,11 @@ def update_metadata(conn: DbConnection, job_id: Union[str, uuid.UUID], key: str, }) -def push_to_stream(conn: DbConnection, job_id: Union[str, uuid.UUID], stream_name: str, content: str): +def push_to_stream( + conn: DbConnection, + job_id: Union[str, uuid.UUID], + stream_name: str, content: str +): """Initialise, and keep adding content to the stream from the provided content.""" with _cursor(conn) as cursor: cursor.execute("SELECT * FROM jobs_standard_outputs " diff --git a/gn_libs/jobs/launcher.py b/gn_libs/jobs/launcher.py index 89884b6..d565f9e 100644 --- a/gn_libs/jobs/launcher.py +++ b/gn_libs/jobs/launcher.py @@ -1,3 +1,4 @@ +"""Default launcher/manager script for background jobs.""" import os import sys import time @@ -54,7 +55,7 @@ def run_job(conn, job, outputs_directory: Path): logger.info("exiting job manager/launcher") return exit_status - except: + except Exception as _exc:# pylint: disable=[broad-exception-caught] logger.error("An exception was raised when attempting to run the job", exc_info=True) jobs.update_metadata(conn, job_id, "status", "error") @@ -90,8 +91,7 @@ def main(): args = parse_args() logger.setLevel(args.log_level.upper()) args.outputs_directory.mkdir(parents=True, exist_ok=True) - with (sqlite3.connection(args.jobs_db_uri) as conn, - sqlite3.cursor(conn) as cursor): + with sqlite3.connection(args.jobs_db_uri) as conn: job = jobs.job(conn, args.job_id) if job: return run_job(conn, job, args.outputs_directory) diff --git a/gn_libs/jobs/migrations.py b/gn_libs/jobs/migrations.py index 86fb958..0c9825b 100644 --- a/gn_libs/jobs/migrations.py +++ b/gn_libs/jobs/migrations.py @@ -1,6 +1,6 @@ """Database migrations for the jobs to ensure consistency downstream.""" from gn_libs.protocols import DbCursor -from gn_libs.sqlite3 import cursor, connection +from gn_libs.sqlite3 import connection, cursor as acquire_cursor def __create_table_jobs__(cursor: DbCursor): """Create the jobs table""" @@ -61,8 +61,9 @@ def __create_table_jobs_output_streams__(cursor: DbCursor): def run_migrations(sqlite_url: str): + """Run the migrations to setup the background jobs database.""" with (connection(sqlite_url) as conn, - cursor(conn) as curr): + acquire_cursor(conn) as curr): __create_table_jobs__(curr) __create_table_jobs_metadata__(curr) __create_table_jobs_output_streams__(curr) |