aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--gn_libs/jobs/__init__.py1
-rw-r--r--gn_libs/jobs/jobs.py21
-rw-r--r--gn_libs/jobs/launcher.py6
-rw-r--r--gn_libs/jobs/migrations.py5
-rw-r--r--gn_libs/monadic_requests.py20
-rw-r--r--gn_libs/protocols/__init__.py1
-rw-r--r--gn_libs/sqlite3.py3
7 files changed, 40 insertions, 17 deletions
diff --git a/gn_libs/jobs/__init__.py b/gn_libs/jobs/__init__.py
index 6f400ef..d6e4ce3 100644
--- a/gn_libs/jobs/__init__.py
+++ b/gn_libs/jobs/__init__.py
@@ -1,3 +1,4 @@
+"""This package deals with launching and managing background/async jobs."""
from .migrations import run_migrations
from .jobs import (job,
launch_job,
diff --git a/gn_libs/jobs/jobs.py b/gn_libs/jobs/jobs.py
index 17c1ac6..8d77139 100644
--- a/gn_libs/jobs/jobs.py
+++ b/gn_libs/jobs/jobs.py
@@ -6,7 +6,6 @@ import shlex
import logging
import subprocess
from pathlib import Path
-from functools import reduce
from functools import partial
from typing import Union, Optional
from datetime import datetime, timezone, timedelta
@@ -87,16 +86,18 @@ def __save_job__(conn: DbConnection, the_job: dict, expiry_seconds: int) -> dict
return the_job
-def initialise_job(
+def initialise_job(# pylint: disable=[too-many-arguments, too-many-positional-arguments]
conn: DbConnection,
job_id: uuid.UUID,
command: list,
job_type: str,
- extra_meta: dict = {},
+ extra_meta: Optional[dict] = None,
expiry_seconds: Optional[int] = _DEFAULT_EXPIRY_SECONDS_
) -> dict:
"""Initialise the job and put the details in a SQLite3 database."""
-
+ if extra_meta is None:
+ extra_meta = {}
+
_job = {
"job_id": job_id,
"command": shlex.join(command),
@@ -121,7 +122,11 @@ stdout_filename = partial(output_file, stream="stdout")
stderr_filename = partial(output_file, stream="stderr")
-def build_environment(extras: dict[str, str] = {}):
+def build_environment(extras: Optional[dict[str, str]] = None) -> dict[str, str]:
+ """Setup the runtime environment variables for the background script."""
+ if extras is None:
+ extras = {}
+
return {
**dict(os.environ),
"PYTHONPATH": ":".join(sys.path),
@@ -180,7 +185,11 @@ def update_metadata(conn: DbConnection, job_id: Union[str, uuid.UUID], key: str,
})
-def push_to_stream(conn: DbConnection, job_id: Union[str, uuid.UUID], stream_name: str, content: str):
+def push_to_stream(
+ conn: DbConnection,
+ job_id: Union[str, uuid.UUID],
+ stream_name: str, content: str
+):
"""Initialise, and keep adding content to the stream from the provided content."""
with _cursor(conn) as cursor:
cursor.execute("SELECT * FROM jobs_standard_outputs "
diff --git a/gn_libs/jobs/launcher.py b/gn_libs/jobs/launcher.py
index 89884b6..d565f9e 100644
--- a/gn_libs/jobs/launcher.py
+++ b/gn_libs/jobs/launcher.py
@@ -1,3 +1,4 @@
+"""Default launcher/manager script for background jobs."""
import os
import sys
import time
@@ -54,7 +55,7 @@ def run_job(conn, job, outputs_directory: Path):
logger.info("exiting job manager/launcher")
return exit_status
- except:
+ except Exception as _exc:# pylint: disable=[broad-exception-caught]
logger.error("An exception was raised when attempting to run the job",
exc_info=True)
jobs.update_metadata(conn, job_id, "status", "error")
@@ -90,8 +91,7 @@ def main():
args = parse_args()
logger.setLevel(args.log_level.upper())
args.outputs_directory.mkdir(parents=True, exist_ok=True)
- with (sqlite3.connection(args.jobs_db_uri) as conn,
- sqlite3.cursor(conn) as cursor):
+ with sqlite3.connection(args.jobs_db_uri) as conn:
job = jobs.job(conn, args.job_id)
if job:
return run_job(conn, job, args.outputs_directory)
diff --git a/gn_libs/jobs/migrations.py b/gn_libs/jobs/migrations.py
index 86fb958..0c9825b 100644
--- a/gn_libs/jobs/migrations.py
+++ b/gn_libs/jobs/migrations.py
@@ -1,6 +1,6 @@
"""Database migrations for the jobs to ensure consistency downstream."""
from gn_libs.protocols import DbCursor
-from gn_libs.sqlite3 import cursor, connection
+from gn_libs.sqlite3 import connection, cursor as acquire_cursor
def __create_table_jobs__(cursor: DbCursor):
"""Create the jobs table"""
@@ -61,8 +61,9 @@ def __create_table_jobs_output_streams__(cursor: DbCursor):
def run_migrations(sqlite_url: str):
+ """Run the migrations to setup the background jobs database."""
with (connection(sqlite_url) as conn,
- cursor(conn) as curr):
+ acquire_cursor(conn) as curr):
__create_table_jobs__(curr)
__create_table_jobs_metadata__(curr)
__create_table_jobs_output_streams__(curr)
diff --git a/gn_libs/monadic_requests.py b/gn_libs/monadic_requests.py
index 0a3c282..a09acc5 100644
--- a/gn_libs/monadic_requests.py
+++ b/gn_libs/monadic_requests.py
@@ -19,8 +19,13 @@ def get(url, params=None, **kwargs) -> Either:
:rtype: pymonad.either.Either
"""
+ timeout = kwargs.get("timeout")
+ kwargs = {key: val for key,val in kwargs.items() if key != "timeout"}
+ if timeout is None:
+ timeout = (9.13, 20)
+
try:
- resp = requests.get(url, params=params, **kwargs)
+ resp = requests.get(url, params=params, timeout=timeout, **kwargs)
if resp.status_code in SUCCESS_CODES:
return Right(resp.json())
return Left(resp)
@@ -36,8 +41,13 @@ def post(url, data=None, json=None, **kwargs) -> Either:
:rtype: pymonad.either.Either
"""
+ timeout = kwargs.get("timeout")
+ kwargs = {key: val for key,val in kwargs.items() if key != "timeout"}
+ if timeout is None:
+ timeout = (9.13, 20)
+
try:
- resp = requests.post(url, data=data, json=json, **kwargs)
+ resp = requests.post(url, data=data, json=json, timeout=timeout, **kwargs)
if resp.status_code in SUCCESS_CODES:
return Right(resp.json())
return Left(resp)
@@ -55,10 +65,10 @@ def make_either_error_handler(msg):
try:
_data = error.json()
except Exception as _exc:
- raise Exception(error.content) from _exc
- raise Exception(_data)
+ raise Exception(error.content) from _exc# pylint: disable=[broad-exception-raised]
+ raise Exception(_data)# pylint: disable=[broad-exception-raised]
logger.debug("\n\n%s\n\n", msg)
- raise Exception(error)
+ raise Exception(error)# pylint: disable=[broad-exception-raised]
return __fail__
diff --git a/gn_libs/protocols/__init__.py b/gn_libs/protocols/__init__.py
index e71f1ce..83a31a8 100644
--- a/gn_libs/protocols/__init__.py
+++ b/gn_libs/protocols/__init__.py
@@ -1 +1,2 @@
+"""This package is a collection of major Protocols/Interfaces definitions."""
from .db import DbCursor, DbConnection
diff --git a/gn_libs/sqlite3.py b/gn_libs/sqlite3.py
index 1dcdf29..78e1c41 100644
--- a/gn_libs/sqlite3.py
+++ b/gn_libs/sqlite3.py
@@ -1,7 +1,8 @@
+"""This module deals with connections to a(n) SQLite3 database."""
import logging
import traceback
import contextlib
-from typing import Any, Protocol, Callable, Iterator
+from typing import Callable, Iterator
import sqlite3
generated by cgit v1.2.3 (git 2.47.1) at 2025-06-17 01:39:54 +0000