diff options
| -rw-r--r-- | gn_libs/jobs/jobs.py | 62 | ||||
| -rw-r--r-- | tests/unit/test_jobs.py | 22 |
2 files changed, 69 insertions, 15 deletions
diff --git a/gn_libs/jobs/jobs.py b/gn_libs/jobs/jobs.py index f45819c..03b9b44 100644 --- a/gn_libs/jobs/jobs.py +++ b/gn_libs/jobs/jobs.py @@ -7,7 +7,7 @@ import logging import subprocess from pathlib import Path from functools import partial -from typing import Mapping, Union, Optional +from typing import Any, Mapping, Union, Optional from datetime import datetime, timezone, timedelta from gn_libs.sqlite3 import DbCursor, DbConnection, cursor as _cursor @@ -160,23 +160,57 @@ def initialise_job(# pylint: disable=[too-many-arguments, too-many-positional-ar job_id: uuid.UUID, command: list, job_type: str, - extra_meta: Optional[dict] = None, + *, expiry_seconds: int = _DEFAULT_EXPIRY_SECONDS_, - external_id: Optional[Union[str, uuid.UUID]] = None + **extra_meta ) -> dict: - """Initialise the job and put the details in a SQLite3 database.""" - if extra_meta is None: - extra_meta = {} + r"""Initialise the job and put the details in a SQLite3 database. + + Metadata is passed as keyword arguments. ``expiry_seconds`` is a + reserved operational argument and is handled explicitly rather + than as user metadata. + + The ``**extra_meta`` takes any type for the values; in practice, + however, those values should be easy to marshall/unmarshall + to/from the ``str`` type. + + The following special metadata value items need attention: + + - An ``external_id`` metadata value, when it exists, should be a + ``str``, ``uuid.UUID`` or ``None`` value. Any other value will + raise an exception. + - A ``workflow_claim`` metadata value of the form + ``<workflow>:<root-id>:<stage>`` can be used, together with a + unique index on the metadata value, to make stage transitions + idempotent. + + :param conn: A connection to the jobs database. + :type conn: gn_libs.sqlite3.DbConnection + :param job_id: The identifier for this job. + :type job_id: uuid.UUID + :param command: The command this job will run. + :type command: Sequence[str] + :param expiry_seconds: How long to keep the job around for from creation + :type expiry_seconds: int + :param \*\*extra_meta: extra metadata fields for the job. + :type \*\*extra_meta: Any + """ + + def __process_external_id__(_id: Any) -> dict[str, str]: + if _id is None: + return {} - def __process_external_id__(_id: Optional[Union[str, uuid.UUID]]) -> str: if isinstance(_id, uuid.UUID): - return str(_id) + return {"external_id": str(_id)} - if _id is not None and bool(_id.strip()): - return str(_id.strip()) - return "" + if isinstance(_id, str): + stripped = _id.strip() + return {"external_id": stripped} if bool(stripped) else {} - _ext_id = __process_external_id__(external_id) + raise ValueError( + "`external_id` MUST be a string or uuid.UUID value when provided") + + _ext_id = __process_external_id__(extra_meta.pop("external_id", None)) _job = { "job_id": job_id, "command": shlex.join(command), @@ -186,10 +220,10 @@ def initialise_job(# pylint: disable=[too-many-arguments, too-many-positional-ar "percent": 0, "job-type": job_type, **extra_meta, - **({"external_id": _ext_id} if bool(_ext_id) else {}) + **_ext_id } } - return __save_job__(conn, _job, expiry_seconds, _ext_id) + return __save_job__(conn, _job, expiry_seconds, _ext_id.get("external_id", "")) def output_file(jobid: uuid.UUID, outdir: Path, stream: str) -> Path: diff --git a/tests/unit/test_jobs.py b/tests/unit/test_jobs.py index f513764..a026870 100644 --- a/tests/unit/test_jobs.py +++ b/tests/unit/test_jobs.py @@ -20,7 +20,7 @@ def jobsdb(tmp_path): def _job(conn, job_type, **metadata): """Create a job with the given type and metadata.""" return jobs.initialise_job( - conn, str(uuid.uuid4()), ["true"], job_type, extra_meta=metadata) + conn, str(uuid.uuid4()), ["true"], job_type, **metadata) @pytest.mark.unit_test @@ -82,3 +82,23 @@ def test_jobs_by_metadata_returns_empty_when_no_match(jobsdb): _job(conn, "qc", genotype_collector_job_id="c1") assert jobs.jobs_by_metadata( conn, {"genotype_collector_job_id": "nope"}) == () + + +@pytest.mark.unit_test +def test_initialise_job_treats_metadata_kwargs_as_metadata(jobsdb): + """ + GIVEN: metadata and reserved operational arguments + WHEN: a job is initialised + THEN: the metadata is stored, while expiry_seconds is not metadata + """ + with sqlite3.connection(str(jobsdb)) as conn: + job = jobs.initialise_job( + conn, str(uuid.uuid4()), ["true"], "qc", + genotype_collector_job_id="c1", + external_id="user-1", expiry_seconds=60) + stored = jobs.job(conn, job["job_id"]) + assert stored["metadata"]["genotype_collector_job_id"] == "c1" + assert stored["metadata"]["external_id"] == "user-1" + assert "expiry_seconds" not in stored["metadata"] + assert jobs.jobs_by_external_id(conn, "user-1")[0]["job_id"] == ( + job["job_id"]) |
