diff --git a/gn_libs/jobs/jobs.py b/gn_libs/jobs/jobs.py
index f45819c..03b9b44 100644
--- a/gn_libs/jobs/jobs.py
+++ b/gn_libs/jobs/jobs.py
@@ -7,7 +7,7 @@ import logging
import subprocess
from pathlib import Path
from functools import partial
-from typing import Mapping, Union, Optional
+from typing import Any, Mapping, Union, Optional
from datetime import datetime, timezone, timedelta
from gn_libs.sqlite3 import DbCursor, DbConnection, cursor as _cursor
@@ -160,23 +160,57 @@ def initialise_job(# pylint: disable=[too-many-arguments, too-many-positional-ar
job_id: uuid.UUID,
command: list,
job_type: str,
- extra_meta: Optional[dict] = None,
+ *,
expiry_seconds: int = _DEFAULT_EXPIRY_SECONDS_,
- external_id: Optional[Union[str, uuid.UUID]] = None
+ **extra_meta
) -> dict:
- """Initialise the job and put the details in a SQLite3 database."""
- if extra_meta is None:
- extra_meta = {}
+ r"""Initialise the job and put the details in a SQLite3 database.
+
+ Metadata is passed as keyword arguments. ``expiry_seconds`` is a
+ reserved operational argument and is handled explicitly rather
+ than as user metadata.
+
+ The ``**extra_meta`` takes any type for the values; in practice,
+ however, those values should be easy to marshall/unmarshall
+ to/from the ``str`` type.
+
+ The following special metadata value items need attention:
+
+ - An ``external_id`` metadata value, when it exists, should be a
+ ``str``, ``uuid.UUID`` or ``None`` value. Any other value will
+ raise an exception.
+ - A ``workflow_claim`` metadata value of the form
+ ``<workflow>:<root-id>:<stage>`` can be used, together with a
+ unique index on the metadata value, to make stage transitions
+ idempotent.
+
+ :param conn: A connection to the jobs database.
+ :type conn: gn_libs.sqlite3.DbConnection
+ :param job_id: The identifier for this job.
+ :type job_id: uuid.UUID
+ :param command: The command this job will run.
+ :type command: Sequence[str]
+ :param expiry_seconds: How long to keep the job around for from creation
+ :type expiry_seconds: int
+ :param \*\*extra_meta: extra metadata fields for the job.
+ :type \*\*extra_meta: Any
+ """
+
+ def __process_external_id__(_id: Any) -> dict[str, str]:
+ if _id is None:
+ return {}
- def __process_external_id__(_id: Optional[Union[str, uuid.UUID]]) -> str:
if isinstance(_id, uuid.UUID):
- return str(_id)
+ return {"external_id": str(_id)}
- if _id is not None and bool(_id.strip()):
- return str(_id.strip())
- return ""
+ if isinstance(_id, str):
+ stripped = _id.strip()
+ return {"external_id": stripped} if bool(stripped) else {}
- _ext_id = __process_external_id__(external_id)
+ raise ValueError(
+ "`external_id` MUST be a string or uuid.UUID value when provided")
+
+ _ext_id = __process_external_id__(extra_meta.pop("external_id", None))
_job = {
"job_id": job_id,
"command": shlex.join(command),
@@ -186,10 +220,10 @@ def initialise_job(# pylint: disable=[too-many-arguments, too-many-positional-ar
"percent": 0,
"job-type": job_type,
**extra_meta,
- **({"external_id": _ext_id} if bool(_ext_id) else {})
+ **_ext_id
}
}
- return __save_job__(conn, _job, expiry_seconds, _ext_id)
+ return __save_job__(conn, _job, expiry_seconds, _ext_id.get("external_id", ""))
def output_file(jobid: uuid.UUID, outdir: Path, stream: str) -> Path:
diff --git a/tests/unit/test_jobs.py b/tests/unit/test_jobs.py
index f513764..a026870 100644
--- a/tests/unit/test_jobs.py
+++ b/tests/unit/test_jobs.py
@@ -20,7 +20,7 @@ def jobsdb(tmp_path):
def _job(conn, job_type, **metadata):
"""Create a job with the given type and metadata."""
return jobs.initialise_job(
- conn, str(uuid.uuid4()), ["true"], job_type, extra_meta=metadata)
+ conn, str(uuid.uuid4()), ["true"], job_type, **metadata)
@pytest.mark.unit_test
@@ -82,3 +82,23 @@ def test_jobs_by_metadata_returns_empty_when_no_match(jobsdb):
_job(conn, "qc", genotype_collector_job_id="c1")
assert jobs.jobs_by_metadata(
conn, {"genotype_collector_job_id": "nope"}) == ()
+
+
+@pytest.mark.unit_test
+def test_initialise_job_treats_metadata_kwargs_as_metadata(jobsdb):
+ """
+ GIVEN: metadata and reserved operational arguments
+ WHEN: a job is initialised
+ THEN: the metadata is stored, while expiry_seconds is not metadata
+ """
+ with sqlite3.connection(str(jobsdb)) as conn:
+ job = jobs.initialise_job(
+ conn, str(uuid.uuid4()), ["true"], "qc",
+ genotype_collector_job_id="c1",
+ external_id="user-1", expiry_seconds=60)
+ stored = jobs.job(conn, job["job_id"])
+ assert stored["metadata"]["genotype_collector_job_id"] == "c1"
+ assert stored["metadata"]["external_id"] == "user-1"
+ assert "expiry_seconds" not in stored["metadata"]
+ assert jobs.jobs_by_external_id(conn, "user-1")[0]["job_id"] == (
+ job["job_id"])
|