about summary refs log tree commit diff
path: root/scripts
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-11-14 15:10:28 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-11-14 15:10:28 +0000
commit6daec6b5d03a80af3e2496b6bf666340b3d718fd (patch)
tree842d614ba0a5cdd72473ea0d96eb061fd8fef0f5 /scripts
parent586b0f63ef31fde987e00a305e91ee4e2136332e (diff)
downloadgenenetwork4-6daec6b5d03a80af3e2496b6bf666340b3d718fd.tar.gz
log: name the person and the credential, and resolve handles from the registry
The access log said who (`user`) but not with what: with short tokens a
request can be made by a shared credential, so "which person" and "which
credential" are now separate fields -- `scheme` ("jwt" | "opaque") and,
for an opaque token, `token` (its registry handle, e.g.
`agent-ravanan-3`). Never the token or its hash, and the path still
carries no query string.

The line also gains `ts` (RFC 3339 UTC), so a redirected stderr file is
self-describing rather than relying on journald's prefix. Hand-rolled
civil-from-days rather than a new crate: one format, and no Cargo.lock
drift.

scripts/gn4-log.py reads such a log and shows it as a table with the
handle joined against tokens.sqlite: the subject as the registry has it
now (flagging a handle whose subject has been repointed), and the token's
status today -- [revoked], [expired], [unknown] -- which is how "an agent
was working at 14:02" is told apart from "an agent was working at 14:02
with a credential revoked since". It reads the registry read-only, passes
non-JSON lines through in table mode (the process log interleaves
backtraces), and offers --json and --summary for jq and for counting.

CallerSlot now carries the whole Caller (scheme, id, subject) rather than
just the subject string.
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/gn4-log.py242
1 files changed, 242 insertions, 0 deletions
diff --git a/scripts/gn4-log.py b/scripts/gn4-log.py
new file mode 100755
index 0000000..a14e653
--- /dev/null
+++ b/scripts/gn4-log.py
@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+"""Read the server's access log and show it with usernames.
+
+Every request writes one JSON line on stderr (src/lib.rs, `request_log_line`):
+
+    {"event":"request","method":"POST","path":"/compute/gemma","status":200,
+     "took_ms":912,"request_id":"...","user":"zachs",
+     "scheme":"opaque","token":"agent-ravanan-3"}
+
+`user` is already the gn-auth user id, so most lines need no translation. The
+`token` handle does: this script joins it against the token registry
+(`tokens.sqlite`, `GN4_API_TOKENS_DB`) and shows
+
+- the subject **as the registry has it now** -- a token's row is authoritative,
+  so a handle whose target changed is visible;
+- the token's status *today* -- `active`, `expired` or `revoked` -- which is
+  how you tell "an agent was working at 14:02" from "an agent was working at
+  14:02 with a credential that has since been revoked";
+- the note on the row, e.g. what the token was minted for.
+
+Use it on a process log or a proxy log that carries `x-gn4-user`:
+
+    journalctl -u genenetwork4 | scripts/gn4-log.py
+    scripts/gn4-log.py -f /var/log/gn4.log --since 2026-11-14 --grep gemma
+    scripts/gn4-log.py -f log.jsonl --json          # enriched JSON, for jq
+    scripts/gn4-log.py -f log.jsonl --summary       # who ran what, counted
+
+Lines that are not JSON (a Ruby backtrace, a GEMMA progress bar) pass through
+unchanged unless --quiet; that is deliberate, because the process log
+interleaves them and dropping them would hide the interesting failures.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sqlite3
+import sys
+import time
+from collections import Counter
+from pathlib import Path
+
+DATE = re.compile(r"^(?P<day>\d{4}-\d{2}-\d{2})")
+TIME = re.compile(r"(?P<time>\d{2}:\d{2}:\d{2})")
+
+
+def registry_path(explicit: str | None) -> Path:
+    if explicit:
+        return Path(explicit)
+    return Path(os.environ.get("GN4_API_TOKENS_DB", "tokens.sqlite"))
+
+
+class Registry:
+    """Read-only view of the token registry, for translating handles."""
+
+    def __init__(self, path: Path):
+        self.path = path
+        self.connection = None
+        self.error = None
+        if not path.exists():
+            self.error = f"no registry at {path} (set GN4_API_TOKENS_DB or --db)"
+            return
+        try:
+            # Read-only, and never creating: this must not touch a live
+            # database's write path.
+            uri = f"file:{path}?mode=ro"
+            self.connection = sqlite3.connect(uri, uri=True, timeout=5)
+            self.connection.row_factory = sqlite3.Row
+        except sqlite3.Error as error:  # pragma: no cover - environment
+            self.error = f"cannot read {path}: {error}"
+
+    def by_id(self, identifier: str) -> sqlite3.Row | None:
+        if self.connection is None:
+            return None
+        try:
+            return self.connection.execute(
+                "SELECT id, subject, scope, prefix, created_at, expires_at,"
+                "       last_used_at, revoked_at, note FROM token WHERE id = ?",
+                (identifier,),
+            ).fetchone()
+        except sqlite3.Error:
+            return None
+
+    def status(self, row: sqlite3.Row | None, at: int | None = None) -> str:
+        if row is None:
+            return "unknown"
+        if row["revoked_at"] is not None:
+            return "revoked"
+        expires = row["expires_at"]
+        if expires is not None and expires <= (at or int(time.time())):
+            return "expired"
+        return "active"
+
+
+def read_lines(handle) -> list[str]:
+    return [line.rstrip("\n") for line in handle]
+
+
+def parse(line: str) -> dict | None:
+    if '"event":"request"' not in line and '"event": "request"' not in line:
+        return None
+    start = line.find("{")
+    if start < 0:
+        return None
+    try:
+        record = json.loads(line[start:])
+    except json.JSONDecodeError:
+        return None
+    return record if record.get("event") == "request" else None
+
+
+def when_of(record: dict, line: str) -> str:
+    """The record's own `ts` wins; a journald/nginx prefix is the fallback."""
+    stamp = record.get("ts")
+    if isinstance(stamp, str) and len(stamp) >= 19:
+        return f"{stamp[:10]} {stamp[11:19]}"
+    return timestamp_of(line)
+
+
+def timestamp_of(line: str) -> str:
+    day = DATE.search(line)
+    clock = TIME.search(line)
+    if day and clock:
+        return f"{day.group('day')} {clock.group('time')}"
+    return ""
+
+
+def enrich(record: dict, registry: Registry, when: int | None) -> dict:
+    out = dict(record)
+    handle = record.get("token")
+    if handle:
+        row = registry.by_id(handle)
+        if row is None:
+            out["token_status"] = "unknown"
+        else:
+            out["token_status"] = registry.status(row, when)
+            out["token_subject"] = row["subject"]
+            out["token_scope"] = row["scope"]
+            if row["note"]:
+                out["token_note"] = row["note"]
+            if row["expires_at"]:
+                out["token_expires_at"] = row["expires_at"]
+            # A row whose subject differs from the logged one is worth
+            # seeing: the registry is authoritative, and this is the only
+            # way an operator notices a token was repointed.
+            if record.get("user") and row["subject"] != record["user"]:
+                out["token_subject_changed"] = True
+                out["user"] = row["subject"]
+    return out
+
+
+def format_line(record: dict, when: str) -> str:
+    who = record.get("user") or "-"
+    credential = record.get("scheme") or "-"
+    if record.get("token"):
+        credential += f"/{record['token']}"
+        status = record.get("token_status")
+        if status and status != "active":
+            credential += f" [{status}]"
+        if record.get("token_subject_changed"):
+            credential += " [subject changed]"
+    took = record.get("took_ms", 0)
+    return (
+        f"{when or '-':<19} {who:<20} {credential:<34} "
+        f"{record.get('method', '-'):<6} {record.get('status', '-'):>4} "
+        f"{took:>6} ms  {record.get('path', '')}"
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("files", nargs="*", help="log files (default: stdin)")
+    parser.add_argument("-f", "--file", action="append", default=[], help="a log file")
+    parser.add_argument("--db", default=None, help="the token registry (default GN4_API_TOKENS_DB, else ./tokens.sqlite)")
+    parser.add_argument("--since", default=None, help="only lines whose date prefix is at or after this (YYYY-MM-DD)")
+    parser.add_argument("--grep", default=None, help="only lines containing this text (matches the raw line)")
+    parser.add_argument("--json", action="store_true", help="emit enriched JSON instead of a table")
+    parser.add_argument("--summary", action="store_true", help="count requests per caller and credential")
+    parser.add_argument("--quiet", action="store_true", help="drop lines that are not request records")
+    args = parser.parse_args()
+
+    registry = Registry(registry_path(args.db))
+    if registry.error and (args.json or args.summary):
+        print(f"gn4-log: {registry.error}", file=sys.stderr)
+
+    files = args.files + args.file
+    handles = [open(path) for path in files] or [sys.stdin]
+    try:
+        lines = [line for handle in handles for line in read_lines(handle)]
+    finally:
+        for handle in handles:
+            if handle is not sys.stdin:
+                handle.close()
+
+    rows: list[tuple[str, dict]] = []
+    for line in lines:
+        if args.grep and args.grep not in line:
+            continue
+        record = parse(line)
+        when = when_of(record, line) if record else timestamp_of(line)
+        if args.since and when:
+            # Compare like with like: the table shows "YYYY-MM-DD HH:MM:SS",
+            # and --since accepts a bare date or a date and a time.
+            stamp = when.replace(" ", "T")
+            if stamp < args.since:
+                continue
+        if record is None:
+            # Non-JSON lines (a Ruby backtrace, a GEMMA progress bar) pass
+            # through in table mode -- the process log interleaves them, and
+            # dropping them would hide the interesting failures -- but never
+            # in a machine-readable mode.
+            if not args.quiet and not args.json and not args.summary:
+                print(line)
+            continue
+        rows.append((when, enrich(record, registry, None)))
+
+    if args.summary:
+        callers = Counter(
+            (record.get("user") or "-", record.get("scheme") or "-", record.get("token") or "-")
+            for _, record in rows
+        )
+        width = max((len(" ".join(key)) for key in callers), default=10)
+        for (user, scheme, handle), count in callers.most_common():
+            credential = scheme if handle == "-" else f"{scheme}/{handle}"
+            label = f"{user}  {credential}"
+            print(f"{count:>7}  {label:<{width}}")
+        print(f"{len(rows):>7}  {'total':<{width}}")
+        return 0
+
+    for when, record in rows:
+        if args.json:
+            print(json.dumps(record))
+        else:
+            print(format_line(record, when))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())