diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-14 15:10:28 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-14 15:10:28 +0000 |
| commit | 6daec6b5d03a80af3e2496b6bf666340b3d718fd (patch) | |
| tree | 842d614ba0a5cdd72473ea0d96eb061fd8fef0f5 /scripts | |
| parent | 586b0f63ef31fde987e00a305e91ee4e2136332e (diff) | |
| download | genenetwork4-6daec6b5d03a80af3e2496b6bf666340b3d718fd.tar.gz | |
log: name the person and the credential, and resolve handles from the registry
The access log said who (`user`) but not with what: with short tokens a
request can be made by a shared credential, so "which person" and "which
credential" are now separate fields -- `scheme` ("jwt" | "opaque") and,
for an opaque token, `token` (its registry handle, e.g.
`agent-ravanan-3`). Never the token or its hash, and the path still
carries no query string.
The line also gains `ts` (RFC 3339 UTC), so a redirected stderr file is
self-describing rather than relying on journald's prefix. Hand-rolled
civil-from-days rather than a new crate: one format, and no Cargo.lock
drift.
scripts/gn4-log.py reads such a log and shows it as a table with the
handle joined against tokens.sqlite: the subject as the registry has it
now (flagging a handle whose subject has been repointed), and the token's
status today -- [revoked], [expired], [unknown] -- which is how "an agent
was working at 14:02" is told apart from "an agent was working at 14:02
with a credential revoked since". It reads the registry read-only, passes
non-JSON lines through in table mode (the process log interleaves
backtraces), and offers --json and --summary for jq and for counting.
CallerSlot now carries the whole Caller (scheme, id, subject) rather than
just the subject string.
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/gn4-log.py | 242 |
1 files changed, 242 insertions, 0 deletions
diff --git a/scripts/gn4-log.py b/scripts/gn4-log.py new file mode 100755 index 0000000..a14e653 --- /dev/null +++ b/scripts/gn4-log.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Read the server's access log and show it with usernames. + +Every request writes one JSON line on stderr (src/lib.rs, `request_log_line`): + + {"event":"request","method":"POST","path":"/compute/gemma","status":200, + "took_ms":912,"request_id":"...","user":"zachs", + "scheme":"opaque","token":"agent-ravanan-3"} + +`user` is already the gn-auth user id, so most lines need no translation. The +`token` handle does: this script joins it against the token registry +(`tokens.sqlite`, `GN4_API_TOKENS_DB`) and shows + +- the subject **as the registry has it now** -- a token's row is authoritative, + so a handle whose target changed is visible; +- the token's status *today* -- `active`, `expired` or `revoked` -- which is + how you tell "an agent was working at 14:02" from "an agent was working at + 14:02 with a credential that has since been revoked"; +- the note on the row, e.g. what the token was minted for. + +Use it on a process log or a proxy log that carries `x-gn4-user`: + + journalctl -u genenetwork4 | scripts/gn4-log.py + scripts/gn4-log.py -f /var/log/gn4.log --since 2026-11-14 --grep gemma + scripts/gn4-log.py -f log.jsonl --json # enriched JSON, for jq + scripts/gn4-log.py -f log.jsonl --summary # who ran what, counted + +Lines that are not JSON (a Ruby backtrace, a GEMMA progress bar) pass through +unchanged unless --quiet; that is deliberate, because the process log +interleaves them and dropping them would hide the interesting failures. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import sqlite3 +import sys +import time +from collections import Counter +from pathlib import Path + +DATE = re.compile(r"^(?P<day>\d{4}-\d{2}-\d{2})") +TIME = re.compile(r"(?P<time>\d{2}:\d{2}:\d{2})") + + +def registry_path(explicit: str | None) -> Path: + if explicit: + return Path(explicit) + return Path(os.environ.get("GN4_API_TOKENS_DB", "tokens.sqlite")) + + +class Registry: + """Read-only view of the token registry, for translating handles.""" + + def __init__(self, path: Path): + self.path = path + self.connection = None + self.error = None + if not path.exists(): + self.error = f"no registry at {path} (set GN4_API_TOKENS_DB or --db)" + return + try: + # Read-only, and never creating: this must not touch a live + # database's write path. + uri = f"file:{path}?mode=ro" + self.connection = sqlite3.connect(uri, uri=True, timeout=5) + self.connection.row_factory = sqlite3.Row + except sqlite3.Error as error: # pragma: no cover - environment + self.error = f"cannot read {path}: {error}" + + def by_id(self, identifier: str) -> sqlite3.Row | None: + if self.connection is None: + return None + try: + return self.connection.execute( + "SELECT id, subject, scope, prefix, created_at, expires_at," + " last_used_at, revoked_at, note FROM token WHERE id = ?", + (identifier,), + ).fetchone() + except sqlite3.Error: + return None + + def status(self, row: sqlite3.Row | None, at: int | None = None) -> str: + if row is None: + return "unknown" + if row["revoked_at"] is not None: + return "revoked" + expires = row["expires_at"] + if expires is not None and expires <= (at or int(time.time())): + return "expired" + return "active" + + +def read_lines(handle) -> list[str]: + return [line.rstrip("\n") for line in handle] + + +def parse(line: str) -> dict | None: + if '"event":"request"' not in line and '"event": "request"' not in line: + return None + start = line.find("{") + if start < 0: + return None + try: + record = json.loads(line[start:]) + except json.JSONDecodeError: + return None + return record if record.get("event") == "request" else None + + +def when_of(record: dict, line: str) -> str: + """The record's own `ts` wins; a journald/nginx prefix is the fallback.""" + stamp = record.get("ts") + if isinstance(stamp, str) and len(stamp) >= 19: + return f"{stamp[:10]} {stamp[11:19]}" + return timestamp_of(line) + + +def timestamp_of(line: str) -> str: + day = DATE.search(line) + clock = TIME.search(line) + if day and clock: + return f"{day.group('day')} {clock.group('time')}" + return "" + + +def enrich(record: dict, registry: Registry, when: int | None) -> dict: + out = dict(record) + handle = record.get("token") + if handle: + row = registry.by_id(handle) + if row is None: + out["token_status"] = "unknown" + else: + out["token_status"] = registry.status(row, when) + out["token_subject"] = row["subject"] + out["token_scope"] = row["scope"] + if row["note"]: + out["token_note"] = row["note"] + if row["expires_at"]: + out["token_expires_at"] = row["expires_at"] + # A row whose subject differs from the logged one is worth + # seeing: the registry is authoritative, and this is the only + # way an operator notices a token was repointed. + if record.get("user") and row["subject"] != record["user"]: + out["token_subject_changed"] = True + out["user"] = row["subject"] + return out + + +def format_line(record: dict, when: str) -> str: + who = record.get("user") or "-" + credential = record.get("scheme") or "-" + if record.get("token"): + credential += f"/{record['token']}" + status = record.get("token_status") + if status and status != "active": + credential += f" [{status}]" + if record.get("token_subject_changed"): + credential += " [subject changed]" + took = record.get("took_ms", 0) + return ( + f"{when or '-':<19} {who:<20} {credential:<34} " + f"{record.get('method', '-'):<6} {record.get('status', '-'):>4} " + f"{took:>6} ms {record.get('path', '')}" + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("files", nargs="*", help="log files (default: stdin)") + parser.add_argument("-f", "--file", action="append", default=[], help="a log file") + parser.add_argument("--db", default=None, help="the token registry (default GN4_API_TOKENS_DB, else ./tokens.sqlite)") + parser.add_argument("--since", default=None, help="only lines whose date prefix is at or after this (YYYY-MM-DD)") + parser.add_argument("--grep", default=None, help="only lines containing this text (matches the raw line)") + parser.add_argument("--json", action="store_true", help="emit enriched JSON instead of a table") + parser.add_argument("--summary", action="store_true", help="count requests per caller and credential") + parser.add_argument("--quiet", action="store_true", help="drop lines that are not request records") + args = parser.parse_args() + + registry = Registry(registry_path(args.db)) + if registry.error and (args.json or args.summary): + print(f"gn4-log: {registry.error}", file=sys.stderr) + + files = args.files + args.file + handles = [open(path) for path in files] or [sys.stdin] + try: + lines = [line for handle in handles for line in read_lines(handle)] + finally: + for handle in handles: + if handle is not sys.stdin: + handle.close() + + rows: list[tuple[str, dict]] = [] + for line in lines: + if args.grep and args.grep not in line: + continue + record = parse(line) + when = when_of(record, line) if record else timestamp_of(line) + if args.since and when: + # Compare like with like: the table shows "YYYY-MM-DD HH:MM:SS", + # and --since accepts a bare date or a date and a time. + stamp = when.replace(" ", "T") + if stamp < args.since: + continue + if record is None: + # Non-JSON lines (a Ruby backtrace, a GEMMA progress bar) pass + # through in table mode -- the process log interleaves them, and + # dropping them would hide the interesting failures -- but never + # in a machine-readable mode. + if not args.quiet and not args.json and not args.summary: + print(line) + continue + rows.append((when, enrich(record, registry, None))) + + if args.summary: + callers = Counter( + (record.get("user") or "-", record.get("scheme") or "-", record.get("token") or "-") + for _, record in rows + ) + width = max((len(" ".join(key)) for key in callers), default=10) + for (user, scheme, handle), count in callers.most_common(): + credential = scheme if handle == "-" else f"{scheme}/{handle}" + label = f"{user} {credential}" + print(f"{count:>7} {label:<{width}}") + print(f"{len(rows):>7} {'total':<{width}}") + return 0 + + for when, record in rows: + if args.json: + print(json.dumps(record)) + else: + print(format_line(record, when)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) |
