diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-16 11:04:35 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-16 11:04:35 +0000 |
| commit | 9ff458ee52079a2385b231f0b7b629170ecfc6a1 (patch) | |
| tree | a74229aaa15a1bd9ecee63d4c8c374f6c4d1d293 /scripts | |
| parent | 71878e9ff616b8a74ff21eb3a54203069b7a8e23 (diff) | |
| download | genenetwork4-9ff458ee52079a2385b231f0b7b629170ecfc6a1.tar.gz | |
gemma-compare: make the parity gate run, and say what it is measuring
The script could not have caught anything: it posted without an Authorization
header (401 against a token-protected server), asked for a trait's values as
`{"name": n, "values": [<trait name>]}` (422: samples need numbers), read
`columns`/`rows` from the envelope's top level (they are under `meta` and
`data`), and compared relatively -- which calls beta 0.0131 against 0.0157 a
16% disagreement on a marker where both runs agree to three decimals.
Now: `--token`/`$GN4_TOKEN`, trait values fetched from the API, the envelope
shape handled, and an ABSOLUTE `--tolerance` (default 0 = exact). It fetches,
compares and reports through the server as deployed.
Verdict on the current deployment, BXDPublish 10001 over 34 individuals:
rows: gemma-wrapper 21055, ravanan+gemma 21055, shared 21055
worst absolute difference: rs230640887 beta -1.015599 vs -1.018739 (0.00314)
--tolerance 0.005 -> agree; --tolerance 0 -> 63 163 values beyond it
So the row difference this started from is gone: same markers, same count. The
residual is that the two paths run *different GEMMA builds* -- the wrapper's
log names gemma-gn2-0.98.5-8cd4cdb, the ravanan manifest gemma 0.98.6 -- which
moves beta/se/p_lrt in the third decimal while leaving af, n_miss, l_mle and
the marker set identical (and the top hits in the same order).
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/gemma-compare.py | 109 |
1 files changed, 85 insertions, 24 deletions
diff --git a/scripts/gemma-compare.py b/scripts/gemma-compare.py index bc0d046..1c6dd5d 100755 --- a/scripts/gemma-compare.py +++ b/scripts/gemma-compare.py @@ -28,20 +28,26 @@ from __future__ import annotations import argparse import json +import os import sys import time import urllib.error +import urllib.parse import urllib.request COMPARED = ("beta", "se", "l_mle", "p_lrt") -def post(base: str, path: str, body: dict, timeout: float) -> tuple[dict, float]: +def post(base: str, path: str, body: dict, timeout: float, + token: str | None = None) -> tuple[dict, float]: """POST BODY to PATH, returning the decoded reply and the wall-clock time.""" + headers = {"content-type": "application/json"} + if token: + headers["authorization"] = "Bearer " + token request = urllib.request.Request( base.rstrip("/") + path, data=json.dumps(body).encode(), - headers={"content-type": "application/json"}, + headers=headers, ) started = time.monotonic() try: @@ -53,15 +59,40 @@ def post(base: str, path: str, body: dict, timeout: float) -> tuple[dict, float] return payload, time.monotonic() - started +def fetch_trait_values(base: str, group: str, trait: str, timeout: float, + token: str | None) -> dict[str, float]: + """The trait's value in each of the group's individuals, via the API.""" + path = f"/groups/{urllib.parse.quote(group)}/traits/{urllib.parse.quote(trait)}/values" + headers = {} + if token: + headers["authorization"] = "Bearer " + token + request = urllib.request.Request(base.rstrip("/") + path, headers=headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.loads(response.read().decode()) + except urllib.error.HTTPError as error: + detail = error.read().decode(errors="replace") + raise SystemExit(f"{path}: HTTP {error.code}: {detail}") from error + rows = payload.get("data") or payload.get("rows") or [] + return { + row["name"]: row["value"] + for row in rows + if isinstance(row, dict) and isinstance(row.get("value"), (int, float)) + } + + def rows_by_rs(payload: dict) -> dict[str, tuple[str, ...]]: """Key the reply's rows by rs, keeping the columns this comparison cares about.""" - columns = payload.get("columns") or [] + # The envelope carries the column names in `meta`; an older shape had them + # at the top level, and `data` versus `rows` varies the same way. + meta = payload.get("meta") or {} + columns = payload.get("columns") or meta.get("columns") or [] index = {name: i for i, name in enumerate(columns)} missing = [name for name in ("rs", *COMPARED) if name not in index] if missing: raise SystemExit(f"reply is missing column(s) {missing}; has {columns}") rows: dict[str, tuple[str, ...]] = {} - for row in payload.get("rows") or []: + for row in payload.get("data") or payload.get("rows") or []: rows[row[index["rs"]]] = tuple(row[index[name]] for name in COMPARED) return rows @@ -73,12 +104,14 @@ def as_float(value: str) -> float | None: return None -def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]]) -> int: +def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]], + tolerance: float = 0.0) -> int: """Print the differences; return how many rows disagreed.""" only_left = sorted(set(left) - set(right)) only_right = sorted(set(right) - set(left)) shared = sorted(set(left) & set(right)) disagreed = [] + beyond = 0 worst = (0.0, "") for rs in shared: a, b = left[rs], right[rs] @@ -92,12 +125,13 @@ def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]]) worst = (float("inf"), f"{rs} {name}: {x} vs {y}") continue delta = abs(fx - fy) - # Relative for anything not near zero, absolute otherwise: p-values - # differ in the 12th decimal and that is not a disagreement. - scale = max(abs(fx), abs(fy)) - relative = delta / scale if scale > 1e-300 else delta - if relative > worst[0]: - worst = (relative, f"{rs} {name}: {x} vs {y}") + # Absolute, not relative: the difference between two GEMMA builds + # is a fixed numerical offset, and beta can be near zero, where a + # relative measure calls 0.013 against 0.016 a 16% disagreement. + if delta > worst[0]: + worst = (delta, f"{rs} {name}: {x} vs {y}") + if delta > tolerance: + beyond += 1 print(f"rows: gemma-wrapper {len(left)}, ravanan+gemma {len(right)}, shared {len(shared)}") if only_left: @@ -108,8 +142,14 @@ def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]]) print(f"disagreeing rows ({len(disagreed)}):") for rs, a, b in disagreed[:10]: print(f" {rs}: wrapper {a} vs ravanan {b}") - print(f"worst relative difference: {worst[1]} ({worst[0]:.3g})") - return len(only_left) + len(only_right) + len(disagreed) + print(f"worst absolute difference: {worst[1]} ({worst[0]:.3g})") + print(f"values beyond the absolute tolerance {tolerance:g}: {beyond}") + # Missing rows are always a failure: the two stacks are answering the same + # question, so a marker in one and not the other is a bug in one of them. + # Value differences are a *tolerance*: today the wrapper path runs + # gemma-gn2-0.98.5 and the ravanan path gemma 0.98.6, which moves beta in + # the third decimal while leaving af, n_miss and the marker set identical. + return len(only_left) + len(only_right) + (1 if beyond else 0) def main() -> int: @@ -122,6 +162,13 @@ def main() -> int: parser.add_argument("--samples", help="comma-separated individual names, with --group") parser.add_argument("--lmm", type=int, default=9) parser.add_argument("--timeout", type=float, default=3600.0) + parser.add_argument("--token", default=os.environ.get("GN4_TOKEN"), + help="gn4ai_ token or JWT; defaults to $GN4_TOKEN") + parser.add_argument("--tolerance", type=float, default=0.0, + help="largest ABSOLUTE difference in beta/se/l_mle/p_lrt " + "that still counts as agreement (default 0: exact). " + "Two different GEMMA builds need 0.005; one build " + "should need nothing.") args = parser.parse_args() if args.body: @@ -130,23 +177,33 @@ def main() -> int: body = {"group": args.group, "samples": [], "lmm": args.lmm} if args.version: body["genotype_version"] = args.version + # The trait's values come from the server, not from the command line: + # `samples` is per individual and needs a *number* per name, which a + # trait name cannot stand in for (it used to be sent as + # {"name": n, "values": [trait]}, and every run answered 422). + values = fetch_trait_values(args.base, args.group, args.trait, args.timeout, args.token) + wanted = None if args.samples: - body["samples"] = [ - {"name": name, "values": [args.trait]} for name in args.samples.split(",") - ] - else: - # Named samples are per individual; without names the server reads - # trait_values positionally, which needs the panel order. - raise SystemExit("--group needs --samples (comma-separated individuals)") + wanted = {name.strip() for name in args.samples.split(",")} + body["samples"] = [ + {"name": name, "value": value} + for name, value in values.items() + if wanted is None or name in wanted + ] + if not body["samples"]: + raise SystemExit( + f"none of the trait's individuals {'/'.join(sorted(wanted or []))} " + "carry a value in " + args.group) + print(f"{args.trait}: {len(body['samples'])} individual(s) with a value") else: parser.error("give --body, or --group with --trait and --samples") print(f"POST /compute/gemma (gemma-wrapper) ...", flush=True) - wrapper, wrapper_seconds = post(args.base, "/compute/gemma", body, args.timeout) + wrapper, wrapper_seconds = post(args.base, "/compute/gemma", body, args.timeout, args.token) print(f" {wrapper_seconds:.2f} s, {len(wrapper.get('rows') or [])} rows") print(f"POST /compute/gemma/ravanan (ravanan + gemma) ...", flush=True) - ravanan, ravanan_seconds = post(args.base, "/compute/gemma/ravanan", body, args.timeout) + ravanan, ravanan_seconds = post(args.base, "/compute/gemma/ravanan", body, args.timeout, args.token) print(f" {ravanan_seconds:.2f} s, {len(ravanan.get('rows') or [])} rows") print() @@ -161,11 +218,15 @@ def main() -> int: print(f" {label}: {note}") print() - disagreements = compare(rows_by_rs(wrapper), rows_by_rs(ravanan)) + disagreements = compare(rows_by_rs(wrapper), rows_by_rs(ravanan), args.tolerance) if disagreements: print(f"\nDISAGREEMENT: {disagreements} row(s)") return 1 - print("\nidentical on " + ", ".join(("rs", *COMPARED))) + if args.tolerance: + print("\nagree on the marker set, and within " + f"{args.tolerance:g} absolute on " + + ", ".join(COMPARED) + " (not identical -- raise the bar with --tolerance 0)") + else: + print("\nidentical on " + ", ".join(("rs", *COMPARED))) return 0 |
