about summary refs log tree commit diff
path: root/scripts
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-11-16 11:04:35 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-11-16 11:04:35 +0000
commit9ff458ee52079a2385b231f0b7b629170ecfc6a1 (patch)
treea74229aaa15a1bd9ecee63d4c8c374f6c4d1d293 /scripts
parent71878e9ff616b8a74ff21eb3a54203069b7a8e23 (diff)
downloadgenenetwork4-9ff458ee52079a2385b231f0b7b629170ecfc6a1.tar.gz
gemma-compare: make the parity gate run, and say what it is measuring
The script could not have caught anything: it posted without an Authorization
header (401 against a token-protected server), asked for a trait's values as
`{"name": n, "values": [<trait name>]}` (422: samples need numbers), read
`columns`/`rows` from the envelope's top level (they are under `meta` and
`data`), and compared relatively -- which calls beta 0.0131 against 0.0157 a
16% disagreement on a marker where both runs agree to three decimals.

Now: `--token`/`$GN4_TOKEN`, trait values fetched from the API, the envelope
shape handled, and an ABSOLUTE `--tolerance` (default 0 = exact). It fetches,
compares and reports through the server as deployed.

Verdict on the current deployment, BXDPublish 10001 over 34 individuals:

  rows: gemma-wrapper 21055, ravanan+gemma 21055, shared 21055
  worst absolute difference: rs230640887 beta -1.015599 vs -1.018739 (0.00314)
  --tolerance 0.005 -> agree; --tolerance 0 -> 63 163 values beyond it

So the row difference this started from is gone: same markers, same count. The
residual is that the two paths run *different GEMMA builds* -- the wrapper's
log names gemma-gn2-0.98.5-8cd4cdb, the ravanan manifest gemma 0.98.6 -- which
moves beta/se/p_lrt in the third decimal while leaving af, n_miss, l_mle and
the marker set identical (and the top hits in the same order).
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/gemma-compare.py109
1 files changed, 85 insertions, 24 deletions
diff --git a/scripts/gemma-compare.py b/scripts/gemma-compare.py
index bc0d046..1c6dd5d 100755
--- a/scripts/gemma-compare.py
+++ b/scripts/gemma-compare.py
@@ -28,20 +28,26 @@ from __future__ import annotations
 
 import argparse
 import json
+import os
 import sys
 import time
 import urllib.error
+import urllib.parse
 import urllib.request
 
 COMPARED = ("beta", "se", "l_mle", "p_lrt")
 
 
-def post(base: str, path: str, body: dict, timeout: float) -> tuple[dict, float]:
+def post(base: str, path: str, body: dict, timeout: float,
+         token: str | None = None) -> tuple[dict, float]:
     """POST BODY to PATH, returning the decoded reply and the wall-clock time."""
+    headers = {"content-type": "application/json"}
+    if token:
+        headers["authorization"] = "Bearer " + token
     request = urllib.request.Request(
         base.rstrip("/") + path,
         data=json.dumps(body).encode(),
-        headers={"content-type": "application/json"},
+        headers=headers,
     )
     started = time.monotonic()
     try:
@@ -53,15 +59,40 @@ def post(base: str, path: str, body: dict, timeout: float) -> tuple[dict, float]
     return payload, time.monotonic() - started
 
 
+def fetch_trait_values(base: str, group: str, trait: str, timeout: float,
+                       token: str | None) -> dict[str, float]:
+    """The trait's value in each of the group's individuals, via the API."""
+    path = f"/groups/{urllib.parse.quote(group)}/traits/{urllib.parse.quote(trait)}/values"
+    headers = {}
+    if token:
+        headers["authorization"] = "Bearer " + token
+    request = urllib.request.Request(base.rstrip("/") + path, headers=headers)
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            payload = json.loads(response.read().decode())
+    except urllib.error.HTTPError as error:
+        detail = error.read().decode(errors="replace")
+        raise SystemExit(f"{path}: HTTP {error.code}: {detail}") from error
+    rows = payload.get("data") or payload.get("rows") or []
+    return {
+        row["name"]: row["value"]
+        for row in rows
+        if isinstance(row, dict) and isinstance(row.get("value"), (int, float))
+    }
+
+
 def rows_by_rs(payload: dict) -> dict[str, tuple[str, ...]]:
     """Key the reply's rows by rs, keeping the columns this comparison cares about."""
-    columns = payload.get("columns") or []
+    # The envelope carries the column names in `meta`; an older shape had them
+    # at the top level, and `data` versus `rows` varies the same way.
+    meta = payload.get("meta") or {}
+    columns = payload.get("columns") or meta.get("columns") or []
     index = {name: i for i, name in enumerate(columns)}
     missing = [name for name in ("rs", *COMPARED) if name not in index]
     if missing:
         raise SystemExit(f"reply is missing column(s) {missing}; has {columns}")
     rows: dict[str, tuple[str, ...]] = {}
-    for row in payload.get("rows") or []:
+    for row in payload.get("data") or payload.get("rows") or []:
         rows[row[index["rs"]]] = tuple(row[index[name]] for name in COMPARED)
     return rows
 
@@ -73,12 +104,14 @@ def as_float(value: str) -> float | None:
         return None
 
 
-def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]]) -> int:
+def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]],
+            tolerance: float = 0.0) -> int:
     """Print the differences; return how many rows disagreed."""
     only_left = sorted(set(left) - set(right))
     only_right = sorted(set(right) - set(left))
     shared = sorted(set(left) & set(right))
     disagreed = []
+    beyond = 0
     worst = (0.0, "")
     for rs in shared:
         a, b = left[rs], right[rs]
@@ -92,12 +125,13 @@ def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]])
                     worst = (float("inf"), f"{rs} {name}: {x} vs {y}")
                 continue
             delta = abs(fx - fy)
-            # Relative for anything not near zero, absolute otherwise: p-values
-            # differ in the 12th decimal and that is not a disagreement.
-            scale = max(abs(fx), abs(fy))
-            relative = delta / scale if scale > 1e-300 else delta
-            if relative > worst[0]:
-                worst = (relative, f"{rs} {name}: {x} vs {y}")
+            # Absolute, not relative: the difference between two GEMMA builds
+            # is a fixed numerical offset, and beta can be near zero, where a
+            # relative measure calls 0.013 against 0.016 a 16% disagreement.
+            if delta > worst[0]:
+                worst = (delta, f"{rs} {name}: {x} vs {y}")
+            if delta > tolerance:
+                beyond += 1
 
     print(f"rows: gemma-wrapper {len(left)}, ravanan+gemma {len(right)}, shared {len(shared)}")
     if only_left:
@@ -108,8 +142,14 @@ def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]])
         print(f"disagreeing rows ({len(disagreed)}):")
         for rs, a, b in disagreed[:10]:
             print(f"  {rs}: wrapper {a} vs ravanan {b}")
-        print(f"worst relative difference: {worst[1]} ({worst[0]:.3g})")
-    return len(only_left) + len(only_right) + len(disagreed)
+        print(f"worst absolute difference: {worst[1]} ({worst[0]:.3g})")
+        print(f"values beyond the absolute tolerance {tolerance:g}: {beyond}")
+    # Missing rows are always a failure: the two stacks are answering the same
+    # question, so a marker in one and not the other is a bug in one of them.
+    # Value differences are a *tolerance*: today the wrapper path runs
+    # gemma-gn2-0.98.5 and the ravanan path gemma 0.98.6, which moves beta in
+    # the third decimal while leaving af, n_miss and the marker set identical.
+    return len(only_left) + len(only_right) + (1 if beyond else 0)
 
 
 def main() -> int:
@@ -122,6 +162,13 @@ def main() -> int:
     parser.add_argument("--samples", help="comma-separated individual names, with --group")
     parser.add_argument("--lmm", type=int, default=9)
     parser.add_argument("--timeout", type=float, default=3600.0)
+    parser.add_argument("--token", default=os.environ.get("GN4_TOKEN"),
+                        help="gn4ai_ token or JWT; defaults to $GN4_TOKEN")
+    parser.add_argument("--tolerance", type=float, default=0.0,
+                        help="largest ABSOLUTE difference in beta/se/l_mle/p_lrt "
+                             "that still counts as agreement (default 0: exact). "
+                             "Two different GEMMA builds need 0.005; one build "
+                             "should need nothing.")
     args = parser.parse_args()
 
     if args.body:
@@ -130,23 +177,33 @@ def main() -> int:
         body = {"group": args.group, "samples": [], "lmm": args.lmm}
         if args.version:
             body["genotype_version"] = args.version
+        # The trait's values come from the server, not from the command line:
+        # `samples` is per individual and needs a *number* per name, which a
+        # trait name cannot stand in for (it used to be sent as
+        # {"name": n, "values": [trait]}, and every run answered 422).
+        values = fetch_trait_values(args.base, args.group, args.trait, args.timeout, args.token)
+        wanted = None
         if args.samples:
-            body["samples"] = [
-                {"name": name, "values": [args.trait]} for name in args.samples.split(",")
-            ]
-        else:
-            # Named samples are per individual; without names the server reads
-            # trait_values positionally, which needs the panel order.
-            raise SystemExit("--group needs --samples (comma-separated individuals)")
+            wanted = {name.strip() for name in args.samples.split(",")}
+        body["samples"] = [
+            {"name": name, "value": value}
+            for name, value in values.items()
+            if wanted is None or name in wanted
+        ]
+        if not body["samples"]:
+            raise SystemExit(
+                f"none of the trait's individuals {'/'.join(sorted(wanted or []))} "
+                "carry a value in " + args.group)
+        print(f"{args.trait}: {len(body['samples'])} individual(s) with a value")
     else:
         parser.error("give --body, or --group with --trait and --samples")
 
     print(f"POST /compute/gemma         (gemma-wrapper) ...", flush=True)
-    wrapper, wrapper_seconds = post(args.base, "/compute/gemma", body, args.timeout)
+    wrapper, wrapper_seconds = post(args.base, "/compute/gemma", body, args.timeout, args.token)
     print(f"  {wrapper_seconds:.2f} s, {len(wrapper.get('rows') or [])} rows")
 
     print(f"POST /compute/gemma/ravanan (ravanan + gemma) ...", flush=True)
-    ravanan, ravanan_seconds = post(args.base, "/compute/gemma/ravanan", body, args.timeout)
+    ravanan, ravanan_seconds = post(args.base, "/compute/gemma/ravanan", body, args.timeout, args.token)
     print(f"  {ravanan_seconds:.2f} s, {len(ravanan.get('rows') or [])} rows")
 
     print()
@@ -161,11 +218,15 @@ def main() -> int:
                 print(f"  {label}: {note}")
 
     print()
-    disagreements = compare(rows_by_rs(wrapper), rows_by_rs(ravanan))
+    disagreements = compare(rows_by_rs(wrapper), rows_by_rs(ravanan), args.tolerance)
     if disagreements:
         print(f"\nDISAGREEMENT: {disagreements} row(s)")
         return 1
-    print("\nidentical on " + ", ".join(("rs", *COMPARED)))
+    if args.tolerance:
+        print("\nagree on the marker set, and within " + f"{args.tolerance:g} absolute on "
+              + ", ".join(COMPARED) + " (not identical -- raise the bar with --tolerance 0)")
+    else:
+        print("\nidentical on " + ", ".join(("rs", *COMPARED)))
     return 0