#!/usr/bin/env python3 """Compare the two GEMMA endpoints, row for row and second for second. `/compute/gemma` runs gemma-wrapper (kinship, then GWA, both --loco) through this server's own subprocess calls; `/compute/gemma/ravanan` runs `gemma` itself, one invocation per chromosome, through a CWL workflow scattered by ravanan. Same request body, same fixtures -- so the differences this prints are the differences between two stacks doing one computation, which is exactly what those two endpoints exist for. scripts/gemma-compare.py --base http://127.0.0.1:3000 \ --body '{"genotype_file": "/path/BXD_geno.txt.gz", "snps_file": "/path/BXD_snps.txt", "trait_values": [...], "lmm": 9}' # or name a group and a trait in that group's sample list scripts/gemma-compare.py --base http://127.0.0.1:3000 \ --group BXD --version v8 --trait 10001 What is compared: every row's `rs`, `beta`, `se`, `l_mle` and `p_lrt`, keyed by `rs` (row order is not part of the contract), and the wall-clock time of each call. A non-zero exit status means the two disagreed -- so this is usable as a release gate, not just a report. Requires a running server (see the README's "Running"). It does not start one. """ from __future__ import annotations import argparse import json import os import sys import time import urllib.error import urllib.parse import urllib.request COMPARED = ("beta", "se", "l_mle", "p_lrt") def post(base: str, path: str, body: dict, timeout: float, token: str | None = None) -> tuple[dict, float]: """POST BODY to PATH, returning the decoded reply and the wall-clock time.""" headers = {"content-type": "application/json"} if token: headers["authorization"] = "Bearer " + token request = urllib.request.Request( base.rstrip("/") + path, data=json.dumps(body).encode(), headers=headers, ) started = time.monotonic() try: with urllib.request.urlopen(request, timeout=timeout) as response: payload = json.loads(response.read().decode()) except urllib.error.HTTPError as error: detail = error.read().decode(errors="replace") raise SystemExit(f"{path}: HTTP {error.code}: {detail}") from error return payload, time.monotonic() - started def fetch_trait_values(base: str, group: str, trait: str, timeout: float, token: str | None) -> dict[str, float]: """The trait's value in each of the group's individuals, via the API.""" path = f"/groups/{urllib.parse.quote(group)}/traits/{urllib.parse.quote(trait)}/values" headers = {} if token: headers["authorization"] = "Bearer " + token request = urllib.request.Request(base.rstrip("/") + path, headers=headers) try: with urllib.request.urlopen(request, timeout=timeout) as response: payload = json.loads(response.read().decode()) except urllib.error.HTTPError as error: detail = error.read().decode(errors="replace") raise SystemExit(f"{path}: HTTP {error.code}: {detail}") from error rows = payload.get("data") or payload.get("rows") or [] return { row["name"]: row["value"] for row in rows if isinstance(row, dict) and isinstance(row.get("value"), (int, float)) } def rows_by_rs(payload: dict) -> dict[str, tuple[str, ...]]: """Key the reply's rows by rs, keeping the columns this comparison cares about.""" # The envelope carries the column names in `meta`; an older shape had them # at the top level, and `data` versus `rows` varies the same way. meta = payload.get("meta") or {} columns = payload.get("columns") or meta.get("columns") or [] index = {name: i for i, name in enumerate(columns)} missing = [name for name in ("rs", *COMPARED) if name not in index] if missing: raise SystemExit(f"reply is missing column(s) {missing}; has {columns}") rows: dict[str, tuple[str, ...]] = {} for row in payload.get("data") or payload.get("rows") or []: rows[row[index["rs"]]] = tuple(row[index[name]] for name in COMPARED) return rows def as_float(value: str) -> float | None: try: return float(value) except (TypeError, ValueError): return None def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]], tolerance: float = 0.0) -> int: """Print the differences; return how many rows disagreed.""" only_left = sorted(set(left) - set(right)) only_right = sorted(set(right) - set(left)) shared = sorted(set(left) & set(right)) disagreed = [] beyond = 0 worst = (0.0, "") for rs in shared: a, b = left[rs], right[rs] if a == b: continue disagreed.append((rs, a, b)) for name, x, y in zip(COMPARED, a, b): fx, fy = as_float(x), as_float(y) if fx is None or fy is None: if x != y: worst = (float("inf"), f"{rs} {name}: {x} vs {y}") continue delta = abs(fx - fy) # Absolute, not relative: the difference between two GEMMA builds # is a fixed numerical offset, and beta can be near zero, where a # relative measure calls 0.013 against 0.016 a 16% disagreement. if delta > worst[0]: worst = (delta, f"{rs} {name}: {x} vs {y}") if delta > tolerance: beyond += 1 print(f"rows: gemma-wrapper {len(left)}, ravanan+gemma {len(right)}, shared {len(shared)}") if only_left: print(f"only in gemma-wrapper ({len(only_left)}): {only_left[:5]}") if only_right: print(f"only in ravanan+gemma ({len(only_right)}): {only_right[:5]}") if disagreed: print(f"disagreeing rows ({len(disagreed)}):") for rs, a, b in disagreed[:10]: print(f" {rs}: wrapper {a} vs ravanan {b}") print(f"worst absolute difference: {worst[1]} ({worst[0]:.3g})") print(f"values beyond the absolute tolerance {tolerance:g}: {beyond}") # Missing rows are always a failure: the two stacks are answering the same # question, so a marker in one and not the other is a bug in one of them. # Value differences are a *tolerance*: today the wrapper path runs # gemma-gn2-0.98.5 and the ravanan path gemma 0.98.6, which moves beta in # the third decimal while leaving af, n_miss and the marker set identical. return len(only_left) + len(only_right) + (1 if beyond else 0) def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--base", default="http://127.0.0.1:3000", help="server base URL") parser.add_argument("--body", help="JSON request body sent to both endpoints") parser.add_argument("--group", help="group name (instead of --body)") parser.add_argument("--version", help="genotype version for --group") parser.add_argument("--trait", help="trait name to take the values of, for --group") parser.add_argument("--samples", help="comma-separated individual names, with --group") parser.add_argument("--lmm", type=int, default=9) parser.add_argument("--timeout", type=float, default=3600.0) parser.add_argument("--token", default=os.environ.get("GN4_TOKEN"), help="gn4ai_ token or JWT; defaults to $GN4_TOKEN") parser.add_argument("--tolerance", type=float, default=0.0, help="largest ABSOLUTE difference in beta/se/l_mle/p_lrt " "that still counts as agreement (default 0: exact). " "Two different GEMMA builds need 0.005; one build " "should need nothing.") args = parser.parse_args() if args.body: body = json.loads(args.body) elif args.group and args.trait: body = {"group": args.group, "samples": [], "lmm": args.lmm} if args.version: body["genotype_version"] = args.version # The trait's values come from the server, not from the command line: # `samples` is per individual and needs a *number* per name, which a # trait name cannot stand in for (it used to be sent as # {"name": n, "values": [trait]}, and every run answered 422). values = fetch_trait_values(args.base, args.group, args.trait, args.timeout, args.token) wanted = None if args.samples: wanted = {name.strip() for name in args.samples.split(",")} body["samples"] = [ {"name": name, "value": value} for name, value in values.items() if wanted is None or name in wanted ] if not body["samples"]: raise SystemExit( f"none of the trait's individuals {'/'.join(sorted(wanted or []))} " "carry a value in " + args.group) print(f"{args.trait}: {len(body['samples'])} individual(s) with a value") else: parser.error("give --body, or --group with --trait and --samples") print(f"POST /compute/gemma (gemma-wrapper) ...", flush=True) wrapper, wrapper_seconds = post(args.base, "/compute/gemma", body, args.timeout, args.token) print(f" {wrapper_seconds:.2f} s, {len(wrapper.get('rows') or [])} rows") print(f"POST /compute/gemma/ravanan (ravanan + gemma) ...", flush=True) ravanan, ravanan_seconds = post(args.base, "/compute/gemma/ravanan", body, args.timeout, args.token) print(f" {ravanan_seconds:.2f} s, {len(ravanan.get('rows') or [])} rows") print() print(f"timing: gemma-wrapper {wrapper_seconds:.2f} s, ravanan+gemma {ravanan_seconds:.2f} s " f"({ravanan_seconds / wrapper_seconds:.2f}x)") # Notes carry the exact commands each runner used, which is what makes a # disagreement diagnosable rather than mysterious. for label, payload in (("gemma-wrapper", wrapper), ("ravanan+gemma", ravanan)): for note in payload.get("notes") or []: if note.lower().startswith(("kinship", "gwa", "ravanan", "scattered")): print(f" {label}: {note}") print() disagreements = compare(rows_by_rs(wrapper), rows_by_rs(ravanan), args.tolerance) if disagreements: print(f"\nDISAGREEMENT: {disagreements} row(s)") return 1 if args.tolerance: print("\nagree on the marker set, and within " + f"{args.tolerance:g} absolute on " + ", ".join(COMPARED) + " (not identical -- raise the bar with --tolerance 0)") else: print("\nidentical on " + ", ".join(("rs", *COMPARED))) return 0 if __name__ == "__main__": sys.exit(main())