diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-14 04:14:07 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-14 04:14:07 +0000 |
| commit | 0e017416628d72ae9814b8225d17f62b28f855ee (patch) | |
| tree | a073e1f8dd009b548cab0cb0bfc95904c3b3d77f /scripts | |
| parent | f8b34ca82b40f9eda65fa3abab580a0ca8e66f49 (diff) | |
| download | genenetwork4-0e017416628d72ae9814b8225d17f62b28f855ee.tar.gz | |
ravanan+gemma: call gemma itself, not gemma-wrapper
gemma-wrapper was doing two things ravanan now does or GEMMA already does: the parallel fan-out (ravanan's CWL scatter replaced it long ago, which is why every job passed --no-parallel) and LOCO orchestration -- gemma 0.98.6 documents `-loco [chr]` natively for both -gk and -lmm and takes the annotation file it needs. What was left was a content-hash K cache and a --json record with results packed into .tar.xz archives. So cwl/gemma/gemma-loco.scm now runs two gemma invocations per chromosome: `-gk -loco <chr> -o <chr>-kin`, then `-k <that matrix> -lmm <n> -loco <chr> -pve -o <chr>-gwa`, with `-outdir .` for flat, deterministic names. The kinship matrix travels from one step to the other as a CWL File output, and each job's .assoc.txt and .log.txt are the workflow's outputs -- so compute_ravanan reads files instead of parsing a wrapper JSON record and unpacking archives, and the pve note comes straight out of the jobs' logs. Measured on tests/test_data/BXD (chr 1 and chr 2): identical rows -- 636/636 and 583/583 on beta, se, l_mle, p_lrt. One chromosome end to end: 496 ms (gemma) vs 1140 ms (wrapper), 2.3x per job. The wrapper also needed Ruby and pfff on PATH, and named its outputs with the extension doubled (...1.assoc.txt.assoc.txt). /compute/gemma keeps gemma-wrapper, deliberately: that is what makes the two endpoints comparable rather than two copies of one stack. scripts/gemma-compare.py runs the same body through both and reports row-level differences and timings. The gemma-wrapper content-hash K cache is the one thing given up; a repeat request recomputes (271 ms of kinship on this fixture).
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/gemma-compare.py | 173 |
1 files changed, 173 insertions, 0 deletions
diff --git a/scripts/gemma-compare.py b/scripts/gemma-compare.py new file mode 100755 index 0000000..bc0d046 --- /dev/null +++ b/scripts/gemma-compare.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Compare the two GEMMA endpoints, row for row and second for second. + +`/compute/gemma` runs gemma-wrapper (kinship, then GWA, both --loco) through +this server's own subprocess calls; `/compute/gemma/ravanan` runs `gemma` +itself, one invocation per chromosome, through a CWL workflow scattered by +ravanan. Same request body, same fixtures -- so the differences this prints are +the differences between two stacks doing one computation, which is exactly what +those two endpoints exist for. + + scripts/gemma-compare.py --base http://127.0.0.1:3000 \ + --body '{"genotype_file": "/path/BXD_geno.txt.gz", + "snps_file": "/path/BXD_snps.txt", + "trait_values": [...], "lmm": 9}' + + # or name a group and a trait in that group's sample list + scripts/gemma-compare.py --base http://127.0.0.1:3000 \ + --group BXD --version v8 --trait 10001 + +What is compared: every row's `rs`, `beta`, `se`, `l_mle` and `p_lrt`, keyed by +`rs` (row order is not part of the contract), and the wall-clock time of each +call. A non-zero exit status means the two disagreed -- so this is usable as a +release gate, not just a report. + +Requires a running server (see the README's "Running"). It does not start one. +""" +from __future__ import annotations + +import argparse +import json +import sys +import time +import urllib.error +import urllib.request + +COMPARED = ("beta", "se", "l_mle", "p_lrt") + + +def post(base: str, path: str, body: dict, timeout: float) -> tuple[dict, float]: + """POST BODY to PATH, returning the decoded reply and the wall-clock time.""" + request = urllib.request.Request( + base.rstrip("/") + path, + data=json.dumps(body).encode(), + headers={"content-type": "application/json"}, + ) + started = time.monotonic() + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.loads(response.read().decode()) + except urllib.error.HTTPError as error: + detail = error.read().decode(errors="replace") + raise SystemExit(f"{path}: HTTP {error.code}: {detail}") from error + return payload, time.monotonic() - started + + +def rows_by_rs(payload: dict) -> dict[str, tuple[str, ...]]: + """Key the reply's rows by rs, keeping the columns this comparison cares about.""" + columns = payload.get("columns") or [] + index = {name: i for i, name in enumerate(columns)} + missing = [name for name in ("rs", *COMPARED) if name not in index] + if missing: + raise SystemExit(f"reply is missing column(s) {missing}; has {columns}") + rows: dict[str, tuple[str, ...]] = {} + for row in payload.get("rows") or []: + rows[row[index["rs"]]] = tuple(row[index[name]] for name in COMPARED) + return rows + + +def as_float(value: str) -> float | None: + try: + return float(value) + except (TypeError, ValueError): + return None + + +def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]]) -> int: + """Print the differences; return how many rows disagreed.""" + only_left = sorted(set(left) - set(right)) + only_right = sorted(set(right) - set(left)) + shared = sorted(set(left) & set(right)) + disagreed = [] + worst = (0.0, "") + for rs in shared: + a, b = left[rs], right[rs] + if a == b: + continue + disagreed.append((rs, a, b)) + for name, x, y in zip(COMPARED, a, b): + fx, fy = as_float(x), as_float(y) + if fx is None or fy is None: + if x != y: + worst = (float("inf"), f"{rs} {name}: {x} vs {y}") + continue + delta = abs(fx - fy) + # Relative for anything not near zero, absolute otherwise: p-values + # differ in the 12th decimal and that is not a disagreement. + scale = max(abs(fx), abs(fy)) + relative = delta / scale if scale > 1e-300 else delta + if relative > worst[0]: + worst = (relative, f"{rs} {name}: {x} vs {y}") + + print(f"rows: gemma-wrapper {len(left)}, ravanan+gemma {len(right)}, shared {len(shared)}") + if only_left: + print(f"only in gemma-wrapper ({len(only_left)}): {only_left[:5]}") + if only_right: + print(f"only in ravanan+gemma ({len(only_right)}): {only_right[:5]}") + if disagreed: + print(f"disagreeing rows ({len(disagreed)}):") + for rs, a, b in disagreed[:10]: + print(f" {rs}: wrapper {a} vs ravanan {b}") + print(f"worst relative difference: {worst[1]} ({worst[0]:.3g})") + return len(only_left) + len(only_right) + len(disagreed) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--base", default="http://127.0.0.1:3000", help="server base URL") + parser.add_argument("--body", help="JSON request body sent to both endpoints") + parser.add_argument("--group", help="group name (instead of --body)") + parser.add_argument("--version", help="genotype version for --group") + parser.add_argument("--trait", help="trait name to take the values of, for --group") + parser.add_argument("--samples", help="comma-separated individual names, with --group") + parser.add_argument("--lmm", type=int, default=9) + parser.add_argument("--timeout", type=float, default=3600.0) + args = parser.parse_args() + + if args.body: + body = json.loads(args.body) + elif args.group and args.trait: + body = {"group": args.group, "samples": [], "lmm": args.lmm} + if args.version: + body["genotype_version"] = args.version + if args.samples: + body["samples"] = [ + {"name": name, "values": [args.trait]} for name in args.samples.split(",") + ] + else: + # Named samples are per individual; without names the server reads + # trait_values positionally, which needs the panel order. + raise SystemExit("--group needs --samples (comma-separated individuals)") + else: + parser.error("give --body, or --group with --trait and --samples") + + print(f"POST /compute/gemma (gemma-wrapper) ...", flush=True) + wrapper, wrapper_seconds = post(args.base, "/compute/gemma", body, args.timeout) + print(f" {wrapper_seconds:.2f} s, {len(wrapper.get('rows') or [])} rows") + + print(f"POST /compute/gemma/ravanan (ravanan + gemma) ...", flush=True) + ravanan, ravanan_seconds = post(args.base, "/compute/gemma/ravanan", body, args.timeout) + print(f" {ravanan_seconds:.2f} s, {len(ravanan.get('rows') or [])} rows") + + print() + print(f"timing: gemma-wrapper {wrapper_seconds:.2f} s, ravanan+gemma {ravanan_seconds:.2f} s " + f"({ravanan_seconds / wrapper_seconds:.2f}x)") + + # Notes carry the exact commands each runner used, which is what makes a + # disagreement diagnosable rather than mysterious. + for label, payload in (("gemma-wrapper", wrapper), ("ravanan+gemma", ravanan)): + for note in payload.get("notes") or []: + if note.lower().startswith(("kinship", "gwa", "ravanan", "scattered")): + print(f" {label}: {note}") + + print() + disagreements = compare(rows_by_rs(wrapper), rows_by_rs(ravanan)) + if disagreements: + print(f"\nDISAGREEMENT: {disagreements} row(s)") + return 1 + print("\nidentical on " + ", ".join(("rs", *COMPARED))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) |
