about summary refs log tree commit diff
path: root/scripts
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-11-14 04:14:07 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-11-14 04:14:07 +0000
commit0e017416628d72ae9814b8225d17f62b28f855ee (patch)
treea073e1f8dd009b548cab0cb0bfc95904c3b3d77f /scripts
parentf8b34ca82b40f9eda65fa3abab580a0ca8e66f49 (diff)
downloadgenenetwork4-0e017416628d72ae9814b8225d17f62b28f855ee.tar.gz
ravanan+gemma: call gemma itself, not gemma-wrapper
gemma-wrapper was doing two things ravanan now does or GEMMA already
does: the parallel fan-out (ravanan's CWL scatter replaced it long ago,
which is why every job passed --no-parallel) and LOCO orchestration --
gemma 0.98.6 documents `-loco [chr]` natively for both -gk and -lmm and
takes the annotation file it needs. What was left was a content-hash K
cache and a --json record with results packed into .tar.xz archives.

So cwl/gemma/gemma-loco.scm now runs two gemma invocations per
chromosome: `-gk -loco <chr> -o <chr>-kin`, then `-k <that matrix>
-lmm <n> -loco <chr> -pve -o <chr>-gwa`, with `-outdir .` for flat,
deterministic names. The kinship matrix travels from one step to the
other as a CWL File output, and each job's .assoc.txt and .log.txt are
the workflow's outputs -- so compute_ravanan reads files instead of
parsing a wrapper JSON record and unpacking archives, and the pve note
comes straight out of the jobs' logs.

Measured on tests/test_data/BXD (chr 1 and chr 2): identical rows --
636/636 and 583/583 on beta, se, l_mle, p_lrt. One chromosome end to
end: 496 ms (gemma) vs 1140 ms (wrapper), 2.3x per job. The wrapper
also needed Ruby and pfff on PATH, and named its outputs with the
extension doubled (...1.assoc.txt.assoc.txt).

/compute/gemma keeps gemma-wrapper, deliberately: that is what makes
the two endpoints comparable rather than two copies of one stack.
scripts/gemma-compare.py runs the same body through both and reports
row-level differences and timings.

The gemma-wrapper content-hash K cache is the one thing given up;
a repeat request recomputes (271 ms of kinship on this fixture).
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/gemma-compare.py173
1 files changed, 173 insertions, 0 deletions
diff --git a/scripts/gemma-compare.py b/scripts/gemma-compare.py
new file mode 100755
index 0000000..bc0d046
--- /dev/null
+++ b/scripts/gemma-compare.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Compare the two GEMMA endpoints, row for row and second for second.
+
+`/compute/gemma` runs gemma-wrapper (kinship, then GWA, both --loco) through
+this server's own subprocess calls; `/compute/gemma/ravanan` runs `gemma`
+itself, one invocation per chromosome, through a CWL workflow scattered by
+ravanan. Same request body, same fixtures -- so the differences this prints are
+the differences between two stacks doing one computation, which is exactly what
+those two endpoints exist for.
+
+    scripts/gemma-compare.py --base http://127.0.0.1:3000 \
+        --body '{"genotype_file": "/path/BXD_geno.txt.gz",
+                 "snps_file": "/path/BXD_snps.txt",
+                 "trait_values": [...], "lmm": 9}'
+
+    # or name a group and a trait in that group's sample list
+    scripts/gemma-compare.py --base http://127.0.0.1:3000 \
+        --group BXD --version v8 --trait 10001
+
+What is compared: every row's `rs`, `beta`, `se`, `l_mle` and `p_lrt`, keyed by
+`rs` (row order is not part of the contract), and the wall-clock time of each
+call. A non-zero exit status means the two disagreed -- so this is usable as a
+release gate, not just a report.
+
+Requires a running server (see the README's "Running"). It does not start one.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+import urllib.error
+import urllib.request
+
+COMPARED = ("beta", "se", "l_mle", "p_lrt")
+
+
+def post(base: str, path: str, body: dict, timeout: float) -> tuple[dict, float]:
+    """POST BODY to PATH, returning the decoded reply and the wall-clock time."""
+    request = urllib.request.Request(
+        base.rstrip("/") + path,
+        data=json.dumps(body).encode(),
+        headers={"content-type": "application/json"},
+    )
+    started = time.monotonic()
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            payload = json.loads(response.read().decode())
+    except urllib.error.HTTPError as error:
+        detail = error.read().decode(errors="replace")
+        raise SystemExit(f"{path}: HTTP {error.code}: {detail}") from error
+    return payload, time.monotonic() - started
+
+
+def rows_by_rs(payload: dict) -> dict[str, tuple[str, ...]]:
+    """Key the reply's rows by rs, keeping the columns this comparison cares about."""
+    columns = payload.get("columns") or []
+    index = {name: i for i, name in enumerate(columns)}
+    missing = [name for name in ("rs", *COMPARED) if name not in index]
+    if missing:
+        raise SystemExit(f"reply is missing column(s) {missing}; has {columns}")
+    rows: dict[str, tuple[str, ...]] = {}
+    for row in payload.get("rows") or []:
+        rows[row[index["rs"]]] = tuple(row[index[name]] for name in COMPARED)
+    return rows
+
+
+def as_float(value: str) -> float | None:
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]]) -> int:
+    """Print the differences; return how many rows disagreed."""
+    only_left = sorted(set(left) - set(right))
+    only_right = sorted(set(right) - set(left))
+    shared = sorted(set(left) & set(right))
+    disagreed = []
+    worst = (0.0, "")
+    for rs in shared:
+        a, b = left[rs], right[rs]
+        if a == b:
+            continue
+        disagreed.append((rs, a, b))
+        for name, x, y in zip(COMPARED, a, b):
+            fx, fy = as_float(x), as_float(y)
+            if fx is None or fy is None:
+                if x != y:
+                    worst = (float("inf"), f"{rs} {name}: {x} vs {y}")
+                continue
+            delta = abs(fx - fy)
+            # Relative for anything not near zero, absolute otherwise: p-values
+            # differ in the 12th decimal and that is not a disagreement.
+            scale = max(abs(fx), abs(fy))
+            relative = delta / scale if scale > 1e-300 else delta
+            if relative > worst[0]:
+                worst = (relative, f"{rs} {name}: {x} vs {y}")
+
+    print(f"rows: gemma-wrapper {len(left)}, ravanan+gemma {len(right)}, shared {len(shared)}")
+    if only_left:
+        print(f"only in gemma-wrapper ({len(only_left)}): {only_left[:5]}")
+    if only_right:
+        print(f"only in ravanan+gemma ({len(only_right)}): {only_right[:5]}")
+    if disagreed:
+        print(f"disagreeing rows ({len(disagreed)}):")
+        for rs, a, b in disagreed[:10]:
+            print(f"  {rs}: wrapper {a} vs ravanan {b}")
+        print(f"worst relative difference: {worst[1]} ({worst[0]:.3g})")
+    return len(only_left) + len(only_right) + len(disagreed)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--base", default="http://127.0.0.1:3000", help="server base URL")
+    parser.add_argument("--body", help="JSON request body sent to both endpoints")
+    parser.add_argument("--group", help="group name (instead of --body)")
+    parser.add_argument("--version", help="genotype version for --group")
+    parser.add_argument("--trait", help="trait name to take the values of, for --group")
+    parser.add_argument("--samples", help="comma-separated individual names, with --group")
+    parser.add_argument("--lmm", type=int, default=9)
+    parser.add_argument("--timeout", type=float, default=3600.0)
+    args = parser.parse_args()
+
+    if args.body:
+        body = json.loads(args.body)
+    elif args.group and args.trait:
+        body = {"group": args.group, "samples": [], "lmm": args.lmm}
+        if args.version:
+            body["genotype_version"] = args.version
+        if args.samples:
+            body["samples"] = [
+                {"name": name, "values": [args.trait]} for name in args.samples.split(",")
+            ]
+        else:
+            # Named samples are per individual; without names the server reads
+            # trait_values positionally, which needs the panel order.
+            raise SystemExit("--group needs --samples (comma-separated individuals)")
+    else:
+        parser.error("give --body, or --group with --trait and --samples")
+
+    print(f"POST /compute/gemma         (gemma-wrapper) ...", flush=True)
+    wrapper, wrapper_seconds = post(args.base, "/compute/gemma", body, args.timeout)
+    print(f"  {wrapper_seconds:.2f} s, {len(wrapper.get('rows') or [])} rows")
+
+    print(f"POST /compute/gemma/ravanan (ravanan + gemma) ...", flush=True)
+    ravanan, ravanan_seconds = post(args.base, "/compute/gemma/ravanan", body, args.timeout)
+    print(f"  {ravanan_seconds:.2f} s, {len(ravanan.get('rows') or [])} rows")
+
+    print()
+    print(f"timing: gemma-wrapper {wrapper_seconds:.2f} s, ravanan+gemma {ravanan_seconds:.2f} s "
+          f"({ravanan_seconds / wrapper_seconds:.2f}x)")
+
+    # Notes carry the exact commands each runner used, which is what makes a
+    # disagreement diagnosable rather than mysterious.
+    for label, payload in (("gemma-wrapper", wrapper), ("ravanan+gemma", ravanan)):
+        for note in payload.get("notes") or []:
+            if note.lower().startswith(("kinship", "gwa", "ravanan", "scattered")):
+                print(f"  {label}: {note}")
+
+    print()
+    disagreements = compare(rows_by_rs(wrapper), rows_by_rs(ravanan))
+    if disagreements:
+        print(f"\nDISAGREEMENT: {disagreements} row(s)")
+        return 1
+    print("\nidentical on " + ", ".join(("rs", *COMPARED)))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())