about summary refs log tree commit diff
path: root/scripts/gemma-compare.py
blob: 1c6dd5dc75d250fb625ceb8a14b07862a9617dd6 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
#!/usr/bin/env python3
"""Compare the two GEMMA endpoints, row for row and second for second.

`/compute/gemma` runs gemma-wrapper (kinship, then GWA, both --loco) through
this server's own subprocess calls; `/compute/gemma/ravanan` runs `gemma`
itself, one invocation per chromosome, through a CWL workflow scattered by
ravanan. Same request body, same fixtures -- so the differences this prints are
the differences between two stacks doing one computation, which is exactly what
those two endpoints exist for.

    scripts/gemma-compare.py --base http://127.0.0.1:3000 \
        --body '{"genotype_file": "/path/BXD_geno.txt.gz",
                 "snps_file": "/path/BXD_snps.txt",
                 "trait_values": [...], "lmm": 9}'

    # or name a group and a trait in that group's sample list
    scripts/gemma-compare.py --base http://127.0.0.1:3000 \
        --group BXD --version v8 --trait 10001

What is compared: every row's `rs`, `beta`, `se`, `l_mle` and `p_lrt`, keyed by
`rs` (row order is not part of the contract), and the wall-clock time of each
call. A non-zero exit status means the two disagreed -- so this is usable as a
release gate, not just a report.

Requires a running server (see the README's "Running"). It does not start one.
"""
from __future__ import annotations

import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

COMPARED = ("beta", "se", "l_mle", "p_lrt")


def post(base: str, path: str, body: dict, timeout: float,
         token: str | None = None) -> tuple[dict, float]:
    """POST BODY to PATH, returning the decoded reply and the wall-clock time."""
    headers = {"content-type": "application/json"}
    if token:
        headers["authorization"] = "Bearer " + token
    request = urllib.request.Request(
        base.rstrip("/") + path,
        data=json.dumps(body).encode(),
        headers=headers,
    )
    started = time.monotonic()
    try:
        with urllib.request.urlopen(request, timeout=timeout) as response:
            payload = json.loads(response.read().decode())
    except urllib.error.HTTPError as error:
        detail = error.read().decode(errors="replace")
        raise SystemExit(f"{path}: HTTP {error.code}: {detail}") from error
    return payload, time.monotonic() - started


def fetch_trait_values(base: str, group: str, trait: str, timeout: float,
                       token: str | None) -> dict[str, float]:
    """The trait's value in each of the group's individuals, via the API."""
    path = f"/groups/{urllib.parse.quote(group)}/traits/{urllib.parse.quote(trait)}/values"
    headers = {}
    if token:
        headers["authorization"] = "Bearer " + token
    request = urllib.request.Request(base.rstrip("/") + path, headers=headers)
    try:
        with urllib.request.urlopen(request, timeout=timeout) as response:
            payload = json.loads(response.read().decode())
    except urllib.error.HTTPError as error:
        detail = error.read().decode(errors="replace")
        raise SystemExit(f"{path}: HTTP {error.code}: {detail}") from error
    rows = payload.get("data") or payload.get("rows") or []
    return {
        row["name"]: row["value"]
        for row in rows
        if isinstance(row, dict) and isinstance(row.get("value"), (int, float))
    }


def rows_by_rs(payload: dict) -> dict[str, tuple[str, ...]]:
    """Key the reply's rows by rs, keeping the columns this comparison cares about."""
    # The envelope carries the column names in `meta`; an older shape had them
    # at the top level, and `data` versus `rows` varies the same way.
    meta = payload.get("meta") or {}
    columns = payload.get("columns") or meta.get("columns") or []
    index = {name: i for i, name in enumerate(columns)}
    missing = [name for name in ("rs", *COMPARED) if name not in index]
    if missing:
        raise SystemExit(f"reply is missing column(s) {missing}; has {columns}")
    rows: dict[str, tuple[str, ...]] = {}
    for row in payload.get("data") or payload.get("rows") or []:
        rows[row[index["rs"]]] = tuple(row[index[name]] for name in COMPARED)
    return rows


def as_float(value: str) -> float | None:
    try:
        return float(value)
    except (TypeError, ValueError):
        return None


def compare(left: dict[str, tuple[str, ...]], right: dict[str, tuple[str, ...]],
            tolerance: float = 0.0) -> int:
    """Print the differences; return how many rows disagreed."""
    only_left = sorted(set(left) - set(right))
    only_right = sorted(set(right) - set(left))
    shared = sorted(set(left) & set(right))
    disagreed = []
    beyond = 0
    worst = (0.0, "")
    for rs in shared:
        a, b = left[rs], right[rs]
        if a == b:
            continue
        disagreed.append((rs, a, b))
        for name, x, y in zip(COMPARED, a, b):
            fx, fy = as_float(x), as_float(y)
            if fx is None or fy is None:
                if x != y:
                    worst = (float("inf"), f"{rs} {name}: {x} vs {y}")
                continue
            delta = abs(fx - fy)
            # Absolute, not relative: the difference between two GEMMA builds
            # is a fixed numerical offset, and beta can be near zero, where a
            # relative measure calls 0.013 against 0.016 a 16% disagreement.
            if delta > worst[0]:
                worst = (delta, f"{rs} {name}: {x} vs {y}")
            if delta > tolerance:
                beyond += 1

    print(f"rows: gemma-wrapper {len(left)}, ravanan+gemma {len(right)}, shared {len(shared)}")
    if only_left:
        print(f"only in gemma-wrapper ({len(only_left)}): {only_left[:5]}")
    if only_right:
        print(f"only in ravanan+gemma ({len(only_right)}): {only_right[:5]}")
    if disagreed:
        print(f"disagreeing rows ({len(disagreed)}):")
        for rs, a, b in disagreed[:10]:
            print(f"  {rs}: wrapper {a} vs ravanan {b}")
        print(f"worst absolute difference: {worst[1]} ({worst[0]:.3g})")
        print(f"values beyond the absolute tolerance {tolerance:g}: {beyond}")
    # Missing rows are always a failure: the two stacks are answering the same
    # question, so a marker in one and not the other is a bug in one of them.
    # Value differences are a *tolerance*: today the wrapper path runs
    # gemma-gn2-0.98.5 and the ravanan path gemma 0.98.6, which moves beta in
    # the third decimal while leaving af, n_miss and the marker set identical.
    return len(only_left) + len(only_right) + (1 if beyond else 0)


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--base", default="http://127.0.0.1:3000", help="server base URL")
    parser.add_argument("--body", help="JSON request body sent to both endpoints")
    parser.add_argument("--group", help="group name (instead of --body)")
    parser.add_argument("--version", help="genotype version for --group")
    parser.add_argument("--trait", help="trait name to take the values of, for --group")
    parser.add_argument("--samples", help="comma-separated individual names, with --group")
    parser.add_argument("--lmm", type=int, default=9)
    parser.add_argument("--timeout", type=float, default=3600.0)
    parser.add_argument("--token", default=os.environ.get("GN4_TOKEN"),
                        help="gn4ai_ token or JWT; defaults to $GN4_TOKEN")
    parser.add_argument("--tolerance", type=float, default=0.0,
                        help="largest ABSOLUTE difference in beta/se/l_mle/p_lrt "
                             "that still counts as agreement (default 0: exact). "
                             "Two different GEMMA builds need 0.005; one build "
                             "should need nothing.")
    args = parser.parse_args()

    if args.body:
        body = json.loads(args.body)
    elif args.group and args.trait:
        body = {"group": args.group, "samples": [], "lmm": args.lmm}
        if args.version:
            body["genotype_version"] = args.version
        # The trait's values come from the server, not from the command line:
        # `samples` is per individual and needs a *number* per name, which a
        # trait name cannot stand in for (it used to be sent as
        # {"name": n, "values": [trait]}, and every run answered 422).
        values = fetch_trait_values(args.base, args.group, args.trait, args.timeout, args.token)
        wanted = None
        if args.samples:
            wanted = {name.strip() for name in args.samples.split(",")}
        body["samples"] = [
            {"name": name, "value": value}
            for name, value in values.items()
            if wanted is None or name in wanted
        ]
        if not body["samples"]:
            raise SystemExit(
                f"none of the trait's individuals {'/'.join(sorted(wanted or []))} "
                "carry a value in " + args.group)
        print(f"{args.trait}: {len(body['samples'])} individual(s) with a value")
    else:
        parser.error("give --body, or --group with --trait and --samples")

    print(f"POST /compute/gemma         (gemma-wrapper) ...", flush=True)
    wrapper, wrapper_seconds = post(args.base, "/compute/gemma", body, args.timeout, args.token)
    print(f"  {wrapper_seconds:.2f} s, {len(wrapper.get('rows') or [])} rows")

    print(f"POST /compute/gemma/ravanan (ravanan + gemma) ...", flush=True)
    ravanan, ravanan_seconds = post(args.base, "/compute/gemma/ravanan", body, args.timeout, args.token)
    print(f"  {ravanan_seconds:.2f} s, {len(ravanan.get('rows') or [])} rows")

    print()
    print(f"timing: gemma-wrapper {wrapper_seconds:.2f} s, ravanan+gemma {ravanan_seconds:.2f} s "
          f"({ravanan_seconds / wrapper_seconds:.2f}x)")

    # Notes carry the exact commands each runner used, which is what makes a
    # disagreement diagnosable rather than mysterious.
    for label, payload in (("gemma-wrapper", wrapper), ("ravanan+gemma", ravanan)):
        for note in payload.get("notes") or []:
            if note.lower().startswith(("kinship", "gwa", "ravanan", "scattered")):
                print(f"  {label}: {note}")

    print()
    disagreements = compare(rows_by_rs(wrapper), rows_by_rs(ravanan), args.tolerance)
    if disagreements:
        print(f"\nDISAGREEMENT: {disagreements} row(s)")
        return 1
    if args.tolerance:
        print("\nagree on the marker set, and within " + f"{args.tolerance:g} absolute on "
              + ", ".join(COMPARED) + " (not identical -- raise the bar with --tolerance 0)")
    else:
        print("\nidentical on " + ", ".join(("rs", *COMPARED)))
    return 0


if __name__ == "__main__":
    sys.exit(main())