about summary refs log tree commit diff
path: root/scripts/pca-parity.py
blob: 6bef16c4cea8e3673867893e0b3b6907f7a30e2c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
#!/usr/bin/env python3
"""Check a matrix PCA against numpy.linalg.eigh, to machine precision.

The correlation matrix is the input and the eigendecomposition is the only
numerically delicate step in the whole correlation surface (ai-memory #106), so
it gets its own harness: the server's own `pca` block is compared against what
numpy computes from the *same* `r` matrix, which is an independent
implementation of the same mathematics (LAPACK's dsyevd against our Jacobi).

    scripts/pca-parity.py --traits BXDPublish:10001,BXDPublish:10002,... \\
        [--base http://127.0.0.1:3000] [--token ...]

What is compared, per component:
  * eigenvalues, relatively (they are O(1) here);
  * loadings, by absolute difference *after* aligning signs -- an eigenvector
    is defined up to sign, so the server fixes a convention (largest absolute
    loading positive) and this script flips numpy's to match before comparing.

Exit status is 0 only if everything agrees, so it is usable as a gate.
Needs numpy (`guix shell python python-numpy` or the scipy profile).
"""

import argparse
import json
import os
import sys
import urllib.error
import urllib.request

TOLERANCE = 1e-10


def post(base: str, path: str, body: dict, token: str | None, timeout: float) -> dict:
    headers = {"content-type": "application/json"}
    if token:
        headers["authorization"] = "Bearer " + token
    request = urllib.request.Request(
        base.rstrip("/") + path,
        data=json.dumps(body).encode(),
        headers=headers,
    )
    try:
        with urllib.request.urlopen(request, timeout=timeout) as response:
            payload = json.loads(response.read().decode())
    except urllib.error.HTTPError as error:
        detail = error.read().decode(errors="replace")
        raise SystemExit(f"{path}: HTTP {error.code}: {detail}") from error
    return payload


def traits_of(spec: str) -> list[dict]:
    out = []
    for item in spec.split(","):
        dataset, _, trait = item.strip().partition(":")
        if not dataset or not trait:
            raise SystemExit(f"--traits wants dataset:trait, got {item!r}")
        out.append({"dataset": dataset, "trait_name": trait})
    return out


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--base", default="http://127.0.0.1:3000")
    parser.add_argument("--token", default=os.environ.get("GN4_TOKEN"))
    parser.add_argument("--traits", required=True,
                        help="dataset:trait[,dataset:trait...]")
    parser.add_argument("--method", default="pearson")
    parser.add_argument("--tolerance", type=float, default=TOLERANCE)
    parser.add_argument("--timeout", type=float, default=600.0)
    args = parser.parse_args()

    try:
        import numpy
    except ImportError:
        raise SystemExit("needs numpy: guix shell python python-numpy -- this script")

    payload = post(args.base, "/compute/correlation/matrix",
                   {"traits": traits_of(args.traits), "method": args.method, "pca": True},
                   args.token, args.timeout)
    data = payload.get("data") or {}
    ours = data.get("pca") or {}
    if not ours.get("components"):
        raise SystemExit(f"no pca in the reply: {json.dumps(payload)[:300]}")

    kept = [t for t in data.get("traits") or []
            if not any(d["dataset"] == t["dataset"] and d["trait_name"] == t["trait_name"]
                       for d in ours.get("dropped") or [])]
    dropped = [f"{d['dataset']}:{d['trait_name']}" for d in ours.get("dropped") or []]
    print(f"{len(kept)} trait(s) in the PCA"
          + (f", {len(dropped)} dropped: {dropped}" if dropped else ""))

    # Rebuild the kept submatrix from the reply's own r and decompose it.
    order = [i for i, t in enumerate(data["traits"]) if t in kept]
    matrix = numpy.array([[data["r"][i][j] for j in order] for i in order], dtype=float)
    values, vectors = numpy.linalg.eigh(matrix)          # ascending
    values = values[::-1]
    vectors = vectors[:, ::-1]

    worst_value, worst_loading = 0.0, 0.0
    for index, component in enumerate(ours["components"]):
        if index >= len(values):
            break
        expected = values[index]
        got = component["eigenvalue"]
        scale = max(abs(expected), abs(got), 1e-300)
        worst_value = max(worst_value, abs(expected - got) / scale)

        reference = vectors[:, index]
        # Align signs on the largest absolute loading, the server's convention.
        pivot = int(numpy.argmax(numpy.abs(reference)))
        if (reference[pivot] < 0) != (component["loadings"][pivot] < 0):
            reference = -reference
        for a, b in zip(component["loadings"], reference):
            worst_loading = max(worst_loading, abs(a - b))

    trace = sum(c["eigenvalue"] for c in ours["components"])
    print(f"eigenvalues: worst relative difference {worst_value:.3g}")
    print(f"loadings:    worst absolute difference {worst_loading:.3g}")
    print(f"trace:       {trace:.12f} (should equal {len(kept)})")
    if abs(trace - len(kept)) > 1e-8 * max(1, len(kept)):
        print("TRACE MISMATCH: the decomposition is not sane")
        return 1
    if worst_value > args.tolerance or worst_loading > args.tolerance:
        print(f"DISAGREEMENT beyond {args.tolerance:g}")
        return 1
    print("agrees with numpy.linalg.eigh")
    return 0


if __name__ == "__main__":
    sys.exit(main())