1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
#!/usr/bin/env python3
# pyright: ignore
"""Generate reduced-v1 parity fixtures from the recorded production corpus.
Reads the full fixture corpus (105 MB of recorded gn3 responses, kept out
of git) and writes one reduced-v1 file per shape: the full sorted
identifier set, the recorded first-ten order, and the recorded status —
exactly what the parity harness grades (design §C.3). Error shapes carry
their diagnostic body instead. Fails loudly on any hit missing its
(dataset, name) identity or on any corpus/table mismatch.
The shape table is parsed from the harness itself (tests/search_parity.rs
SHAPES), so the generator can never drift from what grades the files.
Usage:
python3 reduce_fixtures.py <corpus_dir> <out_dir> <search_parity_rs>
"""
import json
import re
import sys
from pathlib import Path
SHAPE_RE = re.compile(
r'shape!\(\s*"(Q\d+)"\s*,\s*"((?:[^"\\]|\\.)*)"\s*,\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*,'
)
def unescape(raw: str) -> str:
"""Unescape the two Rust escapes used in shape queries (\\ and ")."""
out = []
i = 0
while i < len(raw):
if raw[i] == "\\" and i + 1 < len(raw):
out.append(raw[i + 1])
i += 2
else:
out.append(raw[i])
i += 1
return "".join(out)
def main() -> int:
if len(sys.argv) != 4:
print(__doc__)
return 2
corpus_dir, out_dir, harness_rs = (Path(a) for a in sys.argv[1:4])
shapes = SHAPE_RE.findall(harness_rs.read_text())
if len(shapes) != 54:
print(f"FATAL: parsed {len(shapes)} shapes from {harness_rs}, expected 54")
return 1
out_dir.mkdir(parents=True, exist_ok=True)
total_in = total_out = 0
for shape_id, raw_query, kind, _table_class, status in shapes:
src = corpus_dir / f"{shape_id}.json"
if not src.is_file():
print(f"FATAL: missing corpus fixture {src}")
return 1
body = json.loads(src.read_text())
total_in += src.stat().st_size
reduced = {
"format": "reduced-v1",
"id": shape_id,
"raw_query": unescape(raw_query),
"type": kind,
"recorded_status": int(status),
}
if isinstance(body, list):
identifiers = []
top10 = []
seen = set()
for idx, hit in enumerate(body):
if not isinstance(hit, dict):
print(f"FATAL {shape_id}: hit {idx} is not an object")
return 1
dataset = hit.get("dataset")
name = hit.get("name")
if not isinstance(dataset, str) or not isinstance(name, str):
print(
f"FATAL {shape_id}: hit {idx} lacks (dataset,name) strings: "
f"dataset={dataset!r} name={name!r}"
)
return 1
pair = (dataset, name)
if pair not in seen:
seen.add(pair)
identifiers.append(pair)
if idx < 10:
top10.append(pair)
identifiers.sort()
if len(identifiers) != len(body):
print(
f"NOTE {shape_id}: array has {len(body)} hits, "
f"{len(identifiers)} unique (dataset,name) pairs"
)
reduced["hit_count"] = len(identifiers)
reduced["identifiers"] = identifiers
reduced["top10"] = top10
elif isinstance(body, dict):
reduced["diagnostic"] = body
else:
print(f"FATAL {shape_id}: corpus body is {type(body).__name__}, not array/object")
return 1
dst = out_dir / f"{shape_id}.json"
dst.write_text(json.dumps(reduced, sort_keys=True, separators=(",", ":")))
total_out += dst.stat().st_size
print(
f"{shape_id}: {src.stat().st_size:>10,} B -> {dst.stat().st_size:>9,} B "
f"({reduced.get('hit_count', 'err') if 'identifiers' in reduced else 'diagnostic'} entries)"
)
print(f"TOTAL: {total_in:,} B -> {total_out:,} B ({total_out / total_in:.1%})")
return 0
if __name__ == "__main__":
sys.exit(main())
|