#!/usr/bin/env python3 # pyright: ignore """Generate reduced-v1 parity fixtures from the recorded production corpus. Reads the full fixture corpus (105 MB of recorded gn3 responses, kept out of git) and writes one reduced-v1 file per shape: the full sorted identifier set, the recorded first-ten order, and the recorded status — exactly what the parity harness grades (design §C.3). Error shapes carry their diagnostic body instead. Fails loudly on any hit missing its (dataset, name) identity or on any corpus/table mismatch. The shape table is parsed from the harness itself (tests/search_parity.rs SHAPES), so the generator can never drift from what grades the files. Usage: python3 reduce_fixtures.py """ import json import re import sys from pathlib import Path SHAPE_RE = re.compile( r'shape!\(\s*"(Q\d+)"\s*,\s*"((?:[^"\\]|\\.)*)"\s*,\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*,' ) def unescape(raw: str) -> str: """Unescape the two Rust escapes used in shape queries (\\ and ").""" out = [] i = 0 while i < len(raw): if raw[i] == "\\" and i + 1 < len(raw): out.append(raw[i + 1]) i += 2 else: out.append(raw[i]) i += 1 return "".join(out) def main() -> int: if len(sys.argv) != 4: print(__doc__) return 2 corpus_dir, out_dir, harness_rs = (Path(a) for a in sys.argv[1:4]) shapes = SHAPE_RE.findall(harness_rs.read_text()) if len(shapes) != 54: print(f"FATAL: parsed {len(shapes)} shapes from {harness_rs}, expected 54") return 1 out_dir.mkdir(parents=True, exist_ok=True) total_in = total_out = 0 for shape_id, raw_query, kind, _table_class, status in shapes: src = corpus_dir / f"{shape_id}.json" if not src.is_file(): print(f"FATAL: missing corpus fixture {src}") return 1 body = json.loads(src.read_text()) total_in += src.stat().st_size reduced = { "format": "reduced-v1", "id": shape_id, "raw_query": unescape(raw_query), "type": kind, "recorded_status": int(status), } if isinstance(body, list): identifiers = [] top10 = [] seen = set() for idx, hit in enumerate(body): if not isinstance(hit, dict): print(f"FATAL {shape_id}: hit {idx} is not an object") return 1 dataset = hit.get("dataset") name = hit.get("name") if not isinstance(dataset, str) or not isinstance(name, str): print( f"FATAL {shape_id}: hit {idx} lacks (dataset,name) strings: " f"dataset={dataset!r} name={name!r}" ) return 1 pair = (dataset, name) if pair not in seen: seen.add(pair) identifiers.append(pair) if idx < 10: top10.append(pair) identifiers.sort() if len(identifiers) != len(body): print( f"NOTE {shape_id}: array has {len(body)} hits, " f"{len(identifiers)} unique (dataset,name) pairs" ) reduced["hit_count"] = len(identifiers) reduced["identifiers"] = identifiers reduced["top10"] = top10 elif isinstance(body, dict): reduced["diagnostic"] = body else: print(f"FATAL {shape_id}: corpus body is {type(body).__name__}, not array/object") return 1 dst = out_dir / f"{shape_id}.json" dst.write_text(json.dumps(reduced, sort_keys=True, separators=(",", ":"))) total_out += dst.stat().st_size print( f"{shape_id}: {src.stat().st_size:>10,} B -> {dst.stat().st_size:>9,} B " f"({reduced.get('hit_count', 'err') if 'identifiers' in reduced else 'diagnostic'} entries)" ) print(f"TOTAL: {total_in:,} B -> {total_out:,} B ({total_out / total_in:.1%})") return 0 if __name__ == "__main__": sys.exit(main())