//! GN4 search parity harness (P1 build). //! //! Env-gated integration test that replays the 54 recorded query shapes //! (see `~/spikes/parity-harness/RUNNER-DESIGN.md`) through the real //! `genenetwork4::app()` and grades the result against the frozen fixture //! corpus, per `parity-metric.md`. //! //! Harness v2 (post-`hits.truncate`). The route's documented `limit` //! contract (default 20, capped at 100; `hits.truncate(limit)` since //! e912c87) means a route replay can no longer carry the full match set, //! so clause (a) set equality is graded BELOW the truncate, at the engine //! seam: the harness calls `genenetwork4::search::search` directly, in //! process, with a large limit (PORT-DESIGN §5 per_page invariant: the //! engine retrieves the full match set; the route keeps its limit logic). //! The HTTP route replay is kept for everything truncation preserves: //! status classes, error bodies, and clause (b)'s top-10 (the first ten //! hits in recorded order survive a 20-hit truncation; verified //! empirically — see HARNESS-FIX-REPORT.md). //! //! Hermetic, exactly like `tests/search.rs`: an axum stub stands in for gn3 //! (serving the verbatim recorded body for the forwarded `(query, type)` //! pair) and a second stub stands in for gn-auth's JWKS; tokens are minted //! locally with the committed test key. No live gn3, no network. The only //! I/O is reading the read-only fixture corpus and writing the report. //! //! Env: //! - `GN4_PARITY_FIXTURES` (optional) — corpus directory to replay. //! Unset => the committed reduced-v1 corpus under //! `test/data/parity-fixtures/` (runs hermetically, no downloads). //! Set it to point at any external corpus (full or reduced files, //! mixable). //! - `GN4_PARITY_REPORTS` (optional) — report output dir; default //! `/target/parity-reports/`. //! - `GN4_PARITY_ONLY` (optional) — comma-separated shape ids, debug //! filter; a run with it is marked NON-FULL-RUN in the report. //! //! All shapes run sequentially inside one test because `GN3_URL` / //! `AUTH_SERVER_URL` are process-global and `std::env::set_var` is unsafe in //! edition 2024. The runtime is multi-thread because `src/auth.rs` fetches //! the JWKS with blocking `ureq` (documented in `tests/search.rs`). use axum::Router; use axum::body::Body; use axum::extract::{Query, State}; use axum::http::{Request, StatusCode}; use axum::routing::get; use http_body_util::BodyExt; use jsonwebtoken::{Algorithm, EncodingKey, Header, encode}; use serde::Serialize; use serde_json::{Value, json}; use std::collections::{BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use tower::ServiceExt; /// Committed test-only key material, same files `tests/search.rs` uses. const TEST_JWKS: &str = include_str!("../test/data/test_jwks.json"); const TEST_PRIVATE_KEY: &str = include_str!("../test/data/test_rsa_private.pem"); /// The gn4 tree this harness replays (frozen for the report sidecar). const GN4_SHA: &str = "8166218"; /// `f64` floor for clause (b) (metric §1). const JACCARD_FLOOR: f64 = 0.8; /// The limit the harness passes the engine seam for clause (a) set /// grading. The route caps `limit` at 100 and truncates its response (its /// documented contract), so full-set grading moved below the truncate, to /// the engine (PORT-DESIGN §5: the engine retrieves the full match set; /// the route keeps its limit logic). The value only needs to exceed the /// largest recorded match set (Q02: 57632 hits): the stub gn3 ignores the /// forwarded `per_page` and serves the full recorded body regardless (the /// calibration-stub behavior this harness has always relied on), and a /// native engine sizes retrieval to the whole match set and ignores the /// bound it receives. const ENGINE_FETCH_LIMIT: u64 = 1_000_000; // --------------------------------------------------------------------------- // B.1 Shape metadata, mirroring record_fixtures.py's SHAPES order exactly. // --------------------------------------------------------------------------- /// Effective grading class, fixture-derived (design §D reconciliation table). #[derive(Clone, Copy, PartialEq, Eq, Debug)] enum Class { /// Recorded 200 + top-level array: clause (a) set equality (+ clause (b) /// when the recorded set has >= 10 hits). Match200, /// Recorded 4xx parser diagnostic (Q51, Q52) => expect 422 invalid_query. Error4xx, /// Recorded 500 (Q42) => expect 422 invalid_query — the blessed /// typed-422 contract (owner-approved 2026-10-06; AMENDMENT E): the /// recorded production 500 was an escaping xapian exception, an /// implementation accident, not a contract. The recorded fixture /// itself stays untouched as production truth; only this mapping /// rule re-baselines. Error500, /// C-surface, unobservable at gn3 (Q48, Q49) => expect 400 /// invalid_parameter at the gn4 layer, zero upstream calls. Surface400, } impl Class { fn label(self) -> &'static str { match self { Class::Match200 => "200-match", Class::Error4xx => "error-4xx", Class::Error500 => "error-500", Class::Surface400 => "surface-400", } } } struct Shape { id: &'static str, /// Raw query exactly as recorded (before any URL-encoding). raw_query: &'static str, /// Forwarded `type` (always phenotype/gene). kind: &'static str, /// Behavior class from query-shapes.md (informational). table_class: &'static str, /// Recorded HTTP status (200, or 400/500 for the error shapes; the /// surface shapes' recorded gn3 200 is never served). recorded_status: u16, class: Class, } macro_rules! shape { ($id:literal, $q:literal, $t:literal, $tc:literal, $st:literal, $c:expr) => { Shape { id: $id, raw_query: $q, kind: $t, table_class: $tc, recorded_status: $st, class: $c, } }; } const SHAPES: &[Shape] = &[ shape!( "Q01", "alcohol", "phenotype", "C-match", 200, Class::Match200 ), shape!("Q02", "insulin", "gene", "C-match", 200, Class::Match200), shape!( "Q03", "alcohol dependence", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q04", "\"alcohol dependence\"", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q05", "alcohol*", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q06", "Alcohol", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q07", "alcohol AND dependence", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q08", "alcohol OR dependence", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q09", "alcohol NOT dependence", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q10", "alcohol XOR dependence", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q11", "alcohol NEAR dependence", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q12", "alcohol ADJ dependence", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q13", "alcohol and dependence", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q14", "author:mulligan", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q15", "species:mouse", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q16", "group:B6", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q17", "group:b6", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q18", "tissue:brain", "phenotype", "C-empty", 200, Class::Match200 ), shape!( "Q19", "dataset:bxdpublish", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q20", "symbol:insr", "gene", "C-match", 200, Class::Match200 ), shape!( "Q21", "description:insulin", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q22", "rif:insulin", "gene", "C-match", 200, Class::Match200 ), shape!( "Q23", "RIF:insulin", "gene", "C-match", 200, Class::Match200 ), shape!( "Q24", "wiki:diabetes", "phenotype", "C-empty", 200, Class::Match200 ), shape!( "Q25", "WIKI:diabetes", "phenotype", "C-empty", 200, Class::Match200 ), shape!("Q26", "chr:m", "gene", "C-match", 200, Class::Match200), shape!( "Q27", "peakchr:1", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q28", "mean:10..20", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q29", "mean:..15", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q30", "mean:10..", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q31", "peak:5..15", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q32", "position:1.05M..1.06M", "gene", "C-match", 200, Class::Match200 ), shape!( "Q33", "position:7500000", "gene", "C-match", 200, Class::Match200 ), shape!( "Q34", "pos:1.05M..1.06M", "gene", "C-match", 200, Class::Match200 ), shape!( "Q35", "peakmb:1.5..2.5", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q36", "additive:0.5..1.0", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q37", "year:2010..2020", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q38", "year:2030..2050", "phenotype", "C-empty", 200, Class::Match200 ), shape!( "Q39", "mean:99999..100000", "phenotype", "C-empty", 200, Class::Match200 ), shape!( "Q40", "Hs:chr2:1M..1.2M", "gene", "C-match", 200, Class::Match200 ), shape!("Q41", "Mm:chr1:5M", "gene", "C-match", 200, Class::Match200), shape!( "Q42", "Hs:foo:1M", "phenotype", "C-empty", 500, Class::Error500 ), shape!( "Q43", "species:mouse alcohol", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q44", "author:mulligan AND year:2010..2015", "phenotype", "C-match", 200, Class::Match200 ), shape!( "Q45", "symbol:insr OR rif:insulin", "gene", "C-match", 200, Class::Match200 ), shape!( "Q46", "(alcohol OR insulin) AND species:mouse", "phenotype", "C-match", 200, Class::Match200 ), shape!("Q47", "a&&b", "phenotype", "C-parser", 200, Class::Match200), shape!("Q48", "", "phenotype", "C-surface", 200, Class::Surface400), shape!( "Q49", " ", "phenotype", "C-surface", 200, Class::Surface400 ), shape!( "Q50", "foo:bar", "phenotype", "C-empty", 200, Class::Match200 ), shape!( "Q51", "alcohol AND", "phenotype", "C-parser", 400, Class::Error4xx ), shape!( "Q52", "mean:abc..def", "phenotype", "C-parser", 400, Class::Error4xx ), shape!( "Q53", "position:", "phenotype", "C-parser", 200, Class::Match200 ), shape!( "Q54", "alcohol &&", "phenotype", "C-parser", 200, Class::Match200 ), ]; /// Recorded-status display string (fixture statuses are bodies-only; the /// error statuses are reconstructed, design §B.1). fn recorded_status_display(shape: &Shape) -> String { match shape.class { Class::Error4xx => "4xx".to_string(), Class::Surface400 => "200(gn3,unobservable)".to_string(), _ => shape.recorded_status.to_string(), } } // --------------------------------------------------------------------------- // C.1 Comparator: pure, no I/O, no async — reusable by the port phase. // --------------------------------------------------------------------------- mod comparator { use serde_json::Value; use std::collections::BTreeSet; /// The `(dataset, name)` identity of a hit. Both fields stay `Option` /// so an absent field is a distinct key, never coerced to `""`. pub type HitKey = (Option, Option); /// Extract `(dataset, name)` from one hit object. pub fn hit_key(hit: &Value) -> HitKey { let dataset = hit .get("dataset") .and_then(Value::as_str) .map(str::to_string); let name = hit.get("name").and_then(Value::as_str).map(str::to_string); (dataset, name) } /// Full identifier set of a hit array. pub fn identifier_set(hits: &[Value]) -> BTreeSet { hits.iter().map(hit_key).collect() } /// First 10 hits' keys, in recorded order (order matters only for /// *which* ten; comparison is on the resulting set). pub fn top10_keys(hits: &[Value]) -> Vec { hits.iter().take(10).map(hit_key).collect() } /// `|A ∩ B| / |A ∪ B|`; an empty union is defined as 1.0. pub fn jaccard(a: &BTreeSet, b: &BTreeSet) -> f64 { let union = a.union(b).count(); if union == 0 { return 1.0; } let inter = a.intersection(b).count(); inter as f64 / union as f64 } #[derive(Debug, PartialEq, Eq)] pub struct MatchGrade { pub equal: bool, pub missing: usize, pub extra: usize, } /// Set-equality grade between the fixture set and the replayed set. pub fn grade_match(expected: &BTreeSet, actual: &BTreeSet) -> MatchGrade { let missing = expected.difference(actual).count(); let extra = actual.difference(expected).count(); MatchGrade { equal: missing == 0 && extra == 0, missing, extra, } } /// Recorded status class used for error-shape grading. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum RecordedStatus { /// Kept for the API surface (a 200 shape is never graded as an /// error), so the match arm below is total. #[allow(dead_code)] Ok200, Client4xx, Server500, Surface, } #[derive(Debug, PartialEq, Eq)] pub enum ClassGrade { Pass, Fail(String), } impl ClassGrade { pub fn is_pass(&self) -> bool { matches!(self, ClassGrade::Pass) } } /// Behavior-class grading (design §C.2 error-shape mapping): /// - recorded 4xx => gn4 422 `invalid_query` /// - recorded 500 => gn4 422 `invalid_query` (the blessed typed-422 /// contract; the recorded production 500 was an escaping xapian /// exception — an implementation accident, re-baselined per /// AMENDMENT E. The fixture stays untouched as production truth.) /// - surface => gn4 400 `invalid_parameter` with no upstream call pub fn grade_error_class( recorded: RecordedStatus, replayed_status: u16, replayed_code: &str, upstream_called: bool, ) -> ClassGrade { match recorded { RecordedStatus::Client4xx => { if replayed_status == 422 && replayed_code == "invalid_query" { ClassGrade::Pass } else { ClassGrade::Fail(format!( "recorded 4xx: expected 422 invalid_query, got {replayed_status} {replayed_code}" )) } } RecordedStatus::Server500 => { // AMENDMENT E: the blessed typed-422 contract re-baselines // Q42 — recorded production 500 -> expected 422 // invalid_query. The fixture itself stays untouched as // production truth; this rule is the re-baseline. if replayed_status == 422 && replayed_code == "invalid_query" { ClassGrade::Pass } else { ClassGrade::Fail(format!( "recorded 500 (re-baselined to the typed-422 contract): \ expected 422 invalid_query, got {replayed_status} {replayed_code}" )) } } RecordedStatus::Surface => { if replayed_status == 400 && replayed_code == "invalid_parameter" && !upstream_called { ClassGrade::Pass } else { ClassGrade::Fail(format!( "surface: expected 400 invalid_parameter with 0 upstream calls, got {replayed_status} {replayed_code} upstream_called={upstream_called}" )) } } RecordedStatus::Ok200 => { ClassGrade::Fail("recorded 200 is not an error shape".to_string()) } } } /// Clause (b) applies only to shapes whose recorded set has >= 10 hits. pub fn clause_b_applies(recorded_hits: usize) -> bool { recorded_hits >= 10 } /// `None` when clause (b) does not apply; otherwise whether J meets the /// 0.8 floor. pub fn clause_b_pass(recorded_hits: usize, j: f64) -> Option { if clause_b_applies(recorded_hits) { Some(j >= super::JACCARD_FLOOR) } else { None } } } #[cfg(test)] mod comparator_tests { use super::comparator::*; use serde_json::json; fn hits(pairs: &[(&str, &str)]) -> Vec { pairs .iter() .map(|(d, n)| json!({ "dataset": d, "name": n })) .collect() } fn set(pairs: &[(&str, &str)]) -> std::collections::BTreeSet { identifier_set(&hits(pairs)) } // 1. Set equality is order-insensitive; a swapped member shows up as // one missing + one extra. #[test] fn set_equality_ignores_order_and_reports_missing_extra() { let a = set(&[("d1", "n1"), ("d2", "n2"), ("d3", "n3")]); let b = set(&[("d3", "n3"), ("d1", "n1"), ("d2", "n2")]); let g = grade_match(&a, &b); assert!(g.equal); assert_eq!(g.missing, 0); assert_eq!(g.extra, 0); let c = set(&[("d1", "n1"), ("d2", "n2"), ("d9", "n9")]); let g2 = grade_match(&a, &c); assert!(!g2.equal); assert_eq!(g2.missing, 1); assert_eq!(g2.extra, 1); } // 2. Empty-vs-empty is equal (metric §1 clause (a)). #[test] fn empty_vs_empty_is_equal() { let a = set(&[]); let b = set(&[]); let g = grade_match(&a, &b); assert!(g.equal); assert_eq!(g.missing, 0); assert_eq!(g.extra, 0); assert_eq!(jaccard(&a, &b), 1.0); } // 3. Error-class grading, one passing and one failing case per class. #[test] fn error_class_grading() { // recorded 4xx => 422 invalid_query assert!(grade_error_class(RecordedStatus::Client4xx, 422, "invalid_query", true).is_pass()); assert!(!grade_error_class(RecordedStatus::Client4xx, 200, "", true).is_pass()); // recorded 500 => 422 invalid_query (AMENDMENT E re-baseline: // the blessed typed-422 contract; the recorded production 500 // was an escaping xapian exception, not a contract). assert!(grade_error_class(RecordedStatus::Server500, 422, "invalid_query", true).is_pass()); assert!( !grade_error_class(RecordedStatus::Server500, 502, "backend_error", true).is_pass() ); // surface => 400 invalid_parameter, no upstream call assert!( grade_error_class(RecordedStatus::Surface, 400, "invalid_parameter", false).is_pass() ); assert!( !grade_error_class(RecordedStatus::Surface, 400, "invalid_parameter", true).is_pass() ); assert!(!grade_error_class(RecordedStatus::Surface, 200, "", false).is_pass()); } // Unique keys n0..n{n-1} on dataset "d", so a set of size n really has // n distinct members. fn keys(n: usize) -> std::collections::BTreeSet { (0..n) .map(|i| (Some("d".to_string()), Some(format!("n{i}")))) .collect() } // 4. Top-10 Jaccard: identical => 1.0; 8-of-12 => 0.667 (fails the 0.8 // floor); < 10 recorded hits => clause (b) not applied. #[test] fn top10_jaccard_and_floor() { let a = keys(10); let b = keys(10); assert_eq!(jaccard(&a, &b), 1.0); assert_eq!(clause_b_pass(10, jaccard(&a, &b)), Some(true)); // Two 10-element sets sharing 8 members => union 12, intersection 8, // J = 8/12 = 0.6667. let shared = keys(8); let mut u1 = shared.clone(); u1.insert((Some("d".to_string()), Some("a".to_string()))); u1.insert((Some("d".to_string()), Some("b".to_string()))); let mut u2 = shared; u2.insert((Some("d".to_string()), Some("c".to_string()))); u2.insert((Some("d".to_string()), Some("dd".to_string()))); assert_eq!(u1.len(), 10); assert_eq!(u2.len(), 10); let j = jaccard(&u1, &u2); assert!((j - 8.0 / 12.0).abs() < 1e-9, "jaccard was {j}"); assert_eq!(clause_b_pass(12, j), Some(false)); // Fewer than 10 recorded hits => clause (b) does not apply. assert_eq!(clause_b_pass(9, 0.0), None); assert!(!clause_b_applies(9)); } // Extra: Option fields are distinct keys, never coerced to "". #[test] fn absent_fields_are_distinct_keys() { let with = set(&[("d", "n")]); let without = identifier_set(&[json!({ "dataset": null, "name": "n" })]); assert_ne!(with, without); let g = grade_match(&with, &without); assert_eq!(g.missing, 1); assert_eq!(g.extra, 1); } // 5. top10_keys takes exactly the first ten keys in recorded order; the // eleventh hit is excluded. #[test] fn top10_keys_takes_first_ten_in_recorded_order() { let h = hits(&[ ("d", "n0"), ("d", "n1"), ("d", "n2"), ("d", "n3"), ("d", "n4"), ("d", "n5"), ("d", "n6"), ("d", "n7"), ("d", "n8"), ("d", "n9"), ("d", "n10"), ]); let k = top10_keys(&h); assert_eq!(k.len(), 10); assert_eq!(k[0], hit_key(&h[0])); assert_eq!(k[9], hit_key(&h[9])); assert!(!k.contains(&hit_key(&h[10]))); } // 6. Jaccard of the empty set against a non-empty set is 0.0. #[test] fn jaccard_empty_vs_nonempty_is_zero() { let empty = set(&[]); let nonempty = set(&[("d", "n")]); assert_eq!(jaccard(&empty, &nonempty), 0.0); } } // --------------------------------------------------------------------------- // C.4 Fixture loading: full recorded arrays and reduced-v1 (commit-sized). // --------------------------------------------------------------------------- mod fixture { use super::comparator::HitKey; use serde_json::{Value, json}; use std::collections::BTreeSet; use std::path::Path; pub const REDUCED_FORMAT: &str = "reduced-v1"; /// A fixture as loaded from disk, in either committed form. #[derive(Debug)] pub enum LoadedFixture { /// Top-level array: the recorded hit list in recorded order. /// Served verbatim by the stub. Full { hits: Vec }, /// Top-level object that is NOT reduced-v1: an error-diagnostic /// body. Served verbatim; must never grade as a match shape. RawObject { body: String }, /// reduced-v1 (design §C.3): the full sorted identifier set plus /// the recorded first-ten order, so the 105 MB payload corpus /// never needs to enter git. Error shapes carry `diagnostic` /// instead of identifiers. Reduced { identifiers: Vec, top10: Vec, diagnostic: Option, }, } /// Shape identity for the reduced-file integrity checks: the stub keys /// on (raw_query, kind) and the table pins recorded_status, so a /// reduced file naming a different shape must fail loudly, not serve /// as a silent wrong answer. pub struct ShapeRef<'a> { pub id: &'a str, pub raw_query: &'a str, pub kind: &'a str, pub recorded_status: u16, } impl LoadedFixture { /// Load and sniff `/.json` (design §C.3 loader /// sniffing: top-level array => full; top-level object with /// `"format": "reduced-v1"` => reduced; any other top-level /// object => raw error body; anything else is a loud error). /// `shape` is checked against the reduced file's self-description /// (id, raw_query, type, recorded_status) before it is used. pub fn load(fixtures_dir: &Path, shape: &ShapeRef) -> Result { let path = fixtures_dir.join(format!("{}.json", shape.id)); let text = std::fs::read_to_string(&path) .map_err(|e| format!("cannot read fixture {}: {e}", path.display()))?; let value: Value = serde_json::from_str(&text) .map_err(|e| format!("cannot parse fixture {} as JSON: {e}", path.display()))?; match value { Value::Array(hits) => Ok(LoadedFixture::Full { hits }), Value::Object(obj) => { let format = obj.get("format").and_then(Value::as_str); if format != Some(REDUCED_FORMAT) { return Ok(LoadedFixture::RawObject { body: text }); } // Integrity: the reduced file must describe exactly the // shape the caller thinks it is. let expect = |field: &str| -> Result<&str, String> { obj.get(field).and_then(Value::as_str).ok_or_else(|| { format!( "reduced fixture {} is missing field {field:?}", path.display() ) }) }; let file_id = expect("id")?; let raw_query = expect("raw_query")?; let kind = expect("type")?; let status = obj .get("recorded_status") .and_then(Value::as_u64) .ok_or_else(|| { format!( "reduced fixture {} is missing numeric recorded_status", path.display() ) })?; if file_id != shape.id || raw_query != shape.raw_query || kind != shape.kind || status != u64::from(shape.recorded_status) { return Err(format!( "reduced fixture {} self-describes as (id={file_id:?}, \ raw_query={raw_query:?}, type={kind:?}, \ recorded_status={status}) but was loaded for \ (id={:?}, raw_query={:?}, type={:?}, \ recorded_status={})", path.display(), shape.id, shape.raw_query, shape.kind, shape.recorded_status )); } let pairs = |field: &str| -> Result, String> { let arr = obj.get(field).and_then(Value::as_array).ok_or_else(|| { format!( "reduced fixture {} is missing array field {field:?}", path.display() ) })?; arr.iter() .map(|pair| { let pair = pair.as_array().ok_or_else(|| { format!( "reduced fixture {}: {field} entry is not an array", path.display() ) })?; if pair.len() != 2 { return Err(format!( "reduced fixture {}: {field} entry does not have exactly 2 elements", path.display() )); } Ok(( pair[0].as_str().map(str::to_string), pair[1].as_str().map(str::to_string), )) }) .collect() }; let identifiers = match obj.get("identifiers").and_then(Value::as_array) { Some(_) => pairs("identifiers")?, None => Vec::new(), }; let top10 = match obj.get("top10").and_then(Value::as_array) { Some(_) => pairs("top10")?, None => Vec::new(), }; let diagnostic = match obj.get("diagnostic") { Some(_) => Some( serde_json::to_string(obj.get("diagnostic").unwrap()).map_err(|e| { format!( "reduced fixture {}: diagnostic not serialisable: {e}", path.display() ) })?, ), None => None, }; if let Some(hit_count) = obj.get("hit_count").and_then(Value::as_u64) { if hit_count != identifiers.len() as u64 { return Err(format!( "reduced fixture {}: hit_count {hit_count} != identifiers.len() {}", path.display(), identifiers.len() )); } } Ok(LoadedFixture::Reduced { identifiers, top10, diagnostic, }) } other => Err(format!( "fixture {} is not a top-level JSON array or object (got {}); \ refusing to grade against silent empty ground truth", path.display(), super::json_type_name(&other), )), } } /// Recorded identifier set (match shapes). pub fn recorded_set(&self) -> BTreeSet { match self { LoadedFixture::Full { hits } => super::comparator::identifier_set(hits), LoadedFixture::Reduced { identifiers, .. } => identifiers.iter().cloned().collect(), LoadedFixture::RawObject { .. } => BTreeSet::new(), } } /// First-ten recorded keys (clause (b)); order matters for WHICH ten. pub fn recorded_top10(&self) -> Vec { match self { LoadedFixture::Full { hits } => super::comparator::top10_keys(hits), LoadedFixture::Reduced { top10, .. } => top10.clone(), LoadedFixture::RawObject { .. } => Vec::new(), } } /// Recorded set size (the clause-(b) gate and report count). pub fn recorded_len(&self) -> usize { self.recorded_set().len() } /// The body the gn3 stub serves: full/raw verbatim; reduced /// reconstructs the hit array as top10 first (recorded order), /// then the remaining identifiers in stored (sorted) order — so /// set and top10 grading both see exactly what the recording had. pub fn stub_body(&self) -> String { match self { LoadedFixture::Full { hits } => { serde_json::to_string(hits).expect("full fixture array serialisable") } LoadedFixture::RawObject { body } => body.clone(), LoadedFixture::Reduced { identifiers, top10, diagnostic, } => { if let Some(body) = diagnostic { return body.clone(); } let top_set: BTreeSet = top10.iter().cloned().collect(); let mut hits: Vec = top10.iter().map(|(d, n)| hit_object(d, n)).collect(); hits.extend( identifiers .iter() .filter(|k| !top_set.contains(*k)) .map(|(d, n)| hit_object(d, n)), ); serde_json::to_string(&json!(hits)) .expect("reduced reconstruction serialisable") } } } } fn hit_object(dataset: &Option, name: &Option) -> Value { json!({ "dataset": dataset, "name": name }) } #[cfg(test)] mod fixture_tests { use super::*; use std::path::PathBuf; fn shape_ref<'a>( id: &'a str, raw_query: &'a str, kind: &'a str, status: u16, ) -> ShapeRef<'a> { ShapeRef { id, raw_query, kind, recorded_status: status, } } fn write_fixture(dir: &Path, id: &str, body: &str) -> PathBuf { std::fs::create_dir_all(dir).unwrap(); let path = dir.join(format!("{id}.json")); std::fs::write(&path, body).unwrap(); path } fn tmpdir(tag: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!("gn4-parity-fixture-{tag}")); let _ = std::fs::remove_dir_all(&dir); dir } #[test] fn full_array_fixture_loads_and_serves_verbatim() { let dir = tmpdir("full"); write_fixture( &dir, "Q01", r#"[{"dataset":"d1","name":"n1"},{"dataset":"d2","name":"n2"}]"#, ); let lf = LoadedFixture::load(&dir, &shape_ref("Q01", "q", "phenotype", 200)).unwrap(); assert_eq!(lf.recorded_len(), 2); assert_eq!(lf.recorded_top10().len(), 2); let served: Value = serde_json::from_str(&lf.stub_body()).unwrap(); let served = served.as_array().unwrap(); assert_eq!(served[0]["dataset"], "d1"); } #[test] fn raw_object_fixture_serves_verbatim() { let dir = tmpdir("raw"); write_fixture(&dir, "Q51", r#"{"error":"Syntax: ...","error_type":"x"}"#); let lf = LoadedFixture::load(&dir, &shape_ref("Q51", "q", "phenotype", 400)).unwrap(); let body = lf.stub_body(); let parsed: Value = serde_json::from_str(&body).unwrap(); assert_eq!(parsed["error"], "Syntax: ..."); } #[test] fn reduced_fixture_loads_with_intact_top10_order() { let dir = tmpdir("reduced"); write_fixture( &dir, "Q03", r#"{ "format": "reduced-v1", "id": "Q03", "raw_query": "alcohol dependence", "type": "phenotype", "recorded_status": 200, "hit_count": 3, "identifiers": [["dA","nA"],["dB","nB"],["dC","nC"]], "top10": [["dC","nC"],["dA","nA"],["dB","nB"]] }"#, ); let lf = LoadedFixture::load( &dir, &shape_ref("Q03", "alcohol dependence", "phenotype", 200), ) .unwrap(); assert_eq!(lf.recorded_len(), 3); // top10 preserves RECORDED order, not sorted order: let top = lf.recorded_top10(); assert_eq!(top[0], (Some("dC".into()), Some("nC".into()))); assert_eq!(top[2], (Some("dB".into()), Some("nB".into()))); } #[test] fn reduced_stub_body_serves_top10_first_then_rest_sorted() { let dir = tmpdir("reduced2"); write_fixture( &dir, "Q03", r#"{ "format": "reduced-v1", "id": "Q03", "raw_query": "alcohol dependence", "type": "phenotype", "recorded_status": 200, "hit_count": 4, "identifiers": [["dA","nA"],["dB","nB"],["dC","nC"],["dD","nD"]], "top10": [["dD","nD"],["dA","nA"]] }"#, ); let lf = LoadedFixture::load( &dir, &shape_ref("Q03", "alcohol dependence", "phenotype", 200), ) .unwrap(); let body: Value = serde_json::from_str(&lf.stub_body()).unwrap(); let hits = body.as_array().unwrap(); // First two served are the top10, in recorded order: assert_eq!(hits[0]["dataset"], "dD"); assert_eq!(hits[1]["dataset"], "dA"); // The rest follow in stored (sorted) order, no duplicates: assert_eq!(hits[2]["dataset"], "dB"); assert_eq!(hits[3]["dataset"], "dC"); assert_eq!(hits.len(), 4); } #[test] fn reduced_fixture_with_mismatched_shape_fails_loudly() { let dir = tmpdir("mismatch"); write_fixture( &dir, "Q01", r#"{ "format": "reduced-v1", "id": "Q01", "raw_query": "WRONG QUERY", "type": "phenotype", "recorded_status": 200, "hit_count": 0, "identifiers": [], "top10": [] }"#, ); let err = LoadedFixture::load(&dir, &shape_ref("Q01", "alcohol", "phenotype", 200)) .unwrap_err(); assert!(err.contains("self-describes"), "got: {err}"); } #[test] fn reduced_fixture_with_wrong_hit_count_fails_loudly() { let dir = tmpdir("hitcount"); write_fixture( &dir, "Q01", r#"{ "format": "reduced-v1", "id": "Q01", "raw_query": "alcohol", "type": "phenotype", "recorded_status": 200, "hit_count": 7, "identifiers": [["d1","n1"]], "top10": [["d1","n1"]] }"#, ); let err = LoadedFixture::load(&dir, &shape_ref("Q01", "alcohol", "phenotype", 200)) .unwrap_err(); assert!(err.contains("hit_count"), "got: {err}"); } #[test] fn scalar_fixture_is_a_loud_error() { let dir = tmpdir("scalar"); write_fixture(&dir, "Q01", "42"); let err = LoadedFixture::load(&dir, &shape_ref("Q01", "alcohol", "phenotype", 200)) .unwrap_err(); assert!(err.contains("refusing to grade"), "got: {err}"); } } } // --------------------------------------------------------------------------- // B.2 Fixture-serving gn3 stub. // --------------------------------------------------------------------------- struct StubState { fixtures_dir: PathBuf, /// shape_id -> number of stub requests served for it. counter: Mutex>, /// forwarded params of requests that matched no shape (route drift). unmatched: Mutex>, } /// Stands in for gn3's `/api/search/`. Keys on the forwarded `(query, type)` /// pair; serves the recorded status plus the verbatim fixture body read from /// disk at request time. Surface shapes are deliberately NOT matchable, so a /// route that started forwarding them hits the loud no-match path. async fn serve_gn3( State(state): State>, Query(params): Query>, ) -> (StatusCode, String) { let query = params.get("query").cloned().unwrap_or_default(); let kind = params.get("type").cloned().unwrap_or_default(); let per_page = params.get("per_page").cloned().unwrap_or_default(); let matched = SHAPES .iter() .find(|s| s.class != Class::Surface400 && s.raw_query == query && s.kind == kind); if let Some(shape) = matched { *state .counter .lock() .unwrap() .entry(shape.id.to_string()) .or_insert(0) += 1; let shape_ref = fixture::ShapeRef { id: shape.id, raw_query: shape.raw_query, kind: shape.kind, recorded_status: shape.recorded_status, }; let path = state.fixtures_dir.join(format!("{}.json", shape.id)); match fixture::LoadedFixture::load(&state.fixtures_dir, &shape_ref) { Ok(lf) => { let status = StatusCode::from_u16(shape.recorded_status).unwrap(); (status, lf.stub_body()) } Err(e) => ( StatusCode::INTERNAL_SERVER_ERROR, json!({ "harness_error": "fixture load failed", "shape_id": shape.id, "path": path.display().to_string(), "detail": e, }) .to_string(), ), } } else { // No shape for this forwarded pair: harness/route drift. Fail // loudly as a 502 backend_error DIFF naming the forwarded params. state.unmatched.lock().unwrap().push(format!( "query={query:?} type={kind:?} per_page={per_page:?}" )); ( StatusCode::INTERNAL_SERVER_ERROR, json!({ "harness_error": "no recorded shape for forwarded params", "query": query, "type": kind, "per_page": per_page, }) .to_string(), ) } } async fn gn3_stub_url(state: Arc) -> String { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let app = Router::new() .route("/api/search/", get(serve_gn3)) .with_state(state); tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); format!("http://127.0.0.1:{port}/api/") } /// Stands in for gn-auth's GET /auth/public-jwks. async fn serve_jwks() -> (StatusCode, &'static str) { (StatusCode::OK, TEST_JWKS) } async fn gn_auth_stub_url() -> String { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let app = Router::new().route("/auth/public-jwks", get(serve_jwks)); tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); format!("http://127.0.0.1:{port}") } // --------------------------------------------------------------------------- // Auth + request plumbing (same pattern as tests/search.rs). // --------------------------------------------------------------------------- fn sign_test_token(issuer: &str) -> String { #[derive(Serialize)] struct Claims { sub: String, scope: String, iss: String, iat: u64, exp: u64, } let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_secs(); let claims = Claims { sub: "search-parity-user".to_string(), scope: "profile".to_string(), iss: issuer.to_string(), iat: now, exp: now + 3600, }; let key = EncodingKey::from_rsa_pem(TEST_PRIVATE_KEY.as_bytes()).unwrap(); encode(&Header::new(Algorithm::RS256), &claims, &key).unwrap() } fn set(key: &str, value: &str) { // SAFETY: this test file is the only place that touches these keys, in // one sequential sequence inside a single test (see module doc). unsafe { std::env::set_var(key, value) }; } /// Percent-encode a URI query component, leaving unreserved bytes. Produces /// `%26%26` for `&&`, `%20` for spaces, `%22` for quotes — the encoding /// care `tests/search.rs` documents. fn encode_component(s: &str) -> String { let mut out = String::with_capacity(s.len()); for b in s.bytes() { match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { out.push(b as char); } _ => out.push_str(&format!("%{b:02X}")), } } out } async fn get_json(uri: &str, token: &str) -> (StatusCode, Value) { let request = Request::builder() .uri(uri) .header("Authorization", format!("Bearer {token}")) .body(Body::empty()) .unwrap(); let response = genenetwork4::app().oneshot(request).await.unwrap(); let status = response.status(); let body = response.into_body().collect().await.unwrap().to_bytes(); let value = serde_json::from_slice(&body).unwrap_or(Value::Null); (status, value) } /// Replay URI: `/search?q=&type=` with no `limit`, so /// the route truncates to its documented default of 20 hits — exactly the /// surface a real caller sees, and enough for clause (b)'s top-10 (the /// first ten in recorded order; the stub serves the recorded top-10 first /// and ignores the forwarded `per_page`, design §B.4). Clause (a) full-set /// grading does not use this response; it calls the engine seam directly /// (see `ENGINE_FETCH_LIMIT`). fn replay_uri(shape: &Shape) -> String { format!( "/search?q={}&type={}", encode_component(shape.raw_query), shape.kind ) } // --------------------------------------------------------------------------- // Report writer (design §E). // --------------------------------------------------------------------------- /// One shape's graded outcome. struct ShapeResult { id: String, raw_query: String, kind: String, table_class: String, effective_class: &'static str, recorded_status_display: String, replayed_status: u16, replayed_code: String, recorded_hits: usize, replayed_hits: usize, missing: usize, extra: usize, /// First `DIFF_ID_LIMIT` missing identifiers (metric §2.2.2 diff detail). missing_ids: Vec, /// First `DIFF_ID_LIMIT` extra identifiers (metric §2.2.2 diff detail). extra_ids: Vec, jaccard: Option, verdict: &'static str, upstream_calls: u32, detail: String, } /// Cap on the identifier lists stored/emitted per shape (metric §2.2.2 /// asks for the diff, not necessarily every member of a 50000-hit shape). const DIFF_ID_LIMIT: usize = 20; /// Render a comparison key for the sidecar / report lines. The debug repr /// of the `Option` pair keeps an absent field (`None`) visibly distinct from /// an empty string (`Some("")`) — the same distinction `hit_key` preserves. fn format_key(key: &comparator::HitKey) -> String { format!("{key:?}") } /// First `DIFF_ID_LIMIT` keys of a set-difference iterator, in set order. fn cap_ids<'a>(iter: impl Iterator) -> Vec { iter.take(DIFF_ID_LIMIT).map(format_key).collect() } impl ShapeResult { fn line(&self) -> String { let j = match self.jaccard { Some(v) => format!("J={v:.3}"), None => "J=—".to_string(), }; let q = format!("q=`{}`", self.raw_query); let mut detail_suffix = String::new(); if !self.detail.is_empty() { detail_suffix = format!(" {}", self.detail); } // Identifier lists are diagnostic detail for DIFF lines only // (metric §2.2.2); PASS lines stay compact. let mut id_suffix = String::new(); if self.verdict == "DIFF" { if !self.missing_ids.is_empty() { id_suffix.push_str(&format!(" missing_ids={}", self.missing_ids.join(", "))); } if !self.extra_ids.is_empty() { id_suffix.push_str(&format!(" extra_ids={}", self.extra_ids.join(", "))); } } match self.effective_class { "200-match" => format!( "{:<4} {:<4} [{:<11}] {:<32} rec={}/{} gn4={}/{} missing={} extra={} {}{}{}", self.verdict, self.id, self.effective_class, q, self.recorded_status_display, self.recorded_hits, self.replayed_status, self.replayed_hits, self.missing, self.extra, j, detail_suffix, id_suffix, ), _ => format!( "{:<4} {:<4} [{:<11}] {:<32} rec={} gn4={} code={}{}{}{}", self.verdict, self.id, self.effective_class, q, self.recorded_status_display, self.replayed_status, self.replayed_code, if self.effective_class == "surface-400" { format!(" upstream_calls={}", self.upstream_calls) } else { String::new() }, detail_suffix, id_suffix, ), } } } struct RunMeta { timestamp: String, fixture_dir: String, reports_dir: String, full_run: bool, filter: String, } fn iso8601_utc(secs: u64) -> String { let days = (secs / 86400) as i64; let rem = secs % 86400; let (h, mi, s) = (rem / 3600, (rem % 3600) / 60, rem % 60); let z = days + 719468; let era = if z >= 0 { z } else { z - 146096 } / 146097; let doe = z - era * 146097; let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; let y = yoe + era * 400; let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); let mp = (5 * doy + 2) / 153; let d = doy - (153 * mp + 2) / 5 + 1; let m = if mp < 10 { mp + 3 } else { mp - 9 }; let y = if m <= 2 { y + 1 } else { y }; format!("{y:04}-{m:02}-{d:02}T{h:02}:{mi:02}:{s:02}Z") } fn write_reports( reports_dir: &Path, meta: &RunMeta, results: &[ShapeResult], floor: &str, floor_failures: &[String], unmatched: &[String], ) -> std::io::Result<(PathBuf, PathBuf)> { std::fs::create_dir_all(reports_dir)?; // --- summary counts by effective class --- let mut md = String::new(); md.push_str("# GN4 search parity report\n\n"); md.push_str(&format!("- run: {}\n", meta.timestamp)); md.push_str(&format!("- fixture_dir: {}\n", meta.fixture_dir)); md.push_str(&format!("- gn4_sha: {GN4_SHA}\n")); if !meta.full_run { md.push_str(&format!( "- **NON-FULL-RUN**: GN4_PARITY_ONLY={} ({} of {} shapes)\n", meta.filter, results.len(), SHAPES.len() )); } md.push_str("\n## Per-shape\n\n```\n"); for r in results { md.push_str(&r.line()); md.push('\n'); } if !unmatched.is_empty() { md.push_str("\n# UNEXPECTED STUB HITS (route drift)\n"); for u in unmatched { md.push_str(&format!("DIFF unmatched forwarded params: {u}\n")); } } md.push_str("```\n\n## Summary\n\n"); let count = |pred: &dyn Fn(&ShapeResult) -> bool| results.iter().filter(|r| pred(r)).count(); let pass = |pred: &dyn Fn(&ShapeResult) -> bool| { results .iter() .filter(|r| pred(r) && r.verdict == "PASS") .count() }; let is_match = |r: &ShapeResult| r.effective_class == "200-match"; let is_clause_b = |r: &ShapeResult| is_match(r) && r.jaccard.is_some(); let match_n = count(&is_match); let match_pass = pass(&is_match); let cb_n = count(&is_clause_b); let cb_pass = pass(&is_clause_b); let min_j = results .iter() .filter_map(|r| r.jaccard) .fold(f64::INFINITY, f64::min); md.push_str(&format!( "- 200-match: {match_pass} pass / {} diff (of which clause-(b) shapes: {cb_pass} pass / {} diff, min J = {})\n", match_n - match_pass, cb_n - cb_pass, if min_j.is_finite() { format!("{min_j:.3}") } else { "—".into() }, )); for cls in ["error-4xx", "error-500", "surface-400"] { let sel = |r: &ShapeResult| r.effective_class == cls; md.push_str(&format!( "- {cls}: {} pass / {} diff\n", pass(&sel), count(&sel) - pass(&sel), )); } let zero_hit = results .iter() .filter(|r| is_match(r) && r.recorded_hits == 0) .map(|r| r.id.as_str()) .collect::>(); md.push_str(&format!( "- zero-hit 200-match shapes: {}\n", if zero_hit.is_empty() { "—".into() } else { zero_hit.join(", ") } )); md.push_str(&format!("\n**FLOOR: {floor}**\n")); if !floor_failures.is_empty() { md.push_str(&format!("blocking shapes: {}\n", floor_failures.join(", "))); } let md_path = reports_dir.join("parity-report.md"); std::fs::write(&md_path, md)?; // --- JSON sidecar --- let shapes_json: Vec = results .iter() .map(|r| { json!({ "id": r.id, "raw_query": r.raw_query, "type": r.kind, "table_class": r.table_class, "effective_class": r.effective_class, "recorded_status": r.recorded_status_display, "replayed_status": r.replayed_status, "replayed_code": r.replayed_code, "recorded_hits": r.recorded_hits, "replayed_hits": r.replayed_hits, "missing": r.missing, "extra": r.extra, "missing_ids": r.missing_ids, "extra_ids": r.extra_ids, "jaccard": r.jaccard, "verdict": r.verdict, "upstream_calls": r.upstream_calls, "detail": r.detail, }) }) .collect(); let sidecar = json!({ "run": { "timestamp": meta.timestamp, "fixture_dir": meta.fixture_dir, "reports_dir": meta.reports_dir, "gn4_sha": GN4_SHA, "full_run": meta.full_run, "filter": meta.filter, }, "shapes": shapes_json, "summary": { "total": results.len(), "match200_total": match_n, "match200_pass": match_pass, "clause_b_total": cb_n, "clause_b_pass": cb_pass, "min_jaccard": if min_j.is_finite() { Some(min_j) } else { None }, }, "unmatched_stub_hits": unmatched, "floor": floor, "floor_failures": floor_failures, }); let json_path = reports_dir.join("parity-report.json"); std::fs::write(&json_path, serde_json::to_string_pretty(&sidecar).unwrap())?; Ok((md_path, json_path)) } // --------------------------------------------------------------------------- // Replay driver + grading. // --------------------------------------------------------------------------- fn reports_dir() -> PathBuf { if let Ok(dir) = std::env::var("GN4_PARITY_REPORTS") { return PathBuf::from(dir); } PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("target/parity-reports") } /// Name of a JSON value's top-level type, for panic messages. fn json_type_name(v: &Value) -> &'static str { match v { Value::Null => "null", Value::Bool(_) => "bool", Value::Number(_) => "number", Value::String(_) => "string", Value::Array(_) => "array", Value::Object(_) => "object", } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn search_parity_replay() { let in_repo = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test/data/parity-fixtures"); let fixtures_dir = match std::env::var("GN4_PARITY_FIXTURES") { Ok(dir) if Path::new(&dir).is_dir() => PathBuf::from(dir), Ok(dir) => { eprintln!("skipping search_parity: GN4_PARITY_FIXTURES={dir} is not a directory"); return; } Err(_) if in_repo.is_dir() => in_repo, Err(_) => { eprintln!( "skipping search_parity: no GN4_PARITY_FIXTURES and no committed corpus at {}", in_repo.display() ); return; } }; let only = std::env::var("GN4_PARITY_ONLY") .ok() .filter(|s| !s.trim().is_empty()); let selected: Vec<&Shape> = match &only { Some(filter) => { let ids: Vec<&str> = filter .split(',') .map(str::trim) .filter(|s| !s.is_empty()) .collect(); SHAPES.iter().filter(|s| ids.contains(&s.id)).collect() } None => SHAPES.iter().collect(), }; if selected.is_empty() { eprintln!("skipping search_parity: GN4_PARITY_ONLY selected no shapes"); return; } let full_run = only.is_none() && selected.len() == SHAPES.len(); // --- stubs + env --- let state = Arc::new(StubState { fixtures_dir: fixtures_dir.clone(), counter: Mutex::new(HashMap::new()), unmatched: Mutex::new(Vec::new()), }); let gn3_stub = gn3_stub_url(state.clone()).await; let gn_auth_stub = gn_auth_stub_url().await; set("GN3_URL", &gn3_stub); set("AUTH_SERVER_URL", &gn_auth_stub); // Replaying recorded gn3 responses only means anything through the // proxy: pin the engine, or an ambient SEARCH_ENGINE=native (as the dev // box sets) answers from the local index and makes no upstream calls. set("SEARCH_ENGINE", "forward"); let token = sign_test_token(&gn_auth_stub); // --- sequential replay --- let mut results: Vec = Vec::new(); let mut floor_failures: Vec = Vec::new(); for shape in &selected { let (status, body) = get_json(&replay_uri(shape), &token).await; let replayed_status = status.as_u16(); let replayed_code = body .get("code") .and_then(Value::as_str) .unwrap_or("") .to_string(); let upstream_calls = state .counter .lock() .unwrap() .get(shape.id) .copied() .unwrap_or(0); let mut result = ShapeResult { id: shape.id.to_string(), raw_query: shape.raw_query.to_string(), kind: shape.kind.to_string(), table_class: shape.table_class.to_string(), effective_class: shape.class.label(), recorded_status_display: recorded_status_display(shape), replayed_status, replayed_code: replayed_code.clone(), recorded_hits: 0, replayed_hits: 0, missing: 0, extra: 0, missing_ids: Vec::new(), extra_ids: Vec::new(), jaccard: None, verdict: "DIFF", upstream_calls, detail: String::new(), }; match shape.class { Class::Match200 => { let shape_ref = fixture::ShapeRef { id: shape.id, raw_query: shape.raw_query, kind: shape.kind, recorded_status: shape.recorded_status, }; let lf = fixture::LoadedFixture::load(&fixtures_dir, &shape_ref) .unwrap_or_else(|e| panic!("shape {}: {e}", shape.id)); // A match shape whose fixture carries an error body would // grade against silently-empty ground truth — refuse. match &lf { fixture::LoadedFixture::Full { .. } | fixture::LoadedFixture::Reduced { diagnostic: None, .. } => {} _ => panic!( "shape {}: fixture is an error body, not a match body; refusing to grade", shape.id ), } let expected = lf.recorded_set(); result.recorded_hits = lf.recorded_len(); if replayed_status != 200 { result.detail = format!("expected 200, got {replayed_status} {replayed_code}"); } else { // Clause (b) stays ROUTE-level: the top-10 of the // replayed (limit-truncated) response. The first ten // hits in recorded order survive the truncate, so the // clause keeps its exact prior meaning on exactly the // response real callers receive. let route_hits: Vec = body .get("data") .and_then(Value::as_array) .cloned() .unwrap_or_default(); // Clause (a) moves to the ENGINE seam, below the // route's `hits.truncate`: call the engine directly, // in process, against the same stub gn3, with a limit // large enough to retrieve the full match set // (PORT-DESIGN §5). This is the shape's second stub // request (route replay was the first). let engine = genenetwork4::search::search( shape.raw_query, shape.kind, ENGINE_FETCH_LIMIT, 1, ) .await; // Re-read the stub counter AFTER the engine call so the // count covers both requests (route + engine). let upstream_calls = state .counter .lock() .unwrap() .get(shape.id) .copied() .unwrap_or(0); result.upstream_calls = upstream_calls; let mut ok = true; match engine { Ok(hits) => { result.replayed_hits = hits.len(); let actual: BTreeSet = hits .iter() .map(|h| (h.dataset.clone(), h.name.clone())) .collect(); let grade = comparator::grade_match(&expected, &actual); result.missing = grade.missing; result.extra = grade.extra; result.missing_ids = cap_ids(expected.difference(&actual)); result.extra_ids = cap_ids(actual.difference(&expected)); if !grade.equal { ok = false; } } Err(e) => { ok = false; result.detail = format!("engine-level search failed: {e:?}"); } } if comparator::clause_b_applies(expected.len()) { let a: BTreeSet<_> = comparator::top10_keys(&route_hits).into_iter().collect(); let b: BTreeSet<_> = lf.recorded_top10().into_iter().collect(); let j = comparator::jaccard(&a, &b); result.jaccard = Some(j); if !comparator::clause_b_pass(expected.len(), j).unwrap() { ok = false; if !result.detail.is_empty() { result.detail.push_str("; "); } result .detail .push_str(&format!("clause (b) J={j:.3} below floor")); } } if upstream_calls != 2 { ok = false; if !result.detail.is_empty() { result.detail.push_str("; "); } result.detail.push_str(&format!( "expected exactly 2 upstream calls (route + engine), got {upstream_calls}" )); } result.verdict = if ok { "PASS" } else { "DIFF" }; } } Class::Error4xx | Class::Error500 | Class::Surface400 => { let recorded = match shape.class { Class::Error4xx => comparator::RecordedStatus::Client4xx, Class::Error500 => comparator::RecordedStatus::Server500, _ => comparator::RecordedStatus::Surface, }; let expected_calls = if shape.class == Class::Surface400 { 0 } else { 1 }; let grade = comparator::grade_error_class( recorded, replayed_status, &replayed_code, upstream_calls > 0, ); let mut ok = grade.is_pass(); if let comparator::ClassGrade::Fail(msg) = grade { result.detail = msg; } if upstream_calls != expected_calls { ok = false; if !result.detail.is_empty() { result.detail.push_str("; "); } result.detail.push_str(&format!( "expected {expected_calls} upstream calls, got {upstream_calls}" )); } result.verdict = if ok { "PASS" } else { "DIFF" }; } } if result.verdict != "PASS" { floor_failures.push(shape.id.to_string()); } println!("{}", result.line()); results.push(result); } // --- verdict conjunction (metric §2.2.5) --- let unmatched = state.unmatched.lock().unwrap().clone(); for u in &unmatched { floor_failures.push(format!("unmatched-stub:{u}")); } let floor = if floor_failures.is_empty() { "PASS" } else { "BLOCKED" }; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_secs(); let rdir = reports_dir(); let meta = RunMeta { timestamp: iso8601_utc(now), fixture_dir: fixtures_dir.display().to_string(), reports_dir: rdir.display().to_string(), full_run, filter: only.clone().unwrap_or_default(), }; let (md_path, json_path) = write_reports(&rdir, &meta, &results, floor, &floor_failures, &unmatched) .expect("write parity report"); eprintln!( "parity report written: {} and {}", md_path.display(), json_path.display() ); if !full_run { eprintln!("NON-FULL-RUN: GN4_PARITY_ONLY={}", meta.filter); } assert!( floor_failures.is_empty(), "FLOOR: BLOCKED ({} failures): {} — see {}", floor_failures.len(), floor_failures.join(", "), md_path.display() ); }