//! Dev-floor corpus recorder (the lane's bonus stage): records all 54 //! shapes from the DEVELOPMENT index (`~/spikes/index-self-build/xapian- //! index`, 47 040 docs, 2025-02-18 public data) through the NATIVE //! engine — the same parser configuration as `src/search/native.rs` by //! construction, because it IS that engine — producing a full dev corpus //! in the production recordings' format (a top-level JSON array of hit //! objects per match shape; a gn3-style diagnostic object per error //! shape). The committed `reduce_fixtures.py` then reduces it, and the //! parity harness's floor runs against the reduced dev corpus as a //! SEPARATE report (GN4_PARITY_FIXTURES; the production corpus is never //! touched). //! //! THE NUMBERS ARE DEV-TRUTH, NOT PRODUCTION COUNTS: the dev index is //! built from the 2025-02-18 public database snapshot, so e.g. Q01's //! recorded dev count differs from the production fixture's 1197. The //! dev corpus is the dev-oracle: fully local, no production access, //! pinning that the harness passes 54/54 against recordings the engine //! itself produced — determinism, replay-stability and the grading //! pipeline, not production parity (production parity stays the //! production floor's job). //! //! The shape table is parsed from the harness source itself //! (include_str!("search_parity.rs")), the same no-drift discipline as //! reduce_fixtures.py — the recorder can never record a shape the //! harness does not grade. //! //! Env (skips loudly when unset, so hermetic runs stay green): //! - `XAPIAN_DB_PATH` — the dev index directory; //! - `GN4_DEV_CORPUS_DIR` — output directory for the full corpus. use genenetwork4::search::{Engine, SearchError, SearchHit, native::Native}; /// One row of the harness's shape table, parsed from its source. struct RecordedShape { id: String, raw_query: String, kind: String, /// "Match200" | "Error4xx" | "Error500" | "Surface400". class: String, } /// Unescape the two Rust escapes the harness table uses (`\\` and /// `\"`) — the same rule reduce_fixtures.py applies. fn unescape(raw: &str) -> String { let mut out = String::new(); let mut chars = raw.chars(); while let Some(c) = chars.next() { if c == '\\' { if let Some(next) = chars.next() { out.push(next); } } else { out.push(c); } } out } /// Parse the `shape!("Q01", "alcohol", "phenotype", "C-match", 200, /// Class::Match200)` rows out of the harness source. Fails loudly on /// any row that does not carry exactly four strings plus a /// `Class::…` token, and on any total other than 54. fn parse_shapes(harness_source: &str) -> Vec { let mut shapes = Vec::new(); let mut search_from = 0; while let Some(found) = harness_source[search_from..].find("shape!(") { let args_start = search_from + found + "shape!(".len(); // Find the macro call's closing paren, skipping string literals // (shape queries like Q46's "(alcohol OR insulin) AND ..." carry // parens INSIDE their strings). let bytes = harness_source.as_bytes(); let mut j = args_start; let mut in_string = false; while j < bytes.len() { if in_string { if bytes[j] == b'\\' { j += 2; continue; } if bytes[j] == b'"' { in_string = false; } } else if bytes[j] == b'"' { in_string = true; } else if bytes[j] == b')' { break; } j += 1; } assert!(j < bytes.len(), "shape! call closed"); let args_end = j; let args = &harness_source[args_start..args_end]; let mut strings = Vec::new(); let mut class = String::new(); let mut i = 0; while i < args.len() { let c = args[i..].chars().next().expect("in-bounds char"); if c == '"' { // Slice the escaped-raw literal, then unescape it. let mut j = i + 1; while j < args.len() { if args.as_bytes()[j] == b'\\' && j + 1 < args.len() { j += 2; } else if args.as_bytes()[j] == b'"' { break; } else { j += 1; } } assert!(j < args.len(), "unterminated string in shape!: {args:?}"); strings.push(unescape(&args[i + 1..j])); i = j + 1; } else if args[i..].starts_with("Class::") { let start = i + "Class::".len(); let end = args[start..] .find(|c: char| !c.is_ascii_alphanumeric()) .map(|offset| start + offset) .unwrap_or(args.len()); class = args[start..end].to_string(); i = end; } else { i += c.len_utf8(); } } assert_eq!( strings.len(), 4, "shape! row must carry id, raw_query, type, table_class: {args:?}" ); assert!(!class.is_empty(), "shape! row must name a Class: {args:?}"); shapes.push(RecordedShape { id: strings[0].clone(), raw_query: strings[1].clone(), kind: strings[2].clone(), class, }); search_from = args_end + 1; } assert_eq!( shapes.len(), 54, "the harness table must yield 54 shapes, got {}", shapes.len() ); shapes } /// The fetch limit for recording: the engine ignores it (per-page /// invariant) but the value mirrors the harness's ENGINE_FETCH_LIMIT. const RECORD_FETCH_LIMIT: u64 = 1_000_000; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn record_dev_corpus() { let out_dir = std::env::var("GN4_DEV_CORPUS_DIR").unwrap_or_default(); let index = std::env::var("XAPIAN_DB_PATH").unwrap_or_default(); if out_dir.is_empty() || index.is_empty() || !std::fs::metadata(&index) .map(|m| m.is_dir()) .unwrap_or(false) { eprintln!( "SKIPPED: GN4_DEV_CORPUS_DIR and XAPIAN_DB_PATH must both be set \ (recording the dev corpus needs the dev index)" ); return; } let shapes = parse_shapes(include_str!("search_parity.rs")); std::fs::create_dir_all(&out_dir).expect("create corpus dir"); eprintln!("recording the dev corpus into {out_dir} from index {index}"); let mut recorded = 0usize; for shape in &shapes { let path = std::path::Path::new(&out_dir).join(format!("{}.json", shape.id)); let search = Native .search(&shape.raw_query, &shape.kind, RECORD_FETCH_LIMIT, 1) .await; match shape.class.as_str() { // Match shapes (and the surface shapes' recorded gn3 200 [], // which the engine reproduces: an empty/whitespace query // parses to an empty query, matching nothing): a JSON array // of hit bodies, the production recordings' format. "Match200" | "Surface400" => { let hits = search.unwrap_or_else(|e| { panic!( "{} ({:?}, {}): match shape must record Ok, got {e:?}", shape.id, shape.raw_query, shape.kind ) }); std::fs::write(&path, serde_json::to_string(&hits).expect("hits serialise")) .expect("write corpus file"); eprintln!( " {} q={:?} type={} recorded {} dev hits", shape.id, shape.raw_query, shape.kind, hits.len() ); } // Error shapes: a gn3-style diagnostic object ({"error", // "error_type"}), so the stub can serve it verbatim and the // forwarder's error mapping grades the class exactly as the // production error bodies do. "Error4xx" | "Error500" => match search { Err(SearchError::InvalidQuery { message, xapian_error_type, }) => { let body = serde_json::json!({ "error": message, "error_type": xapian_error_type, }); std::fs::write(&path, body.to_string()).expect("write corpus file"); eprintln!( " {} q={:?} type={} recorded diagnostic {:?}", shape.id, shape.raw_query, shape.kind, message ); } other => panic!( "{} ({:?}, {}): error shape must record InvalidQuery, got {:?}", shape.id, shape.raw_query, shape.kind, variant_name(&other) ), }, other => panic!("{}: unknown class {other:?}", shape.id), } // The recorded hits must carry the identifier fields the parity // comparison (and reduce_fixtures.py) require — a loud refusal, // never a silently-unusable recording. if let Ok(body) = serde_json::from_str::( &std::fs::read_to_string(&path).expect("read back"), ) { if let Some(hits) = body.as_array() { for (idx, hit) in hits.iter().enumerate() { assert!( hit.get("dataset") .and_then(serde_json::Value::as_str) .is_some() && hit .get("name") .and_then(serde_json::Value::as_str) .is_some(), "{}: hit {idx} lacks (dataset, name) strings: {hit:?}", shape.id ); } } } recorded += 1; } assert_eq!(recorded, 54, "all 54 shapes recorded"); eprintln!("DEV CORPUS RECORDED: {recorded} shapes into {out_dir}"); } /// The name of a result's variant, for the loud mismatch panic above /// (`SearchHit` does not implement `Debug`). fn variant_name(result: &Result, SearchError>) -> &'static str { match result { Ok(_) => "Ok", Err(SearchError::InvalidQuery { .. }) => "InvalidQuery", Err(SearchError::Unreachable(_)) => "Unreachable", Err(SearchError::Misbehaving(_)) => "Misbehaving", Err(SearchError::Backend(_)) => "Backend", Err(SearchError::Internal(_)) => "Internal", } }