about summary refs log tree commit diff
path: root/tests/search_native_dev_corpus.rs
blob: 66ed86aa1ba27f38db479b675b02fbdc6511d77f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
//! Dev-floor corpus recorder (the lane's bonus stage): records all 54
//! shapes from the DEVELOPMENT index (`~/spikes/index-self-build/xapian-
//! index`, 47 040 docs, 2025-02-18 public data) through the NATIVE
//! engine — the same parser configuration as `src/search/native.rs` by
//! construction, because it IS that engine — producing a full dev corpus
//! in the production recordings' format (a top-level JSON array of hit
//! objects per match shape; a gn3-style diagnostic object per error
//! shape). The committed `reduce_fixtures.py` then reduces it, and the
//! parity harness's floor runs against the reduced dev corpus as a
//! SEPARATE report (GN4_PARITY_FIXTURES; the production corpus is never
//! touched).
//!
//! THE NUMBERS ARE DEV-TRUTH, NOT PRODUCTION COUNTS: the dev index is
//! built from the 2025-02-18 public database snapshot, so e.g. Q01's
//! recorded dev count differs from the production fixture's 1197. The
//! dev corpus is the dev-oracle: fully local, no production access,
//! pinning that the harness passes 54/54 against recordings the engine
//! itself produced — determinism, replay-stability and the grading
//! pipeline, not production parity (production parity stays the
//! production floor's job).
//!
//! The shape table is parsed from the harness source itself
//! (include_str!("search_parity.rs")), the same no-drift discipline as
//! reduce_fixtures.py — the recorder can never record a shape the
//! harness does not grade.
//!
//! Env (skips loudly when unset, so hermetic runs stay green):
//!   - `XAPIAN_DB_PATH` — the dev index directory;
//!   - `GN4_DEV_CORPUS_DIR` — output directory for the full corpus.

use genenetwork4::search::{Engine, SearchError, SearchHit, native::Native};

/// One row of the harness's shape table, parsed from its source.
struct RecordedShape {
    id: String,
    raw_query: String,
    kind: String,
    /// "Match200" | "Error4xx" | "Error500" | "Surface400".
    class: String,
}

/// Unescape the two Rust escapes the harness table uses (`\\` and
/// `\"`) — the same rule reduce_fixtures.py applies.
fn unescape(raw: &str) -> String {
    let mut out = String::new();
    let mut chars = raw.chars();
    while let Some(c) = chars.next() {
        if c == '\\' {
            if let Some(next) = chars.next() {
                out.push(next);
            }
        } else {
            out.push(c);
        }
    }
    out
}

/// Parse the `shape!("Q01", "alcohol", "phenotype", "C-match", 200,
/// Class::Match200)` rows out of the harness source. Fails loudly on
/// any row that does not carry exactly four strings plus a
/// `Class::…` token, and on any total other than 54.
fn parse_shapes(harness_source: &str) -> Vec<RecordedShape> {
    let mut shapes = Vec::new();
    let mut search_from = 0;
    while let Some(found) = harness_source[search_from..].find("shape!(") {
        let args_start = search_from + found + "shape!(".len();
        // Find the macro call's closing paren, skipping string literals
        // (shape queries like Q46's "(alcohol OR insulin) AND ..." carry
        // parens INSIDE their strings).
        let bytes = harness_source.as_bytes();
        let mut j = args_start;
        let mut in_string = false;
        while j < bytes.len() {
            if in_string {
                if bytes[j] == b'\\' {
                    j += 2;
                    continue;
                }
                if bytes[j] == b'"' {
                    in_string = false;
                }
            } else if bytes[j] == b'"' {
                in_string = true;
            } else if bytes[j] == b')' {
                break;
            }
            j += 1;
        }
        assert!(j < bytes.len(), "shape! call closed");
        let args_end = j;
        let args = &harness_source[args_start..args_end];
        let mut strings = Vec::new();
        let mut class = String::new();
        let mut i = 0;
        while i < args.len() {
            let c = args[i..].chars().next().expect("in-bounds char");
            if c == '"' {
                // Slice the escaped-raw literal, then unescape it.
                let mut j = i + 1;
                while j < args.len() {
                    if args.as_bytes()[j] == b'\\' && j + 1 < args.len() {
                        j += 2;
                    } else if args.as_bytes()[j] == b'"' {
                        break;
                    } else {
                        j += 1;
                    }
                }
                assert!(j < args.len(), "unterminated string in shape!: {args:?}");
                strings.push(unescape(&args[i + 1..j]));
                i = j + 1;
            } else if args[i..].starts_with("Class::") {
                let start = i + "Class::".len();
                let end = args[start..]
                    .find(|c: char| !c.is_ascii_alphanumeric())
                    .map(|offset| start + offset)
                    .unwrap_or(args.len());
                class = args[start..end].to_string();
                i = end;
            } else {
                i += c.len_utf8();
            }
        }
        assert_eq!(
            strings.len(),
            4,
            "shape! row must carry id, raw_query, type, table_class: {args:?}"
        );
        assert!(!class.is_empty(), "shape! row must name a Class: {args:?}");
        shapes.push(RecordedShape {
            id: strings[0].clone(),
            raw_query: strings[1].clone(),
            kind: strings[2].clone(),
            class,
        });
        search_from = args_end + 1;
    }
    assert_eq!(
        shapes.len(),
        54,
        "the harness table must yield 54 shapes, got {}",
        shapes.len()
    );
    shapes
}

/// The fetch limit for recording: the engine ignores it (per-page
/// invariant) but the value mirrors the harness's ENGINE_FETCH_LIMIT.
const RECORD_FETCH_LIMIT: u64 = 1_000_000;

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn record_dev_corpus() {
    let out_dir = std::env::var("GN4_DEV_CORPUS_DIR").unwrap_or_default();
    let index = std::env::var("XAPIAN_DB_PATH").unwrap_or_default();
    if out_dir.is_empty()
        || index.is_empty()
        || !std::fs::metadata(&index)
            .map(|m| m.is_dir())
            .unwrap_or(false)
    {
        eprintln!(
            "SKIPPED: GN4_DEV_CORPUS_DIR and XAPIAN_DB_PATH must both be set \
             (recording the dev corpus needs the dev index)"
        );
        return;
    }
    let shapes = parse_shapes(include_str!("search_parity.rs"));
    std::fs::create_dir_all(&out_dir).expect("create corpus dir");
    eprintln!("recording the dev corpus into {out_dir} from index {index}");

    let mut recorded = 0usize;
    for shape in &shapes {
        let path = std::path::Path::new(&out_dir).join(format!("{}.json", shape.id));
        let search = Native
            .search(&shape.raw_query, &shape.kind, RECORD_FETCH_LIMIT, 1)
            .await;
        match shape.class.as_str() {
            // Match shapes (and the surface shapes' recorded gn3 200 [],
            // which the engine reproduces: an empty/whitespace query
            // parses to an empty query, matching nothing): a JSON array
            // of hit bodies, the production recordings' format.
            "Match200" | "Surface400" => {
                let hits = search.unwrap_or_else(|e| {
                    panic!(
                        "{} ({:?}, {}): match shape must record Ok, got {e:?}",
                        shape.id, shape.raw_query, shape.kind
                    )
                });
                std::fs::write(&path, serde_json::to_string(&hits).expect("hits serialise"))
                    .expect("write corpus file");
                eprintln!(
                    "  {} q={:?} type={} recorded {} dev hits",
                    shape.id,
                    shape.raw_query,
                    shape.kind,
                    hits.len()
                );
            }
            // Error shapes: a gn3-style diagnostic object ({"error",
            // "error_type"}), so the stub can serve it verbatim and the
            // forwarder's error mapping grades the class exactly as the
            // production error bodies do.
            "Error4xx" | "Error500" => match search {
                Err(SearchError::InvalidQuery {
                    message,
                    xapian_error_type,
                }) => {
                    let body = serde_json::json!({
                        "error": message,
                        "error_type": xapian_error_type,
                    });
                    std::fs::write(&path, body.to_string()).expect("write corpus file");
                    eprintln!(
                        "  {} q={:?} type={} recorded diagnostic {:?}",
                        shape.id, shape.raw_query, shape.kind, message
                    );
                }
                other => panic!(
                    "{} ({:?}, {}): error shape must record InvalidQuery, got {:?}",
                    shape.id,
                    shape.raw_query,
                    shape.kind,
                    variant_name(&other)
                ),
            },
            other => panic!("{}: unknown class {other:?}", shape.id),
        }
        // The recorded hits must carry the identifier fields the parity
        // comparison (and reduce_fixtures.py) require — a loud refusal,
        // never a silently-unusable recording.
        if let Ok(body) = serde_json::from_str::<serde_json::Value>(
            &std::fs::read_to_string(&path).expect("read back"),
        ) {
            if let Some(hits) = body.as_array() {
                for (idx, hit) in hits.iter().enumerate() {
                    assert!(
                        hit.get("dataset")
                            .and_then(serde_json::Value::as_str)
                            .is_some()
                            && hit
                                .get("name")
                                .and_then(serde_json::Value::as_str)
                                .is_some(),
                        "{}: hit {idx} lacks (dataset, name) strings: {hit:?}",
                        shape.id
                    );
                }
            }
        }
        recorded += 1;
    }
    assert_eq!(recorded, 54, "all 54 shapes recorded");
    eprintln!("DEV CORPUS RECORDED: {recorded} shapes into {out_dir}");
}

/// The name of a result's variant, for the loud mismatch panic above
/// (`SearchHit` does not implement `Debug`).
fn variant_name(result: &Result<Vec<SearchHit>, SearchError>) -> &'static str {
    match result {
        Ok(_) => "Ok",
        Err(SearchError::InvalidQuery { .. }) => "InvalidQuery",
        Err(SearchError::Unreachable(_)) => "Unreachable",
        Err(SearchError::Misbehaving(_)) => "Misbehaving",
        Err(SearchError::Backend(_)) => "Backend",
        Err(SearchError::Internal(_)) => "Internal",
    }
}