diff options
| author | Alexander_Kabui <alexanderkabua@gmail.com> | 2026-10-07 06:34:51 +0300 |
|---|---|---|
| committer | Alexander_Kabui <alexanderkabua@gmail.com> | 2026-10-07 06:34:51 +0300 |
| commit | a5c40fc78f6b3cb32fba3daeb9f2a8e95558ba09 (patch) | |
| tree | 51dba2ecba0e7371e220b6c6d7eaa99a13b0adf5 /src | |
| parent | f7423aaaf71194a169ded303baa3cb7bda8f0b53 (diff) | |
| download | genenetwork4-a5c40fc78f6b3cb32fba3daeb9f2a8e95558ba09.tar.gz | |
src/search/native.rs (native engine): port the xapian query path behind the Engine seam (C3.1).
New native engine, a Rust port of gn3 search path (gn3/api/search.py),
opt-in via SEARCH_ENGINE=native (the gn3 forwarder stays the default;
src/search/mod.rs now dispatches on config, routes/search.rs delegates):
- startup-cached QueryParser behind a mutex: custom-prefix registration
is Box::leak-based inside the binding, so a per-request parser would
leak; built once per process (binding-report §5 item 3, R2 pattern).
Minimal configuration this commit per PORT-DESIGN C3.1: Stem("en") +
STEM_ALL_Z only.
- checked parse via the fork's parse_query_checked: malformed queries
are typed errors, not panics; Err maps to SearchError::InvalidQuery
(422 invalid_query, the owner-blessed contract), carrying the
parser diagnostic. Parse happens before the index is opened, as in
gn3 (search.py:291 vs :296), so a bad query reports InvalidQuery
even when the index is misconfigured.
- XT{type} document filter: OP_FILTER(parsed, "XT"+type)
(search.py:298-300).
- FULL-match-set retrieval ignoring the forwarded limit (per-page
invariant, PORT-DESIGN §5): mset bound is a deliberately huge fixed
constant; the route keeps its validation and limit logic unchanged.
- per-hit mapping: doc data JSON -> SearchHit via serde
(search.py:302 equivalent); unknown fields flow through extra.
Decode failure maps to SearchError::Misbehaving.
- index open behind an app-level guard (the binding aborts on open
failure; see the module doc): unset XAPIAN_DB_PATH -> Internal,
unopenable path -> Unreachable.
tests/search_native.rs: error-contract scenarios (hermetic) + the
per-page invariant + dispatch pinning, one sequential test fn per the
process-global env rule; the dev-index part skips loudly when
XAPIAN_DB_PATH is unset. tests/search_native_shapes.rs: the shape-CLASS
gate (this lane s AMENDMENT-B translation of the C3.1 stage gate) --
16 shapes (Q01,Q03-Q13,Q47,Q50,Q51,Q54), each expected class cited to
its tests/search_parity.rs shape! line; dev-index counts are printed
as reference data only, never compared to production.
Evidence: logs/c3-1.log (build EXIT=0, native tests green, lib 14/14),
logs/c3-1-full.log (FULL suite EXIT=0, parity floor green),
logs/shape-class-c3-1.log (SHAPE-CLASS GATE 16/16 PASS vs the dev
index; Q51 byte-identical diagnostic; per-page invariant: limit=10
forwarded, 2330-hit full set returned).
Diffstat (limited to 'src')
| -rw-r--r-- | src/routes/search.rs | 15 | ||||
| -rw-r--r-- | src/search/mod.rs | 21 | ||||
| -rw-r--r-- | src/search/native.rs | 250 |
3 files changed, 275 insertions, 11 deletions
diff --git a/src/routes/search.rs b/src/routes/search.rs index 8d4841b..a10f431 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -7,8 +7,8 @@ //! //! The backend itself lives behind the search engine seam (see //! `crate::search`); this handler validates parameters, delegates to -//! the [`Engine`], and maps the engine's [`SearchError`] onto the -//! existing [`ApiError`] shapes. +//! the configured [`Engine`], and maps the engine's [`SearchError`] +//! onto the existing [`ApiError`] shapes. use axum::Json; use axum::extract::{Extension, Query}; @@ -20,7 +20,7 @@ use crate::auth::AuthToken; use crate::error::{ApiError, Unauthorized}; use crate::request_id_string; use crate::response::Envelope; -use crate::search::{Engine, SearchError, SearchHit, gn3::Gn3}; +use crate::search::{self, SearchError, SearchHit}; /// Query parameters for `GET /search`. /// @@ -79,10 +79,12 @@ fn engine_error(error: SearchError, rid: Option<String>) -> ApiError { #[utoipa::path(get, path = "/search", summary = "Search traits across GN datasets", - description = "Proxies GeneNetwork3's xapian search (phase A; see \ + description = "Search traits across GN datasets. Runs through the \ +configured engine: GeneNetwork3's xapian search by default, or the \ +native xapian engine when SEARCH_ENGINE=native (phase A; see \ topics/gn4/gn4-api-foundation.gmi). Hits are gn3's trait objects passed \ through unchanged, wrapped in the descriptive envelope. Malformed \ -queries forward gn3's parser diagnostic as a 422.", +queries carry the backend's parser diagnostic as a 422.", params(SearchParams), responses( (status = 200, description = "Matching traits, gn3 shape", body = Envelope<Vec<SearchHit>>), @@ -120,8 +122,7 @@ pub async fn search( } let limit = params.limit.unwrap_or(20).min(100); - let hits = Gn3 - .search(q, search_type, limit) + let hits = search::search(q, search_type, limit) .await .map_err(|e| engine_error(e, rid))?; diff --git a/src/search/mod.rs b/src/search/mod.rs index e4897ac..936c977 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -78,17 +78,30 @@ //! enrichment is its own private design, noted here only so the effort //! estimate prices the enrichment in. //! -//! # Trait now, one impl +//! # Trait now, two impls //! //! The phase-2 plan originally said "no trait yet" (a trait with one //! impl is speculation). The overnight schedule overrides that: the -//! [`Engine`] trait is defined now so the native engine can land behind -//! it, with the gn3-forwarding implementation as its first (and, until -//! then, only) implementation. +//! [`Engine`] trait is defined so the native engine can land behind +//! it. The gn3-forwarding implementation ([`gn3::Gn3`]) remains the +//! default backend; [`native::Native`] is opt-in via `SEARCH_ENGINE` +//! (see [`search`]). use std::future::Future; pub mod gn3; +pub mod native; + +/// Run a search through the configured engine: the gn3 forwarder by +/// default, the native xapian engine when `SEARCH_ENGINE=native` +/// (PORT-DESIGN §6 wiring). Selecting the engine is a config change, +/// so flipping back never needs a rebuild or a redeploy. +pub async fn search(q: &str, type_: &str, limit: u64) -> Result<Vec<SearchHit>, SearchError> { + match crate::config::Config::load().search_engine.as_str() { + "native" => native::Native.search(q, type_, limit).await, + _ => gn3::Gn3.search(q, type_, limit).await, + } +} /// One search hit, in gn3's shape. /// diff --git a/src/search/native.rs b/src/search/native.rs new file mode 100644 index 0000000..c87f711 --- /dev/null +++ b/src/search/native.rs @@ -0,0 +1,250 @@ +//! The native [`Engine`]: a Rust port of gn3's xapian search path +//! (`gn3/gn3/api/search.py`), querying the same python-built xapian +//! index directly instead of proxying gn3. +//! +//! # Query grammar +//! +//! The [`QueryParser`] is built once per process with the gn3 parser +//! configuration (see the seam's grammar doc and +//! `~/spikes/port-design/PORT-DESIGN.md` §2). Custom registration is +//! `Box::leak`-based inside the binding, so building a parser per +//! request would leak per call; the parser is therefore built at +//! startup and cached behind a mutex (binding-report §5 item 3, the +//! round-3 R2 skeleton pattern). +//! +//! # Per-page invariant +//! +//! The engine retrieves the FULL match set and ignores the forwarded +//! `limit`, exactly as the parity recording did (`per_page=50000`; +//! PORT-DESIGN §5). The route keeps its validation/limit logic +//! unchanged; `search_native.rs` pins the invariant with a test that +//! forwards `limit=10` and asserts the full set still comes back. +//! +//! # Error contract +//! +//! gn3's order is preserved: the query is parsed first, the index +//! opened second (search.py parses at :291 before opening at :296), so +//! a malformed query reports as [`SearchError::InvalidQuery`] (422 +//! `invalid_query`, carrying the parser's diagnostic) even when the +//! index is misconfigured. Other mappings per the seam's vocabulary +//! (DESIGN-NOTES S3): an unset index path is a local misconfiguration +//! ([`SearchError::Internal`], mirroring the bad-GN3_URL rule); an +//! index that cannot be opened is [`SearchError::Unreachable`]; a +//! document whose stored data is not a search hit is +//! [`SearchError::Misbehaving`] (the doc-data analogue of a response +//! body that fails to decode). +//! +//! The binding's `Database::open` aborts the process on failure (the +//! C++ exception is not translated across the FFI boundary), so +//! [`open_database`] pre-validates the path and maps failures onto the +//! seam vocabulary before any xapian call — the same app-level-guard +//! pattern PORT-DESIGN C3.6 pins for the mset-time error. A corrupt +//! index that passes the file checks would still abort; gn3 crashes on +//! such an index too, and no gate exercises it. + +use super::{Engine, SearchError, SearchHit}; +use std::sync::{Mutex, OnceLock}; +use xapian_rs::{Database, Enquire, Operator, Query, QueryParser, Stem, StemStrategy}; + +/// MSet bound for full-match-set retrieval: deliberately large so the +/// whole match set comes back regardless of the forwarded `limit` +/// (PORT-DESIGN §5; the recorded Q02 set alone is 57 632 hits, above +/// any limit the gn4 surface can express). Xapian stops at the actual +/// match count. +const FULL_MATCH_SET_MAX_ITEMS: u32 = u32::MAX; + +/// A sendable wrapper around the process-cached parser. +struct Parser(QueryParser); + +// SAFETY: `QueryParser` holds a raw C++ handle that the binding does +// not mark `Send`, and the handle itself is not safe to use from +// several threads concurrently — but every access goes through the +// enclosing `Mutex`, so the handle is only ever touched by one thread +// at a time. Xapian objects may be used from multiple threads when +// access is serialized; that is exactly what the mutex guarantees. +unsafe impl Send for Parser {} + +/// The process-cached parser, built once with the gn3 parser +/// configuration (see [`build_query_parser`]). +fn parser() -> &'static Mutex<Parser> { + static PARSER: OnceLock<Mutex<Parser>> = OnceLock::new(); + PARSER.get_or_init(|| Mutex::new(Parser(build_query_parser()))) +} + +/// Build the gn3-equivalent QueryParser configuration +/// (search.py:213-273). The parser is query- and index-independent, so +/// one per process serves every request. +fn build_query_parser() -> QueryParser { + let mut qp = QueryParser::default(); + // search.py:216-217: English stemmer, STEM_ALL_Z (free-text terms + // are stemmed and matched case-insensitively). + qp.set_stemmer(Stem::for_language("en")); + qp.set_stemming_strategy(StemStrategy::AllZ); + qp +} + +/// The native xapian search engine. Configure the index directory with +/// `XAPIAN_DB_PATH` and select it with `SEARCH_ENGINE=native`; the +/// default remains the gn3 forwarder (see [`crate::search::search`]). +pub struct Native; + +impl Engine for Native { + async fn search( + &self, + q: &str, + type_: &str, + limit: u64, + ) -> Result<Vec<SearchHit>, SearchError> { + // xapian calls are synchronous C++ work: keep them off the + // async runtime's workers (the house pattern for blocking + // calls, cf. every spawn_blocking handler in src/routes). + let query = q.to_string(); + let search_type = type_.to_string(); + tokio::task::spawn_blocking(move || search_blocking(&query, &search_type, limit)) + .await + .map_err(|e| { + SearchError::Internal(format!("native search task failed to join: {e}")) + })? + } +} + +/// The synchronous search path: parse, open, filter, retrieve, map. +fn search_blocking(q: &str, type_: &str, limit: u64) -> Result<Vec<SearchHit>, SearchError> { + // 1. Parse first (search.py:291 parses before :296 opens the + // index), through the checked-parse API: a malformed query is a + // typed error, not a panic (the fork's parse_query_checked). + let parsed = { + let mut parser = parser().lock().expect("native parser mutex poisoned"); + parser + .0 + .parse_query_checked::<&str>(q, None, None) + .map_err(|e| SearchError::InvalidQuery(e.message))? + }; + + // 2. Open the index. + let db_path = crate::config::Config::load().xapian_db_path; + let db = open_database(&db_path)?; + + // 3. Filter to the document type: OP_FILTER(parsed, "XT" + type) + // (search.py:298-300; XT terms are written at index time by the + // production indexer). + let filtered = Query::combine( + Operator::Filter, + parsed, + Query::term(format!("XT{type_}"), None, None), + ); + + // 4. Retrieve the full match set — the forwarded `limit` is + // deliberately not applied here (per-page invariant, §5). + let _ = limit; + let mut enquire = Enquire::new(&db); + enquire.set_query(&filtered, None); + let mset = enquire.mset(0, FULL_MATCH_SET_MAX_ITEMS, None, None); + + // 5. Map each hit's stored doc data (the indexer's JSON, the + // schema gn3 serves verbatim: json.loads(document.get_data()), + // search.py:302) onto SearchHit; unknown fields flow through + // `extra`. + let mut hits = Vec::with_capacity(mset.size() as usize); + for mset_item in mset.matches() { + let data = mset_item.document().data(); + hits.push(hit_from_doc_data(&data)?); + } + Ok(hits) +} + +/// Open the xapian index at `path`, with the pre-validation the binding +/// needs (its `Database::open` aborts on failure instead of returning +/// an error). The file checks cover the misconfiguration classes an +/// operator can actually hit; see the module doc for the corrupt-index +/// residual. +fn open_database(path: &str) -> Result<Database, SearchError> { + if path.is_empty() { + return Err(SearchError::Internal( + "XAPIAN_DB_PATH is not configured; set it to the xapian index directory to \ + use the native search engine (SEARCH_ENGINE=native)" + .to_string(), + )); + } + let metadata = std::fs::metadata(path).map_err(|e| { + SearchError::Unreachable(format!("xapian index at {path} could not be opened: {e}")) + })?; + if !metadata.is_dir() { + return Err(SearchError::Unreachable(format!( + "xapian index at {path} is not a directory" + ))); + } + // The glass backend writes this marker into every index directory; + // it distinguishes a real index from an arbitrary directory. + if !std::path::Path::new(path).join("iamglass").is_file() { + return Err(SearchError::Unreachable(format!( + "xapian index at {path} does not look like a glass index (no iamglass marker)" + ))); + } + Ok(Database::open(path, None)) +} + +/// Map one hit's stored document data onto [`SearchHit`]. The data is +/// the indexer's own JSON (`doc.set_data(json.dumps(trait.data))`, +/// index-genenetwork:361/408), so decode failures mean the index does +/// not hold what gn3 would serve — `Misbehaving`, the doc-data analogue +/// of an undecodable backend response body. +fn hit_from_doc_data(data: &[u8]) -> Result<SearchHit, SearchError> { + serde_json::from_slice::<SearchHit>(data).map_err(|e| { + SearchError::Misbehaving(format!("xapian doc data is not a search hit: {e}")) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Hermetic parser tests: no index, no environment, just the + /// grammar configuration and the checked-parse contract. + #[test] + fn checked_parse_rejects_and_accepts_like_gn3() { + let mut qp = build_query_parser(); + + // search.py:291: a truncated AND expression is a + // QueryParserError; the checked API surfaces it typed (the + // fork's round-3 R1 gate) instead of panicking. + let err = qp + .parse_query_checked::<&str>("alcohol AND", None, None) + .expect_err("alcohol AND must be rejected"); + assert_eq!(err.error_type, "QueryParserError"); + assert_eq!(err.message, "Syntax: <expression> AND <expression>"); + + // The && shapes: production (and libxapian 1.4.29) serve these + // as 200s (Q47/Q54, harness-report §4 F2/F4), so they must + // parse Ok here too. + qp.parse_query_checked::<&str>("a&&b", None, None) + .expect("a&&b parses (Q47)"); + qp.parse_query_checked::<&str>("alcohol &&", None, None) + .expect("alcohol && parses (Q54)"); + + // An unregistered prefix is plain text (Q50, foo:bar). + qp.parse_query_checked::<&str>("foo:bar", None, None) + .expect("unregistered prefixes parse as text (Q50)"); + } + + /// The index-open guards, without touching a real index. + #[test] + fn open_database_maps_failures_onto_the_seam_vocabulary() { + assert!(matches!( + open_database(""), + Err(SearchError::Internal(_)) + )); + assert!(matches!( + open_database("/nonexistent/gn4-native-test-index"), + Err(SearchError::Unreachable(_)) + )); + // A real directory that is not an index. + let not_an_index = std::env::temp_dir().join(format!("gn4-not-an-index-{}", std::process::id())); + std::fs::create_dir_all(¬_an_index).unwrap(); + assert!(matches!( + open_database(not_an_index.to_str().unwrap()), + Err(SearchError::Unreachable(_)) + )); + std::fs::remove_dir_all(¬_an_index).unwrap(); + } +} |
