about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorAlexander_Kabui <alexanderkabua@gmail.com>2026-10-07 06:34:51 +0300
committerAlexander_Kabui <alexanderkabua@gmail.com>2026-10-07 06:34:51 +0300
commita5c40fc78f6b3cb32fba3daeb9f2a8e95558ba09 (patch)
tree51dba2ecba0e7371e220b6c6d7eaa99a13b0adf5 /src
parentf7423aaaf71194a169ded303baa3cb7bda8f0b53 (diff)
downloadgenenetwork4-a5c40fc78f6b3cb32fba3daeb9f2a8e95558ba09.tar.gz
src/search/native.rs (native engine): port the xapian query path behind the Engine seam (C3.1).
New native engine, a Rust port of gn3 search path (gn3/api/search.py),
opt-in via SEARCH_ENGINE=native (the gn3 forwarder stays the default;
src/search/mod.rs now dispatches on config, routes/search.rs delegates):

- startup-cached QueryParser behind a mutex: custom-prefix registration
  is Box::leak-based inside the binding, so a per-request parser would
  leak; built once per process (binding-report §5 item 3, R2 pattern).
  Minimal configuration this commit per PORT-DESIGN C3.1: Stem("en") +
  STEM_ALL_Z only.
- checked parse via the fork's parse_query_checked: malformed queries
  are typed errors, not panics; Err maps to SearchError::InvalidQuery
  (422 invalid_query, the owner-blessed contract), carrying the
  parser diagnostic. Parse happens before the index is opened, as in
  gn3 (search.py:291 vs :296), so a bad query reports InvalidQuery
  even when the index is misconfigured.
- XT{type} document filter: OP_FILTER(parsed, "XT"+type)
  (search.py:298-300).
- FULL-match-set retrieval ignoring the forwarded limit (per-page
  invariant, PORT-DESIGN §5): mset bound is a deliberately huge fixed
  constant; the route keeps its validation and limit logic unchanged.
- per-hit mapping: doc data JSON -> SearchHit via serde
  (search.py:302 equivalent); unknown fields flow through extra.
  Decode failure maps to SearchError::Misbehaving.
- index open behind an app-level guard (the binding aborts on open
  failure; see the module doc): unset XAPIAN_DB_PATH -> Internal,
  unopenable path -> Unreachable.

tests/search_native.rs: error-contract scenarios (hermetic) + the
per-page invariant + dispatch pinning, one sequential test fn per the
process-global env rule; the dev-index part skips loudly when
XAPIAN_DB_PATH is unset. tests/search_native_shapes.rs: the shape-CLASS
gate (this lane s AMENDMENT-B translation of the C3.1 stage gate) --
16 shapes (Q01,Q03-Q13,Q47,Q50,Q51,Q54), each expected class cited to
its tests/search_parity.rs shape! line; dev-index counts are printed
as reference data only, never compared to production.

Evidence: logs/c3-1.log (build EXIT=0, native tests green, lib 14/14),
logs/c3-1-full.log (FULL suite EXIT=0, parity floor green),
logs/shape-class-c3-1.log (SHAPE-CLASS GATE 16/16 PASS vs the dev
index; Q51 byte-identical diagnostic; per-page invariant: limit=10
forwarded, 2330-hit full set returned).
Diffstat (limited to 'src')
-rw-r--r--src/routes/search.rs15
-rw-r--r--src/search/mod.rs21
-rw-r--r--src/search/native.rs250
3 files changed, 275 insertions, 11 deletions
diff --git a/src/routes/search.rs b/src/routes/search.rs
index 8d4841b..a10f431 100644
--- a/src/routes/search.rs
+++ b/src/routes/search.rs
@@ -7,8 +7,8 @@
 //!
 //! The backend itself lives behind the search engine seam (see
 //! `crate::search`); this handler validates parameters, delegates to
-//! the [`Engine`], and maps the engine's [`SearchError`] onto the
-//! existing [`ApiError`] shapes.
+//! the configured [`Engine`], and maps the engine's [`SearchError`]
+//! onto the existing [`ApiError`] shapes.
 
 use axum::Json;
 use axum::extract::{Extension, Query};
@@ -20,7 +20,7 @@ use crate::auth::AuthToken;
 use crate::error::{ApiError, Unauthorized};
 use crate::request_id_string;
 use crate::response::Envelope;
-use crate::search::{Engine, SearchError, SearchHit, gn3::Gn3};
+use crate::search::{self, SearchError, SearchHit};
 
 /// Query parameters for `GET /search`.
 ///
@@ -79,10 +79,12 @@ fn engine_error(error: SearchError, rid: Option<String>) -> ApiError {
 
 #[utoipa::path(get, path = "/search",
     summary = "Search traits across GN datasets",
-    description = "Proxies GeneNetwork3's xapian search (phase A; see \
+    description = "Search traits across GN datasets. Runs through the \
+configured engine: GeneNetwork3's xapian search by default, or the \
+native xapian engine when SEARCH_ENGINE=native (phase A; see \
 topics/gn4/gn4-api-foundation.gmi). Hits are gn3's trait objects passed \
 through unchanged, wrapped in the descriptive envelope. Malformed \
-queries forward gn3's parser diagnostic as a 422.",
+queries carry the backend's parser diagnostic as a 422.",
     params(SearchParams),
     responses(
         (status = 200, description = "Matching traits, gn3 shape", body = Envelope<Vec<SearchHit>>),
@@ -120,8 +122,7 @@ pub async fn search(
     }
     let limit = params.limit.unwrap_or(20).min(100);
 
-    let hits = Gn3
-        .search(q, search_type, limit)
+    let hits = search::search(q, search_type, limit)
         .await
         .map_err(|e| engine_error(e, rid))?;
 
diff --git a/src/search/mod.rs b/src/search/mod.rs
index e4897ac..936c977 100644
--- a/src/search/mod.rs
+++ b/src/search/mod.rs
@@ -78,17 +78,30 @@
 //! enrichment is its own private design, noted here only so the effort
 //! estimate prices the enrichment in.
 //!
-//! # Trait now, one impl
+//! # Trait now, two impls
 //!
 //! The phase-2 plan originally said "no trait yet" (a trait with one
 //! impl is speculation). The overnight schedule overrides that: the
-//! [`Engine`] trait is defined now so the native engine can land behind
-//! it, with the gn3-forwarding implementation as its first (and, until
-//! then, only) implementation.
+//! [`Engine`] trait is defined so the native engine can land behind
+//! it. The gn3-forwarding implementation ([`gn3::Gn3`]) remains the
+//! default backend; [`native::Native`] is opt-in via `SEARCH_ENGINE`
+//! (see [`search`]).
 
 use std::future::Future;
 
 pub mod gn3;
+pub mod native;
+
+/// Run a search through the configured engine: the gn3 forwarder by
+/// default, the native xapian engine when `SEARCH_ENGINE=native`
+/// (PORT-DESIGN §6 wiring). Selecting the engine is a config change,
+/// so flipping back never needs a rebuild or a redeploy.
+pub async fn search(q: &str, type_: &str, limit: u64) -> Result<Vec<SearchHit>, SearchError> {
+    match crate::config::Config::load().search_engine.as_str() {
+        "native" => native::Native.search(q, type_, limit).await,
+        _ => gn3::Gn3.search(q, type_, limit).await,
+    }
+}
 
 /// One search hit, in gn3's shape.
 ///
diff --git a/src/search/native.rs b/src/search/native.rs
new file mode 100644
index 0000000..c87f711
--- /dev/null
+++ b/src/search/native.rs
@@ -0,0 +1,250 @@
+//! The native [`Engine`]: a Rust port of gn3's xapian search path
+//! (`gn3/gn3/api/search.py`), querying the same python-built xapian
+//! index directly instead of proxying gn3.
+//!
+//! # Query grammar
+//!
+//! The [`QueryParser`] is built once per process with the gn3 parser
+//! configuration (see the seam's grammar doc and
+//! `~/spikes/port-design/PORT-DESIGN.md` §2). Custom registration is
+//! `Box::leak`-based inside the binding, so building a parser per
+//! request would leak per call; the parser is therefore built at
+//! startup and cached behind a mutex (binding-report §5 item 3, the
+//! round-3 R2 skeleton pattern).
+//!
+//! # Per-page invariant
+//!
+//! The engine retrieves the FULL match set and ignores the forwarded
+//! `limit`, exactly as the parity recording did (`per_page=50000`;
+//! PORT-DESIGN §5). The route keeps its validation/limit logic
+//! unchanged; `search_native.rs` pins the invariant with a test that
+//! forwards `limit=10` and asserts the full set still comes back.
+//!
+//! # Error contract
+//!
+//! gn3's order is preserved: the query is parsed first, the index
+//! opened second (search.py parses at :291 before opening at :296), so
+//! a malformed query reports as [`SearchError::InvalidQuery`] (422
+//! `invalid_query`, carrying the parser's diagnostic) even when the
+//! index is misconfigured. Other mappings per the seam's vocabulary
+//! (DESIGN-NOTES S3): an unset index path is a local misconfiguration
+//! ([`SearchError::Internal`], mirroring the bad-GN3_URL rule); an
+//! index that cannot be opened is [`SearchError::Unreachable`]; a
+//! document whose stored data is not a search hit is
+//! [`SearchError::Misbehaving`] (the doc-data analogue of a response
+//! body that fails to decode).
+//!
+//! The binding's `Database::open` aborts the process on failure (the
+//! C++ exception is not translated across the FFI boundary), so
+//! [`open_database`] pre-validates the path and maps failures onto the
+//! seam vocabulary before any xapian call — the same app-level-guard
+//! pattern PORT-DESIGN C3.6 pins for the mset-time error. A corrupt
+//! index that passes the file checks would still abort; gn3 crashes on
+//! such an index too, and no gate exercises it.
+
+use super::{Engine, SearchError, SearchHit};
+use std::sync::{Mutex, OnceLock};
+use xapian_rs::{Database, Enquire, Operator, Query, QueryParser, Stem, StemStrategy};
+
+/// MSet bound for full-match-set retrieval: deliberately large so the
+/// whole match set comes back regardless of the forwarded `limit`
+/// (PORT-DESIGN §5; the recorded Q02 set alone is 57 632 hits, above
+/// any limit the gn4 surface can express). Xapian stops at the actual
+/// match count.
+const FULL_MATCH_SET_MAX_ITEMS: u32 = u32::MAX;
+
+/// A sendable wrapper around the process-cached parser.
+struct Parser(QueryParser);
+
+// SAFETY: `QueryParser` holds a raw C++ handle that the binding does
+// not mark `Send`, and the handle itself is not safe to use from
+// several threads concurrently — but every access goes through the
+// enclosing `Mutex`, so the handle is only ever touched by one thread
+// at a time. Xapian objects may be used from multiple threads when
+// access is serialized; that is exactly what the mutex guarantees.
+unsafe impl Send for Parser {}
+
+/// The process-cached parser, built once with the gn3 parser
+/// configuration (see [`build_query_parser`]).
+fn parser() -> &'static Mutex<Parser> {
+    static PARSER: OnceLock<Mutex<Parser>> = OnceLock::new();
+    PARSER.get_or_init(|| Mutex::new(Parser(build_query_parser())))
+}
+
+/// Build the gn3-equivalent QueryParser configuration
+/// (search.py:213-273). The parser is query- and index-independent, so
+/// one per process serves every request.
+fn build_query_parser() -> QueryParser {
+    let mut qp = QueryParser::default();
+    // search.py:216-217: English stemmer, STEM_ALL_Z (free-text terms
+    // are stemmed and matched case-insensitively).
+    qp.set_stemmer(Stem::for_language("en"));
+    qp.set_stemming_strategy(StemStrategy::AllZ);
+    qp
+}
+
+/// The native xapian search engine. Configure the index directory with
+/// `XAPIAN_DB_PATH` and select it with `SEARCH_ENGINE=native`; the
+/// default remains the gn3 forwarder (see [`crate::search::search`]).
+pub struct Native;
+
+impl Engine for Native {
+    async fn search(
+        &self,
+        q: &str,
+        type_: &str,
+        limit: u64,
+    ) -> Result<Vec<SearchHit>, SearchError> {
+        // xapian calls are synchronous C++ work: keep them off the
+        // async runtime's workers (the house pattern for blocking
+        // calls, cf. every spawn_blocking handler in src/routes).
+        let query = q.to_string();
+        let search_type = type_.to_string();
+        tokio::task::spawn_blocking(move || search_blocking(&query, &search_type, limit))
+            .await
+            .map_err(|e| {
+                SearchError::Internal(format!("native search task failed to join: {e}"))
+            })?
+    }
+}
+
+/// The synchronous search path: parse, open, filter, retrieve, map.
+fn search_blocking(q: &str, type_: &str, limit: u64) -> Result<Vec<SearchHit>, SearchError> {
+    // 1. Parse first (search.py:291 parses before :296 opens the
+    //    index), through the checked-parse API: a malformed query is a
+    //    typed error, not a panic (the fork's parse_query_checked).
+    let parsed = {
+        let mut parser = parser().lock().expect("native parser mutex poisoned");
+        parser
+            .0
+            .parse_query_checked::<&str>(q, None, None)
+            .map_err(|e| SearchError::InvalidQuery(e.message))?
+    };
+
+    // 2. Open the index.
+    let db_path = crate::config::Config::load().xapian_db_path;
+    let db = open_database(&db_path)?;
+
+    // 3. Filter to the document type: OP_FILTER(parsed, "XT" + type)
+    //    (search.py:298-300; XT terms are written at index time by the
+    //    production indexer).
+    let filtered = Query::combine(
+        Operator::Filter,
+        parsed,
+        Query::term(format!("XT{type_}"), None, None),
+    );
+
+    // 4. Retrieve the full match set — the forwarded `limit` is
+    //    deliberately not applied here (per-page invariant, §5).
+    let _ = limit;
+    let mut enquire = Enquire::new(&db);
+    enquire.set_query(&filtered, None);
+    let mset = enquire.mset(0, FULL_MATCH_SET_MAX_ITEMS, None, None);
+
+    // 5. Map each hit's stored doc data (the indexer's JSON, the
+    //    schema gn3 serves verbatim: json.loads(document.get_data()),
+    //    search.py:302) onto SearchHit; unknown fields flow through
+    //    `extra`.
+    let mut hits = Vec::with_capacity(mset.size() as usize);
+    for mset_item in mset.matches() {
+        let data = mset_item.document().data();
+        hits.push(hit_from_doc_data(&data)?);
+    }
+    Ok(hits)
+}
+
+/// Open the xapian index at `path`, with the pre-validation the binding
+/// needs (its `Database::open` aborts on failure instead of returning
+/// an error). The file checks cover the misconfiguration classes an
+/// operator can actually hit; see the module doc for the corrupt-index
+/// residual.
+fn open_database(path: &str) -> Result<Database, SearchError> {
+    if path.is_empty() {
+        return Err(SearchError::Internal(
+            "XAPIAN_DB_PATH is not configured; set it to the xapian index directory to \
+             use the native search engine (SEARCH_ENGINE=native)"
+                .to_string(),
+        ));
+    }
+    let metadata = std::fs::metadata(path).map_err(|e| {
+        SearchError::Unreachable(format!("xapian index at {path} could not be opened: {e}"))
+    })?;
+    if !metadata.is_dir() {
+        return Err(SearchError::Unreachable(format!(
+            "xapian index at {path} is not a directory"
+        )));
+    }
+    // The glass backend writes this marker into every index directory;
+    // it distinguishes a real index from an arbitrary directory.
+    if !std::path::Path::new(path).join("iamglass").is_file() {
+        return Err(SearchError::Unreachable(format!(
+            "xapian index at {path} does not look like a glass index (no iamglass marker)"
+        )));
+    }
+    Ok(Database::open(path, None))
+}
+
+/// Map one hit's stored document data onto [`SearchHit`]. The data is
+/// the indexer's own JSON (`doc.set_data(json.dumps(trait.data))`,
+/// index-genenetwork:361/408), so decode failures mean the index does
+/// not hold what gn3 would serve — `Misbehaving`, the doc-data analogue
+/// of an undecodable backend response body.
+fn hit_from_doc_data(data: &[u8]) -> Result<SearchHit, SearchError> {
+    serde_json::from_slice::<SearchHit>(data).map_err(|e| {
+        SearchError::Misbehaving(format!("xapian doc data is not a search hit: {e}"))
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Hermetic parser tests: no index, no environment, just the
+    /// grammar configuration and the checked-parse contract.
+    #[test]
+    fn checked_parse_rejects_and_accepts_like_gn3() {
+        let mut qp = build_query_parser();
+
+        // search.py:291: a truncated AND expression is a
+        // QueryParserError; the checked API surfaces it typed (the
+        // fork's round-3 R1 gate) instead of panicking.
+        let err = qp
+            .parse_query_checked::<&str>("alcohol AND", None, None)
+            .expect_err("alcohol AND must be rejected");
+        assert_eq!(err.error_type, "QueryParserError");
+        assert_eq!(err.message, "Syntax: <expression> AND <expression>");
+
+        // The && shapes: production (and libxapian 1.4.29) serve these
+        // as 200s (Q47/Q54, harness-report §4 F2/F4), so they must
+        // parse Ok here too.
+        qp.parse_query_checked::<&str>("a&&b", None, None)
+            .expect("a&&b parses (Q47)");
+        qp.parse_query_checked::<&str>("alcohol &&", None, None)
+            .expect("alcohol && parses (Q54)");
+
+        // An unregistered prefix is plain text (Q50, foo:bar).
+        qp.parse_query_checked::<&str>("foo:bar", None, None)
+            .expect("unregistered prefixes parse as text (Q50)");
+    }
+
+    /// The index-open guards, without touching a real index.
+    #[test]
+    fn open_database_maps_failures_onto_the_seam_vocabulary() {
+        assert!(matches!(
+            open_database(""),
+            Err(SearchError::Internal(_))
+        ));
+        assert!(matches!(
+            open_database("/nonexistent/gn4-native-test-index"),
+            Err(SearchError::Unreachable(_))
+        ));
+        // A real directory that is not an index.
+        let not_an_index = std::env::temp_dir().join(format!("gn4-not-an-index-{}", std::process::id()));
+        std::fs::create_dir_all(&not_an_index).unwrap();
+        assert!(matches!(
+            open_database(not_an_index.to_str().unwrap()),
+            Err(SearchError::Unreachable(_))
+        ));
+        std::fs::remove_dir_all(&not_an_index).unwrap();
+    }
+}