;;; index.scm --- a topic index over the ai-memory store, in Xapian ;;; Copyright © 2026 Pjotr Prins ;;; ;;; ai-memory's SQLite store is the truth; this is a derived, disposable ;;; Xapian index over it, in the notmuch style: one document per unit, keyed ;;; by a unique term, fielded terms for routing, free text stemmed. Drop the ;;; directory and `update --rebuild' to regenerate it. ;;; ;;; Why a separate program rather than FFI inside ai-memory: libxapian is ;;; C++-only, so a foreign call means a hand-written C shim. guile-xapian ;;; already has one -- tested, maintained -- and this is the API it exposes. ;;; ai-memory shells out to this script; the boundary is a store path in and ;;; JSON out, and SQLite stays the source of truth. ;;; ;;; Usage ;;; index.scm update --store STORE.sqlite --index DIR [--rebuild] [--quiet] ;;; index.scm search --index DIR [--limit N] [--or] QUERY... ;;; index.scm count --index DIR ;;; ;;; Documents ;;; reasoning: one per reasoning entry (session, kind, commit) ;;; commit: one per commit (subject + body, author date) ;;; ;;; Terms (boolean, case-folded on both sides, so prefix values are compared ;;; exactly rather than stemmed) ;;; K identity: the replace key, never queried by a user ;;; T reasoning | commit ;;; R repository name ;;; S session id ;;; O one per word of the session id, so `topic:pve' is an entry ;;; point into the pve story ;;; C the commit an entry is attached to (7 characters) ;;; D author date ;;; ;;;; Value slots ;;; 0 date (yyyymmdd; lexicographic order is chronological) ;;; 1 digest of the indexed text, for the incremental compare ;;; 2 the document's JSON record -- so a hit is one value read and needs ;;; no second query ;;; ;;; `update' is incremental: each row's text is digested and compared with ;;; slot 1 of the document already in the index; only new or changed rows are ;;; replaced. Deletions are not propagated -- the store is append-only, so a ;;; row that disappears is a `--rebuild'. (use-modules (ice-9 format) (ice-9 match) (ice-9 getopt-long) (rnrs arithmetic bitwise) (srfi srfi-1) (srfi srfi-13) (sqlite3) (xapian wrap)) ;;; --------------------------------------------------------------------- ;;; Schema. One place, so the indexer and the query parser cannot drift. (define SCHEMA-VERSION 1) (define SLOT-DATE 0) (define SLOT-DIGEST 1) (define SLOT-DOCDATA 2) ;; User-facing prefix -> boolean term prefix. (define PREFIXES '(("type:" . "T") ("repo:" . "R") ("session:" . "S") ("topic:" . "O") ("commit:" . "C") ("date:" . "D"))) (define (term prefix value) (string-append prefix (string-downcase value))) ;;; --------------------------------------------------------------------- ;;; Helpers (define CURRENT-OPTIONS (make-parameter (list))) (define (quiet?) (option-ref (CURRENT-OPTIONS) 'quiet #f)) (define (die fmt . args) (format (current-error-port) "index.scm: ~a\n" (apply format #f fmt args)) (exit 1)) (define (note fmt . args) (unless (quiet?) (apply format (current-error-port) fmt args) (newline (current-error-port)))) ;; FNV-1a over the text: the only property needed is "changes when the text ;; changes", and this avoids a crypto dependency for it. (define (digest str) (let loop ((i 0) (h 2166136261)) (if (>= i (string-length str)) (number->string h) (loop (+ i 1) (modulo (* (bitwise-xor h (char->integer (string-ref str i))) 16777619) 4294967296))))) (define (yyyymmdd epoch) (let* ((t (gmtime epoch)) (y (+ 1900 (tm:year t))) (m (+ 1 (tm:mon t))) (d (tm:mday t))) (+ (* y 10000) (* m 100) d))) ;; The store knows its repository only as a directory; the basename is the ;; name a reader uses ("genenetwork4"), so that is the R term. (define (repo-name store) (basename (dirname (canonicalize-path store)))) (define (alist-set table key value) (cons (cons key value) table)) (define (non-empty-string? value) (and (string? value) (not (string-null? value)))) ;;; --------------------------------------------------------------------- ;;; Database handles (define (open-writable index rebuild) ;; Xapian creates its files inside DIR, but not DIR -- and only the plain ;; `update' path used to create it, so `update --rebuild' on a store whose ;; index directory was deleted (the documented way to recover, and the state ;; of every store that has never been indexed) died in new-WritableDatabase ;; with "mkdir failed". `mkdir -p' is a no-op when the directory is there. (unless (zero? (system* "mkdir" "-p" index)) (die "could not create ~a" index)) (new-WritableDatabase index (if rebuild (DB-CREATE-OR-OVERWRITE) (DB-CREATE-OR-OPEN)))) (define (open-readable index) (unless (file-exists? index) (die "no index at ~a: run `update' first" index)) (new-Database index)) ;;; --------------------------------------------------------------------- ;;; JSON, emitted rather than parsed ;;; ;;; The index stores a JSON record per document (slot 2) and hands it back ;;; verbatim, so nothing here ever parses JSON -- which is why this is a ;;; writer of thirty lines instead of a library dependency. Objects are ;;; alists of dotted pairs; values are strings, numbers, or lists of ;;; strings; nothing else is stored. (define (json-escape text) (let ((out (open-output-string))) (string-for-each (lambda (ch) (case ch ((#\") (display "\\\"" out)) ((#\\) (display "\\\\" out)) ((#\newline) (display "\\n" out)) ((#\tab) (display "\\t" out)) ((#\return) (display "\\r" out)) (else (if (< (char->integer ch) 32) (format out "\\u~4,'0x" (char->integer ch)) (write-char ch out))))) text) (get-output-string out))) (define (json-value value) (cond ((string? value) (string-append "\"" (json-escape value) "\"")) ((number? value) (number->string value)) ((boolean? value) (if value "true" "false")) ((list? value) (string-append "[" (string-join (map json-value value) ",") "]")) ((null? value) "null") (else (json-value (format #f "~a" value))))) (define (json-object record) (string-append "{" (string-join (map (lambda (pair) (string-append (json-value (symbol->string (car pair))) ":" (json-value (second pair)))) record) ",") "}")) ;;; --------------------------------------------------------------------- ;;; Records -> documents ;;; ;;; A record is an alist with symbol keys, so it is both what we index and ;;; what slot 2 carries: the caller renders exactly what the indexer stored. ;;; guile-json wants object entries as two-element lists (key value). (define (field key value) (list key value)) ;; An entry is (key value), so the cdr is a one-element list -- assq-ref would ;; hand back the wrapping list rather than the value. (define (lookup record key) (let ((entry (assq key record))) (and entry (second entry)))) ;;; The title is the entry's own gist -- the assessed one-line claim, which is ;;; what `list' and `why' show. It is deliberately NOT the subject of the ;;; commit the entry is attached to: that subject is already the title of the ;;; commit document, and an entry recorded with `--commit none' has no commit ;;; to borrow from, which left it indexed with an empty title and its gist ;;; unsearchable. (define (reasoning-record repo row) (match row ((id session step kind body commit_hash created_at gist) (let* ((key (format #f "reasoning:~a" id)) (date (yyyymmdd created_at)) (title (if (string? gist) (string-trim-both gist) "")) (topics (if (non-empty-string? session) (map (lambda (word) (term "O" word)) (string-split session #\-)) (list))) (terms (append (list (term "T" "reasoning") (term "R" repo) (term "D" (number->string date))) (if (non-empty-string? session) (cons (term "S" session) topics) (list)) (if (non-empty-string? commit_hash) (list (term "C" (substring commit_hash 0 7))) (list))))) (list key (list (field 'type "reasoning") (field 'key key) (field 'repo repo) (field 'session (or session "")) (field 'kind (or kind "")) (field 'step (or step 0)) (field 'date date) (field 'commit (or commit_hash "")) (field 'title title) (field 'terms terms) (field 'body (or body "")) (field 'text (string-append title " " (or body ""))))))) (else (die "unexpected reasoning row: ~s" row)))) (define (commit-record repo row sessions) (match row ((hash subject body author_ts) (let* ((key (format #f "commit:~a" hash)) (date (yyyymmdd author_ts)) (subject (or subject "")) ;; A commit inherits the topics of the reasoning attached to it: ;; otherwise the pve story answers with its reasoning and not with ;; the commits that carry it, which is exactly backwards for a ;; reader following it. (topics (append-map (lambda (session) (map (lambda (word) (term "O" word)) (string-split session #\-))) sessions))) (list key (list (field 'type "commit") (field 'key key) (field 'repo repo) (field 'date date) (field 'commit hash) (field 'title subject) (field 'session (if (null? sessions) "" (car sessions))) (field 'terms (append (list (term "T" "commit") (term "R" repo) (term "D" (number->string date)) (term "C" (substring hash 0 7))) (map (lambda (session) (term "S" session)) sessions) topics)) (field 'body (or body "")) (field 'text (string-append subject " " (or body ""))))))) (else (die "unexpected commit row: ~s" row)))) (define (sessions-by-commit attached) (fold (lambda (row table) (match row ((commit_hash sessions) (alist-set table commit_hash (if sessions (string-split sessions #\,) (list)))) (else table))) '() attached)) (define (records repo reasoning commits attached) (let ((sessions (sessions-by-commit attached))) (append (map (lambda (row) (reasoning-record repo row)) reasoning) (map (lambda (row) (commit-record repo row (or (assoc-ref sessions (first row)) (list)))) commits)))) ;;; --------------------------------------------------------------------- ;;; Writing (define (build-document stemmer record) (let ((doc (new-Document)) (text (lookup record 'text)) (date (lookup record 'date))) (Document-set-data doc (json-object record)) ;; The identity term first: it is what the next run deletes by, so a ;; document without it can never be replaced -- only duplicated. (Document-add-boolean-term doc (string-append "K" (lookup record 'key))) (for-each (lambda (t) (Document-add-boolean-term doc t)) (lookup record 'terms)) (Document-add-value doc SLOT-DATE (number->string date)) (Document-add-value doc SLOT-DIGEST (digest text)) (Document-add-value doc SLOT-DOCDATA (json-object record)) (let ((tg (new-TermGenerator))) (TermGenerator-set-stemmer tg stemmer) (TermGenerator-set-document tg doc) (TermGenerator-index-text tg text)) doc)) ;; Every document is re-created: deleted by its unique term, then added. ;; Xapian applies that as a delete plus an insert of the changed postings, so ;; the on-disk work is incremental; this program merely re-tokenises every ;; body (~1 s at a few thousand documents), which buys not having to remember ;; what was indexed last time -- state that can go stale. (define (index-record! db stemmer key record) ;; delete-then-add rather than replace-document: `delete_document' by ;; unique term works through this binding, while `replace_document' with a ;; term silently appends (it lands on the docid overload), which duplicates ;; the whole store on the second run. Deleting a term that is not there is ;; a no-op, so this is also the first-index case. (let ((term (string-append "K" key))) (false-if-exception (WritableDatabase-delete-document db term)) (WritableDatabase-add-document db (build-document stemmer record)))) ;;; --------------------------------------------------------------------- ;;; Reading the store ;; guile-sqlite3 gives results through a prepared statement: sqlite-map ;; runs a proc per row and returns them as a list. (define (query db sql) (let ((statement (sqlite-prepare db sql))) (let ((rows (sqlite-map vector->list statement))) (sqlite-finalize statement) rows))) (define (store-rows store) (let ((db (sqlite-open store SQLITE_OPEN_READONLY))) (let ((reasoning (query db "SELECT r.id, r.session_id, r.step, r.kind, r.body, COALESCE(r.commit_hash, ''), r.created_at, COALESCE(r.gist, '') FROM reasoning r ORDER BY r.id")) (commits (query db "SELECT hash, subject, COALESCE(body, ''), author_ts FROM commits ORDER BY author_ts")) ;; Which sessions' reasoning is attached to which commit, so a commit ;; document can carry those sessions' topics. (attached (query db "SELECT commit_hash, GROUP_CONCAT(DISTINCT session_id) FROM reasoning WHERE commit_hash IS NOT NULL AND session_id <> '' GROUP BY commit_hash"))) (sqlite-close db) (values reasoning commits attached)))) ;;; --------------------------------------------------------------------- ;;; Commands (define (cmd-update opts) (let ((store (option-ref opts 'store #f)) (index (option-ref opts 'index #f)) (rebuild (option-ref opts 'rebuild #f))) (unless store (die "update needs --store STORE.sqlite")) (unless index (die "update needs --index DIR")) (unless (file-exists? store) (die "no store at ~a" store)) (call-with-values (lambda () (store-rows store)) (lambda (reasoning commits attached) (let* ((repo (repo-name store)) (records (records repo reasoning commits attached)) (db (open-writable index rebuild)) (stemmer (new-Stem "english")) (written 0)) (for-each (lambda (entry) (index-record! db stemmer (first entry) (second entry)) (set! written (+ written 1))) records) (WritableDatabase-commit db) (delete-WritableDatabase db) (note "~a: ~a documents (~a reasoning, ~a commits)" (if rebuild "rebuilt" "updated") written (length reasoning) (length commits))))))) (define (make-query-parser db default-op) (let ((qp (new-QueryParser))) (QueryParser-set-stemmer qp (new-Stem "english")) (QueryParser-set-database qp db) (QueryParser-set-default-op qp default-op) (for-each (lambda (pair) (QueryParser-add-boolean-prefix qp (car pair) (cdr pair))) PREFIXES) qp)) (define (cmd-search opts words) (let ((index (option-ref opts 'index #f)) (limit (string->number (option-ref opts 'limit "10"))) (query-string (string-join words " "))) (unless index (die "search needs --index DIR")) (when (string-null? (string-trim-both query-string)) (die "search needs a query")) (let* ((db (open-readable index)) (qp (make-query-parser db (if (option-ref opts 'or #f) (Query-OP-OR) (Query-OP-AND)))) (enq (new-Enquire db))) (Enquire-set-query enq (QueryParser-parse-query qp query-string)) (Enquire-set-sort-by-value enq SLOT-DATE #t) (let* ((mset (Enquire-get-mset enq 0 limit)) (n (MSet-size mset)) (hits (let loop ((i 0) (acc (list))) (if (>= i n) acc (let* ((item (MSet-get-hit mset i)) (doc (MSetIterator-get-document item)) (record (Document-get-value doc SLOT-DOCDATA))) (loop (+ i 1) (cons (string-append "{\"docid\":" (number->string (MSetIterator-get-docid item)) ",\"percent\":" (number->string (MSetIterator-get-percent item)) ",\"record\":" record "}") acc))))))) (display (string-append "{\"query\":" (json-value query-string) ",\"count\":" (number->string (length hits)) ",\"matches\":" (number->string (MSet-get-matches-estimated mset)) ",\"hits\":[" (string-join (reverse hits) ",") "]}")) (newline)) (delete-Database db)))) (define (cmd-count opts) (let ((db (open-readable (option-ref opts 'index #f)))) (format #t "~a\n" (Database-get-doccount db)) (delete-Database db))) (define (usage) (format #t "usage: index.scm update --store STORE.sqlite --index DIR [--rebuild]~@ \x20 index.scm search --index DIR [--limit N] [--or] QUERY...~@ \x20 index.scm count --index DIR~%")) (define OPTION-SPEC '((store (value #t)) (index (value #t)) (limit (value #t)) (rebuild) (or) (quiet) (help (single-char #\h)))) (define (main args) (let* ((opts (getopt-long args OPTION-SPEC)) (rest (option-ref opts '() '()))) (parameterize ((CURRENT-OPTIONS opts)) (match rest (("update" . _) (cmd-update opts)) (("search" . words) (cmd-search opts words)) (("count" . _) (cmd-count opts)) (("schema" . _) (format #t "~a\n" SCHEMA-VERSION)) (("--help" . _) (usage)) (((? (lambda (a) (member a '("-h" "--help"))) _) . _) (usage)) ((cmd . _) (die "unknown command: ~a" cmd)) (() (usage)))))) (main (command-line))