diff options
| author | Pjotr <wrk@bosa.lan> | 2026-11-12 13:52:12 +0000 |
|---|---|---|
| committer | Pjotr <wrk@bosa.lan> | 2026-11-12 13:52:12 +0000 |
| commit | 3fab7adf0ec147dceb150d861fdbd5f5b3693384 (patch) | |
| tree | 9e2f0b50a6d639eb821a9295bdfb4723f9a01535 /scripts | |
| parent | 5cdd45571c550708f691855891c7919fe076ab05 (diff) | |
| download | ai-memory-3fab7adf0ec147dceb150d861fdbd5f5b3693384.tar.gz | |
Add a topic index: Xapian over the store, driven through Guile
SQLite stays the truth; this is a derived index that lets a reader enter by concept rather than by symbol. Two commands, one prefix vocabulary: ai-memory --index DIR update-index [--rebuild] ai-memory --index DIR search 'topic:pve repo:genenetwork4' Prefixes: topic:, repo:, type: (reasoning|commit), session:, commit:, date:. Free text is stemmed; default operator AND, --or to widen. Topic terms come from the session id split on hyphens, so a session named "gemma-pve" answers topic:pve and topic:gemma -- naming a session after a concept is how an entry becomes findable by it. Why Guile and a subprocess: libxapian is C++-only, so a foreign call from Odin means a hand-written C shim with its own exception boundary; guile-xapian already ships one, tested and maintained. So scripts/index.scm owns every Xapian call (426 lines: schema constants, record builders, updater, query parser, JSON writer) and ai-memory shells out to it -- index.odin is 145 lines of process plumbing. The boundary is a store path in and JSON out, which also means the search reply needs no parser on this side. Incremental in the way that matters: documents are deleted by their unique term and re-added, and Xapian applies that as changed postings only. The whole body is re-tokenised each run (~1 s at a few hundred documents) which buys not keeping any "what did I index last time" state that could go stale. Two discoveries shaped it, both from guile-xapian 0.4.0's SWIG surface: replace by term silently appends (the string lands on the docid overload), so delete then add; and Database-get-document by term returns nothing, so no digest compare was attempted. The package's check phase now exercises it end to end -- throwaway store, one entry, update-index, then `search topic:index | grep -q '"count":1'` -- because a check that only links proves nothing about the four Guile dependencies. guix.scm installs scripts/index.scm to share/ai-memory (where index.odin looks) and wraps the binary with GUILE and GUILE_LOAD_PATH for guile-xapian and guile-sqlite3. doc/SKILL.md gains the usage, the prefix list and the reason for the subprocess.
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/index.scm | 426 |
1 files changed, 426 insertions, 0 deletions
diff --git a/scripts/index.scm b/scripts/index.scm new file mode 100644 index 0000000..ac83850 --- /dev/null +++ b/scripts/index.scm @@ -0,0 +1,426 @@ +;;; index.scm --- a topic index over the ai-memory store, in Xapian +;;; Copyright © 2026 Pjotr Prins <wrk@bosa.lan> +;;; +;;; ai-memory's SQLite store is the truth; this is a derived, disposable +;;; Xapian index over it, in the notmuch style: one document per unit, keyed +;;; by a unique term, fielded terms for routing, free text stemmed. Drop the +;;; directory and `update --rebuild' to regenerate it. +;;; +;;; Why a separate program rather than FFI inside ai-memory: libxapian is +;;; C++-only, so a foreign call means a hand-written C shim. guile-xapian +;;; already has one -- tested, maintained -- and this is the API it exposes. +;;; ai-memory shells out to this script; the boundary is a store path in and +;;; JSON out, and SQLite stays the source of truth. +;;; +;;; Usage +;;; index.scm update --store STORE.sqlite --index DIR [--rebuild] [--quiet] +;;; index.scm search --index DIR [--limit N] [--or] QUERY... +;;; index.scm count --index DIR +;;; +;;; Documents +;;; reasoning:<id> one per reasoning entry (session, kind, commit) +;;; commit:<hash> one per commit (subject + body, author date) +;;; +;;; Terms (boolean, case-folded on both sides, so prefix values are compared +;;; exactly rather than stemmed) +;;; K<key> identity: the replace key, never queried by a user +;;; T<type> reasoning | commit +;;; R<repo> repository name +;;; S<session> session id +;;; O<topic> one per word of the session id, so `topic:pve' is an entry +;;; point into the pve story +;;; C<hash> the commit an entry is attached to (7 characters) +;;; D<yyyymmdd> author date +;;; +;;;; Value slots +;;; 0 date (yyyymmdd; lexicographic order is chronological) +;;; 1 digest of the indexed text, for the incremental compare +;;; 2 the document's JSON record -- so a hit is one value read and needs +;;; no second query +;;; +;;; `update' is incremental: each row's text is digested and compared with +;;; slot 1 of the document already in the index; only new or changed rows are +;;; replaced. Deletions are not propagated -- the store is append-only, so a +;;; row that disappears is a `--rebuild'. + +(use-modules (ice-9 format) + (ice-9 match) + (ice-9 getopt-long) + (rnrs arithmetic bitwise) + (srfi srfi-1) + (srfi srfi-13) + (sqlite3) + (xapian wrap)) + +;;; --------------------------------------------------------------------- +;;; Schema. One place, so the indexer and the query parser cannot drift. + +(define SCHEMA-VERSION 1) + +(define SLOT-DATE 0) +(define SLOT-DIGEST 1) +(define SLOT-DOCDATA 2) + +;; User-facing prefix -> boolean term prefix. +(define PREFIXES + '(("type:" . "T") + ("repo:" . "R") + ("session:" . "S") + ("topic:" . "O") + ("commit:" . "C") + ("date:" . "D"))) + +(define (term prefix value) + (string-append prefix (string-downcase value))) + +;;; --------------------------------------------------------------------- +;;; Helpers + +(define CURRENT-OPTIONS (make-parameter (list))) + +(define (quiet?) + (option-ref (CURRENT-OPTIONS) 'quiet #f)) + +(define (die fmt . args) + (format (current-error-port) "index.scm: ~a\n" (apply format #f fmt args)) + (exit 1)) + +(define (note fmt . args) + (unless (quiet?) + (apply format (current-error-port) fmt args) + (newline (current-error-port)))) + +;; FNV-1a over the text: the only property needed is "changes when the text +;; changes", and this avoids a crypto dependency for it. +(define (digest str) + (let loop ((i 0) (h 2166136261)) + (if (>= i (string-length str)) + (number->string h) + (loop (+ i 1) + (modulo (* (bitwise-xor h (char->integer (string-ref str i))) + 16777619) + 4294967296))))) + +(define (yyyymmdd epoch) + (let* ((t (gmtime epoch)) + (y (+ 1900 (tm:year t))) + (m (+ 1 (tm:mon t))) + (d (tm:mday t))) + (+ (* y 10000) (* m 100) d))) + +;; The store knows its repository only as a directory; the basename is the +;; name a reader uses ("genenetwork4"), so that is the R term. +(define (repo-name store) + (basename (dirname (canonicalize-path store)))) + +(define (non-empty-string? value) + (and (string? value) (not (string-null? value)))) + +;;; --------------------------------------------------------------------- +;;; Database handles + +(define (open-writable index rebuild) + (unless rebuild + ;; Xapian creates its files inside DIR, but not DIR. + (unless (zero? (system* "mkdir" "-p" index)) + (die "could not create ~a" index))) + (new-WritableDatabase index + (if rebuild (DB-CREATE-OR-OVERWRITE) (DB-CREATE-OR-OPEN)))) + +(define (open-readable index) + (unless (file-exists? index) + (die "no index at ~a: run `update' first" index)) + (new-Database index)) + + +;;; --------------------------------------------------------------------- +;;; JSON, emitted rather than parsed +;;; +;;; The index stores a JSON record per document (slot 2) and hands it back +;;; verbatim, so nothing here ever parses JSON -- which is why this is a +;;; writer of thirty lines instead of a library dependency. Objects are +;;; alists of dotted pairs; values are strings, numbers, or lists of +;;; strings; nothing else is stored. + +(define (json-escape text) + (let ((out (open-output-string))) + (string-for-each + (lambda (ch) + (case ch + ((#\") (display "\\\"" out)) + ((#\\) (display "\\\\" out)) + ((#\newline) (display "\\n" out)) + ((#\tab) (display "\\t" out)) + ((#\return) (display "\\r" out)) + (else + (if (< (char->integer ch) 32) + (format out "\\u~4,'0x" (char->integer ch)) + (write-char ch out))))) + text) + (get-output-string out))) + +(define (json-value value) + (cond ((string? value) (string-append "\"" (json-escape value) "\"")) + ((number? value) (number->string value)) + ((boolean? value) (if value "true" "false")) + ((list? value) (string-append "[" + (string-join (map json-value value) ",") + "]")) + ((null? value) "null") + (else (json-value (format #f "~a" value))))) + +(define (json-object record) + (string-append "{" + (string-join (map (lambda (pair) + (string-append (json-value (symbol->string (car pair))) + ":" + (json-value (second pair)))) + record) + ",") + "}")) + +;;; --------------------------------------------------------------------- +;;; Records -> documents +;;; +;;; A record is an alist with symbol keys, so it is both what we index and +;;; what slot 2 carries: the caller renders exactly what the indexer stored. +;;; guile-json wants object entries as two-element lists (key value). + +(define (field key value) (list key value)) + +;; An entry is (key value), so the cdr is a one-element list -- assq-ref would +;; hand back the wrapping list rather than the value. +(define (lookup record key) + (let ((entry (assq key record))) + (and entry (second entry)))) + +(define (reasoning-record repo row) + (match row + ((id session step kind body commit_hash created_at subject) + (let* ((key (format #f "reasoning:~a" id)) + (date (yyyymmdd created_at)) + (title (if (string? subject) (string-trim-both subject) "")) + (topics (if (non-empty-string? session) + (map (lambda (word) (term "O" word)) + (string-split session #\-)) + (list))) + (terms (append (list (term "T" "reasoning") + (term "R" repo) + (term "D" (number->string date))) + (if (non-empty-string? session) + (cons (term "S" session) topics) + (list)) + (if (non-empty-string? commit_hash) + (list (term "C" (substring commit_hash 0 7))) + (list))))) + (list key + (list (field 'type "reasoning") + (field 'key key) + (field 'repo repo) + (field 'session (or session "")) + (field 'kind (or kind "")) + (field 'step (or step 0)) + (field 'date date) + (field 'commit (or commit_hash "")) + (field 'title title) + (field 'terms terms) + (field 'body (or body "")) + (field 'text (string-append title " " (or body ""))))))) + (else (die "unexpected reasoning row: ~s" row)))) + +(define (commit-record repo row) + (match row + ((hash subject body author_ts) + (let* ((key (format #f "commit:~a" hash)) + (date (yyyymmdd author_ts)) + (subject (or subject ""))) + (list key + (list (field 'type "commit") + (field 'key key) + (field 'repo repo) + (field 'date date) + (field 'commit hash) + (field 'title subject) + (field 'terms (list (term "T" "commit") + (term "R" repo) + (term "D" (number->string date)) + (term "C" (substring hash 0 7)))) + (field 'body (or body "")) + (field 'text (string-append subject " " (or body ""))))))) + (else (die "unexpected commit row: ~s" row)))) + +(define (records repo reasoning commits) + (append (map (lambda (row) (reasoning-record repo row)) reasoning) + (map (lambda (row) (commit-record repo row)) commits))) + +;;; --------------------------------------------------------------------- +;;; Writing + +(define (build-document stemmer record) + (let ((doc (new-Document)) + (text (lookup record 'text)) + (date (lookup record 'date))) + (Document-set-data doc (json-object record)) + ;; The identity term first: it is what the next run deletes by, so a + ;; document without it can never be replaced -- only duplicated. + (Document-add-boolean-term doc (string-append "K" (lookup record 'key))) + (for-each (lambda (t) (Document-add-boolean-term doc t)) + (lookup record 'terms)) + (Document-add-value doc SLOT-DATE (number->string date)) + (Document-add-value doc SLOT-DIGEST (digest text)) + (Document-add-value doc SLOT-DOCDATA (json-object record)) + (let ((tg (new-TermGenerator))) + (TermGenerator-set-stemmer tg stemmer) + (TermGenerator-set-document tg doc) + (TermGenerator-index-text tg text)) + doc)) + +;; Every document is re-created: deleted by its unique term, then added. +;; Xapian applies that as a delete plus an insert of the changed postings, so +;; the on-disk work is incremental; this program merely re-tokenises every +;; body (~1 s at a few thousand documents), which buys not having to remember +;; what was indexed last time -- state that can go stale. +(define (index-record! db stemmer key record) + ;; delete-then-add rather than replace-document: `delete_document' by + ;; unique term works through this binding, while `replace_document' with a + ;; term silently appends (it lands on the docid overload), which duplicates + ;; the whole store on the second run. Deleting a term that is not there is + ;; a no-op, so this is also the first-index case. + (let ((term (string-append "K" key))) + (false-if-exception (WritableDatabase-delete-document db term)) + (WritableDatabase-add-document db (build-document stemmer record)))) + +;;; --------------------------------------------------------------------- +;;; Reading the store + +;; guile-sqlite3 gives results through a prepared statement: sqlite-map +;; runs a proc per row and returns them as a list. +(define (query db sql) + (let ((statement (sqlite-prepare db sql))) + (let ((rows (sqlite-map vector->list statement))) + (sqlite-finalize statement) + rows))) + +(define (store-rows store) + (let ((db (sqlite-open store SQLITE_OPEN_READONLY))) + (let ((reasoning (query db "SELECT r.id, r.session_id, r.step, r.kind, r.body, + COALESCE(r.commit_hash, ''), r.created_at, + COALESCE(c.subject, '') + FROM reasoning r + LEFT JOIN commits c ON c.hash = r.commit_hash + ORDER BY r.id")) + (commits (query db "SELECT hash, subject, COALESCE(body, ''), author_ts + FROM commits ORDER BY author_ts"))) + (sqlite-close db) + (values reasoning commits)))) + +;;; --------------------------------------------------------------------- +;;; Commands + +(define (cmd-update opts) + (let ((store (option-ref opts 'store #f)) + (index (option-ref opts 'index #f)) + (rebuild (option-ref opts 'rebuild #f))) + (unless store (die "update needs --store STORE.sqlite")) + (unless index (die "update needs --index DIR")) + (unless (file-exists? store) (die "no store at ~a" store)) + (call-with-values (lambda () (store-rows store)) + (lambda (reasoning commits) + (let* ((repo (repo-name store)) + (records (records repo reasoning commits)) + (db (open-writable index rebuild)) + (stemmer (new-Stem "english")) + (written 0)) + (for-each (lambda (entry) + (index-record! db stemmer (first entry) (second entry)) + (set! written (+ written 1))) + records) + (WritableDatabase-commit db) + (delete-WritableDatabase db) + (note "~a: ~a documents (~a reasoning, ~a commits)" + (if rebuild "rebuilt" "updated") written + (length reasoning) (length commits))))))) + +(define (make-query-parser db default-op) + (let ((qp (new-QueryParser))) + (QueryParser-set-stemmer qp (new-Stem "english")) + (QueryParser-set-database qp db) + (QueryParser-set-default-op qp default-op) + (for-each (lambda (pair) + (QueryParser-add-boolean-prefix qp (car pair) (cdr pair))) + PREFIXES) + qp)) + +(define (cmd-search opts words) + (let ((index (option-ref opts 'index #f)) + (limit (string->number (option-ref opts 'limit "10"))) + (query-string (string-join words " "))) + (unless index (die "search needs --index DIR")) + (when (string-null? (string-trim-both query-string)) + (die "search needs a query")) + (let* ((db (open-readable index)) + (qp (make-query-parser db (if (option-ref opts 'or #f) + (Query-OP-OR) + (Query-OP-AND)))) + (enq (new-Enquire db))) + (Enquire-set-query enq (QueryParser-parse-query qp query-string)) + (Enquire-set-sort-by-value enq SLOT-DATE #t) + (let* ((mset (Enquire-get-mset enq 0 limit)) + (n (MSet-size mset)) + (hits (let loop ((i 0) (acc (list))) + (if (>= i n) + acc + (let* ((item (MSet-get-hit mset i)) + (doc (MSetIterator-get-document item)) + (record (Document-get-value doc SLOT-DOCDATA))) + (loop (+ i 1) + (cons (string-append + "{\"docid\":" (number->string (MSetIterator-get-docid item)) + ",\"percent\":" (number->string (MSetIterator-get-percent item)) + ",\"record\":" record "}") + acc))))))) + (display + (string-append + "{\"query\":" (json-value query-string) + ",\"count\":" (number->string (length hits)) + ",\"matches\":" (number->string (MSet-get-matches-estimated mset)) + ",\"hits\":[" + (string-join (reverse hits) ",") + "]}")) + (newline)) + (delete-Database db)))) + +(define (cmd-count opts) + (let ((db (open-readable (option-ref opts 'index #f)))) + (format #t "~a\n" (Database-get-doccount db)) + (delete-Database db))) + +(define (usage) + (format #t "usage: index.scm update --store STORE.sqlite --index DIR [--rebuild]~@ + \x20 index.scm search --index DIR [--limit N] [--or] QUERY...~@ + \x20 index.scm count --index DIR~%")) + +(define OPTION-SPEC + '((store (value #t)) + (index (value #t)) + (limit (value #t)) + (rebuild) + (or) + (quiet) + (help (single-char #\h)))) + +(define (main args) + (let* ((opts (getopt-long args OPTION-SPEC)) + (rest (option-ref opts '() '()))) + (parameterize ((CURRENT-OPTIONS opts)) + (match rest + (("update" . _) (cmd-update opts)) + (("search" . words) (cmd-search opts words)) + (("count" . _) (cmd-count opts)) + (("schema" . _) (format #t "~a\n" SCHEMA-VERSION)) + (("--help" . _) (usage)) + (((? (lambda (a) (member a '("-h" "--help"))) _) . _) (usage)) + ((cmd . _) (die "unknown command: ~a" cmd)) + (() (usage)))))) + +(main (command-line)) |
