about summary refs log tree commit diff
path: root/scripts
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-11-12 13:52:12 +0000
committerPjotr <wrk@bosa.lan>2026-11-12 13:52:12 +0000
commit3fab7adf0ec147dceb150d861fdbd5f5b3693384 (patch)
tree9e2f0b50a6d639eb821a9295bdfb4723f9a01535 /scripts
parent5cdd45571c550708f691855891c7919fe076ab05 (diff)
downloadai-memory-3fab7adf0ec147dceb150d861fdbd5f5b3693384.tar.gz
Add a topic index: Xapian over the store, driven through Guile
SQLite stays the truth; this is a derived index that lets a reader enter by
concept rather than by symbol. Two commands, one prefix vocabulary:

  ai-memory --index DIR update-index [--rebuild]
  ai-memory --index DIR search 'topic:pve repo:genenetwork4'

Prefixes: topic:, repo:, type: (reasoning|commit), session:, commit:,
date:. Free text is stemmed; default operator AND, --or to widen. Topic
terms come from the session id split on hyphens, so a session named
"gemma-pve" answers topic:pve and topic:gemma -- naming a session after a
concept is how an entry becomes findable by it.

Why Guile and a subprocess: libxapian is C++-only, so a foreign call from
Odin means a hand-written C shim with its own exception boundary; guile-xapian
already ships one, tested and maintained. So scripts/index.scm owns every
Xapian call (426 lines: schema constants, record builders, updater, query
parser, JSON writer) and ai-memory shells out to it -- index.odin is 145 lines
of process plumbing. The boundary is a store path in and JSON out, which also
means the search reply needs no parser on this side.

Incremental in the way that matters: documents are deleted by their unique
term and re-added, and Xapian applies that as changed postings only. The whole
body is re-tokenised each run (~1 s at a few hundred documents) which buys not
keeping any "what did I index last time" state that could go stale. Two
discoveries shaped it, both from guile-xapian 0.4.0's SWIG surface: replace by
term silently appends (the string lands on the docid overload), so delete then
add; and Database-get-document by term returns nothing, so no digest compare
was attempted.

The package's check phase now exercises it end to end -- throwaway store,
one entry, update-index, then `search topic:index | grep -q '"count":1'` --
because a check that only links proves nothing about the four Guile
dependencies. guix.scm installs scripts/index.scm to share/ai-memory (where
index.odin looks) and wraps the binary with GUILE and GUILE_LOAD_PATH for
guile-xapian and guile-sqlite3.

doc/SKILL.md gains the usage, the prefix list and the reason for the
subprocess.
Diffstat (limited to 'scripts')
-rw-r--r--scripts/index.scm426
1 files changed, 426 insertions, 0 deletions
diff --git a/scripts/index.scm b/scripts/index.scm
new file mode 100644
index 0000000..ac83850
--- /dev/null
+++ b/scripts/index.scm
@@ -0,0 +1,426 @@
+;;; index.scm --- a topic index over the ai-memory store, in Xapian
+;;; Copyright © 2026 Pjotr Prins <wrk@bosa.lan>
+;;;
+;;; ai-memory's SQLite store is the truth; this is a derived, disposable
+;;; Xapian index over it, in the notmuch style: one document per unit, keyed
+;;; by a unique term, fielded terms for routing, free text stemmed. Drop the
+;;; directory and `update --rebuild' to regenerate it.
+;;;
+;;; Why a separate program rather than FFI inside ai-memory: libxapian is
+;;; C++-only, so a foreign call means a hand-written C shim. guile-xapian
+;;; already has one -- tested, maintained -- and this is the API it exposes.
+;;; ai-memory shells out to this script; the boundary is a store path in and
+;;; JSON out, and SQLite stays the source of truth.
+;;;
+;;; Usage
+;;;   index.scm update --store STORE.sqlite --index DIR [--rebuild] [--quiet]
+;;;   index.scm search --index DIR [--limit N] [--or] QUERY...
+;;;   index.scm count  --index DIR
+;;;
+;;; Documents
+;;;   reasoning:<id>   one per reasoning entry (session, kind, commit)
+;;;   commit:<hash>    one per commit (subject + body, author date)
+;;;
+;;; Terms (boolean, case-folded on both sides, so prefix values are compared
+;;; exactly rather than stemmed)
+;;;   K<key>       identity: the replace key, never queried by a user
+;;;   T<type>      reasoning | commit
+;;;   R<repo>      repository name
+;;;   S<session>   session id
+;;;   O<topic>     one per word of the session id, so `topic:pve' is an entry
+;;;                point into the pve story
+;;;   C<hash>      the commit an entry is attached to (7 characters)
+;;;   D<yyyymmdd>  author date
+;;;
+;;;; Value slots
+;;;   0  date (yyyymmdd; lexicographic order is chronological)
+;;;   1  digest of the indexed text, for the incremental compare
+;;;   2  the document's JSON record -- so a hit is one value read and needs
+;;;      no second query
+;;;
+;;; `update' is incremental: each row's text is digested and compared with
+;;; slot 1 of the document already in the index; only new or changed rows are
+;;; replaced. Deletions are not propagated -- the store is append-only, so a
+;;; row that disappears is a `--rebuild'.
+
+(use-modules (ice-9 format)
+             (ice-9 match)
+             (ice-9 getopt-long)
+             (rnrs arithmetic bitwise)
+             (srfi srfi-1)
+             (srfi srfi-13)
+             (sqlite3)
+             (xapian wrap))
+
+;;; ---------------------------------------------------------------------
+;;; Schema. One place, so the indexer and the query parser cannot drift.
+
+(define SCHEMA-VERSION 1)
+
+(define SLOT-DATE 0)
+(define SLOT-DIGEST 1)
+(define SLOT-DOCDATA 2)
+
+;; User-facing prefix -> boolean term prefix.
+(define PREFIXES
+  '(("type:"    . "T")
+    ("repo:"    . "R")
+    ("session:" . "S")
+    ("topic:"   . "O")
+    ("commit:"  . "C")
+    ("date:"    . "D")))
+
+(define (term prefix value)
+  (string-append prefix (string-downcase value)))
+
+;;; ---------------------------------------------------------------------
+;;; Helpers
+
+(define CURRENT-OPTIONS (make-parameter (list)))
+
+(define (quiet?)
+  (option-ref (CURRENT-OPTIONS) 'quiet #f))
+
+(define (die fmt . args)
+  (format (current-error-port) "index.scm: ~a\n" (apply format #f fmt args))
+  (exit 1))
+
+(define (note fmt . args)
+  (unless (quiet?)
+    (apply format (current-error-port) fmt args)
+    (newline (current-error-port))))
+
+;; FNV-1a over the text: the only property needed is "changes when the text
+;; changes", and this avoids a crypto dependency for it.
+(define (digest str)
+  (let loop ((i 0) (h 2166136261))
+    (if (>= i (string-length str))
+        (number->string h)
+        (loop (+ i 1)
+              (modulo (* (bitwise-xor h (char->integer (string-ref str i)))
+                         16777619)
+                      4294967296)))))
+
+(define (yyyymmdd epoch)
+  (let* ((t (gmtime epoch))
+         (y (+ 1900 (tm:year t)))
+         (m (+ 1 (tm:mon t)))
+         (d (tm:mday t)))
+    (+ (* y 10000) (* m 100) d)))
+
+;; The store knows its repository only as a directory; the basename is the
+;; name a reader uses ("genenetwork4"), so that is the R term.
+(define (repo-name store)
+  (basename (dirname (canonicalize-path store))))
+
+(define (non-empty-string? value)
+  (and (string? value) (not (string-null? value))))
+
+;;; ---------------------------------------------------------------------
+;;; Database handles
+
+(define (open-writable index rebuild)
+  (unless rebuild
+    ;; Xapian creates its files inside DIR, but not DIR.
+    (unless (zero? (system* "mkdir" "-p" index))
+      (die "could not create ~a" index)))
+  (new-WritableDatabase index
+                        (if rebuild (DB-CREATE-OR-OVERWRITE) (DB-CREATE-OR-OPEN))))
+
+(define (open-readable index)
+  (unless (file-exists? index)
+    (die "no index at ~a: run `update' first" index))
+  (new-Database index))
+
+
+;;; ---------------------------------------------------------------------
+;;; JSON, emitted rather than parsed
+;;;
+;;; The index stores a JSON record per document (slot 2) and hands it back
+;;; verbatim, so nothing here ever parses JSON -- which is why this is a
+;;; writer of thirty lines instead of a library dependency. Objects are
+;;; alists of dotted pairs; values are strings, numbers, or lists of
+;;; strings; nothing else is stored.
+
+(define (json-escape text)
+  (let ((out (open-output-string)))
+    (string-for-each
+     (lambda (ch)
+       (case ch
+         ((#\") (display "\\\"" out))
+         ((#\\) (display "\\\\" out))
+         ((#\newline) (display "\\n" out))
+         ((#\tab) (display "\\t" out))
+         ((#\return) (display "\\r" out))
+         (else
+          (if (< (char->integer ch) 32)
+              (format out "\\u~4,'0x" (char->integer ch))
+              (write-char ch out)))))
+     text)
+    (get-output-string out)))
+
+(define (json-value value)
+  (cond ((string? value) (string-append "\"" (json-escape value) "\""))
+        ((number? value) (number->string value))
+        ((boolean? value) (if value "true" "false"))
+        ((list? value) (string-append "["
+                                      (string-join (map json-value value) ",")
+                                      "]"))
+        ((null? value) "null")
+        (else (json-value (format #f "~a" value)))))
+
+(define (json-object record)
+  (string-append "{"
+                 (string-join (map (lambda (pair)
+                                     (string-append (json-value (symbol->string (car pair)))
+                                                    ":"
+                                                    (json-value (second pair))))
+                                   record)
+                              ",")
+                 "}"))
+
+;;; ---------------------------------------------------------------------
+;;; Records -> documents
+;;;
+;;; A record is an alist with symbol keys, so it is both what we index and
+;;; what slot 2 carries: the caller renders exactly what the indexer stored.
+;;; guile-json wants object entries as two-element lists (key value).
+
+(define (field key value) (list key value))
+
+;; An entry is (key value), so the cdr is a one-element list -- assq-ref would
+;; hand back the wrapping list rather than the value.
+(define (lookup record key)
+  (let ((entry (assq key record)))
+    (and entry (second entry))))
+
+(define (reasoning-record repo row)
+  (match row
+    ((id session step kind body commit_hash created_at subject)
+     (let* ((key (format #f "reasoning:~a" id))
+            (date (yyyymmdd created_at))
+            (title (if (string? subject) (string-trim-both subject) ""))
+            (topics (if (non-empty-string? session)
+                        (map (lambda (word) (term "O" word))
+                             (string-split session #\-))
+                        (list)))
+            (terms (append (list (term "T" "reasoning")
+                                 (term "R" repo)
+                                 (term "D" (number->string date)))
+                           (if (non-empty-string? session)
+                               (cons (term "S" session) topics)
+                               (list))
+                           (if (non-empty-string? commit_hash)
+                               (list (term "C" (substring commit_hash 0 7)))
+                               (list)))))
+       (list key
+             (list (field 'type "reasoning")
+                   (field 'key key)
+                   (field 'repo repo)
+                   (field 'session (or session ""))
+                   (field 'kind (or kind ""))
+                   (field 'step (or step 0))
+                   (field 'date date)
+                   (field 'commit (or commit_hash ""))
+                   (field 'title title)
+                   (field 'terms terms)
+                   (field 'body (or body ""))
+                   (field 'text (string-append title " " (or body "")))))))
+    (else (die "unexpected reasoning row: ~s" row))))
+
+(define (commit-record repo row)
+  (match row
+    ((hash subject body author_ts)
+     (let* ((key (format #f "commit:~a" hash))
+            (date (yyyymmdd author_ts))
+            (subject (or subject "")))
+       (list key
+             (list (field 'type "commit")
+                   (field 'key key)
+                   (field 'repo repo)
+                   (field 'date date)
+                   (field 'commit hash)
+                   (field 'title subject)
+                   (field 'terms (list (term "T" "commit")
+                                       (term "R" repo)
+                                       (term "D" (number->string date))
+                                       (term "C" (substring hash 0 7))))
+                   (field 'body (or body ""))
+                   (field 'text (string-append subject " " (or body "")))))))
+    (else (die "unexpected commit row: ~s" row))))
+
+(define (records repo reasoning commits)
+  (append (map (lambda (row) (reasoning-record repo row)) reasoning)
+          (map (lambda (row) (commit-record repo row)) commits)))
+
+;;; ---------------------------------------------------------------------
+;;; Writing
+
+(define (build-document stemmer record)
+  (let ((doc (new-Document))
+        (text (lookup record 'text))
+        (date (lookup record 'date)))
+    (Document-set-data doc (json-object record))
+    ;; The identity term first: it is what the next run deletes by, so a
+    ;; document without it can never be replaced -- only duplicated.
+    (Document-add-boolean-term doc (string-append "K" (lookup record 'key)))
+    (for-each (lambda (t) (Document-add-boolean-term doc t))
+              (lookup record 'terms))
+    (Document-add-value doc SLOT-DATE (number->string date))
+    (Document-add-value doc SLOT-DIGEST (digest text))
+    (Document-add-value doc SLOT-DOCDATA (json-object record))
+    (let ((tg (new-TermGenerator)))
+      (TermGenerator-set-stemmer tg stemmer)
+      (TermGenerator-set-document tg doc)
+      (TermGenerator-index-text tg text))
+    doc))
+
+;; Every document is re-created: deleted by its unique term, then added.
+;; Xapian applies that as a delete plus an insert of the changed postings, so
+;; the on-disk work is incremental; this program merely re-tokenises every
+;; body (~1 s at a few thousand documents), which buys not having to remember
+;; what was indexed last time -- state that can go stale.
+(define (index-record! db stemmer key record)
+  ;; delete-then-add rather than replace-document: `delete_document' by
+  ;; unique term works through this binding, while `replace_document' with a
+  ;; term silently appends (it lands on the docid overload), which duplicates
+  ;; the whole store on the second run. Deleting a term that is not there is
+  ;; a no-op, so this is also the first-index case.
+  (let ((term (string-append "K" key)))
+    (false-if-exception (WritableDatabase-delete-document db term))
+    (WritableDatabase-add-document db (build-document stemmer record))))
+
+;;; ---------------------------------------------------------------------
+;;; Reading the store
+
+;; guile-sqlite3 gives results through a prepared statement: sqlite-map
+;; runs a proc per row and returns them as a list.
+(define (query db sql)
+  (let ((statement (sqlite-prepare db sql)))
+    (let ((rows (sqlite-map vector->list statement)))
+      (sqlite-finalize statement)
+      rows)))
+
+(define (store-rows store)
+  (let ((db (sqlite-open store SQLITE_OPEN_READONLY)))
+    (let ((reasoning (query db "SELECT r.id, r.session_id, r.step, r.kind, r.body,
+                                      COALESCE(r.commit_hash, ''), r.created_at,
+                                      COALESCE(c.subject, '')
+                               FROM reasoning r
+                               LEFT JOIN commits c ON c.hash = r.commit_hash
+                               ORDER BY r.id"))
+          (commits (query db "SELECT hash, subject, COALESCE(body, ''), author_ts
+                              FROM commits ORDER BY author_ts")))
+      (sqlite-close db)
+      (values reasoning commits))))
+
+;;; ---------------------------------------------------------------------
+;;; Commands
+
+(define (cmd-update opts)
+  (let ((store (option-ref opts 'store #f))
+        (index (option-ref opts 'index #f))
+        (rebuild (option-ref opts 'rebuild #f)))
+    (unless store (die "update needs --store STORE.sqlite"))
+    (unless index (die "update needs --index DIR"))
+    (unless (file-exists? store) (die "no store at ~a" store))
+    (call-with-values (lambda () (store-rows store))
+      (lambda (reasoning commits)
+        (let* ((repo (repo-name store))
+               (records (records repo reasoning commits))
+               (db (open-writable index rebuild))
+               (stemmer (new-Stem "english"))
+               (written 0))
+          (for-each (lambda (entry)
+                      (index-record! db stemmer (first entry) (second entry))
+                      (set! written (+ written 1)))
+                    records)
+          (WritableDatabase-commit db)
+          (delete-WritableDatabase db)
+          (note "~a: ~a documents (~a reasoning, ~a commits)"
+                (if rebuild "rebuilt" "updated") written
+                (length reasoning) (length commits)))))))
+
+(define (make-query-parser db default-op)
+  (let ((qp (new-QueryParser)))
+    (QueryParser-set-stemmer qp (new-Stem "english"))
+    (QueryParser-set-database qp db)
+    (QueryParser-set-default-op qp default-op)
+    (for-each (lambda (pair)
+                (QueryParser-add-boolean-prefix qp (car pair) (cdr pair)))
+              PREFIXES)
+    qp))
+
+(define (cmd-search opts words)
+  (let ((index (option-ref opts 'index #f))
+        (limit (string->number (option-ref opts 'limit "10")))
+        (query-string (string-join words " ")))
+    (unless index (die "search needs --index DIR"))
+    (when (string-null? (string-trim-both query-string))
+      (die "search needs a query"))
+    (let* ((db (open-readable index))
+           (qp (make-query-parser db (if (option-ref opts 'or #f)
+                                         (Query-OP-OR)
+                                         (Query-OP-AND))))
+           (enq (new-Enquire db)))
+      (Enquire-set-query enq (QueryParser-parse-query qp query-string))
+      (Enquire-set-sort-by-value enq SLOT-DATE #t)
+      (let* ((mset (Enquire-get-mset enq 0 limit))
+             (n (MSet-size mset))
+             (hits (let loop ((i 0) (acc (list)))
+                     (if (>= i n)
+                         acc
+                         (let* ((item (MSet-get-hit mset i))
+                                (doc (MSetIterator-get-document item))
+                                (record (Document-get-value doc SLOT-DOCDATA)))
+                           (loop (+ i 1)
+                                 (cons (string-append
+                                        "{\"docid\":" (number->string (MSetIterator-get-docid item))
+                                        ",\"percent\":" (number->string (MSetIterator-get-percent item))
+                                        ",\"record\":" record "}")
+                                       acc)))))))
+        (display
+         (string-append
+          "{\"query\":" (json-value query-string)
+          ",\"count\":" (number->string (length hits))
+          ",\"matches\":" (number->string (MSet-get-matches-estimated mset))
+          ",\"hits\":["
+          (string-join (reverse hits) ",")
+          "]}"))
+        (newline))
+      (delete-Database db))))
+
+(define (cmd-count opts)
+  (let ((db (open-readable (option-ref opts 'index #f))))
+    (format #t "~a\n" (Database-get-doccount db))
+    (delete-Database db)))
+
+(define (usage)
+  (format #t "usage: index.scm update --store STORE.sqlite --index DIR [--rebuild]~@
+              \x20      index.scm search --index DIR [--limit N] [--or] QUERY...~@
+              \x20      index.scm count  --index DIR~%"))
+
+(define OPTION-SPEC
+  '((store (value #t))
+    (index (value #t))
+    (limit (value #t))
+    (rebuild)
+    (or)
+    (quiet)
+    (help (single-char #\h))))
+
+(define (main args)
+  (let* ((opts (getopt-long args OPTION-SPEC))
+         (rest (option-ref opts '() '())))
+    (parameterize ((CURRENT-OPTIONS opts))
+      (match rest
+        (("update" . _) (cmd-update opts))
+        (("search" . words) (cmd-search opts words))
+        (("count" . _) (cmd-count opts))
+        (("schema" . _) (format #t "~a\n" SCHEMA-VERSION))
+        (("--help" . _) (usage))
+        (((? (lambda (a) (member a '("-h" "--help"))) _) . _) (usage))
+        ((cmd . _) (die "unknown command: ~a" cmd))
+        (() (usage))))))
+
+(main (command-line))