about summary refs log tree commit diff
path: root/scripts/index.scm
blob: e812f94c5eff08333d6ded1e85a01afd9f1906c4 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
;;; index.scm --- a topic index over the ai-memory store, in Xapian
;;; Copyright © 2026 Pjotr Prins <wrk@bosa.lan>
;;;
;;; ai-memory's SQLite store is the truth; this is a derived, disposable
;;; Xapian index over it, in the notmuch style: one document per unit, keyed
;;; by a unique term, fielded terms for routing, free text stemmed. Drop the
;;; directory and `update --rebuild' to regenerate it.
;;;
;;; Why a separate program rather than FFI inside ai-memory: libxapian is
;;; C++-only, so a foreign call means a hand-written C shim. guile-xapian
;;; already has one -- tested, maintained -- and this is the API it exposes.
;;; ai-memory shells out to this script; the boundary is a store path in and
;;; JSON out, and SQLite stays the source of truth.
;;;
;;; Usage
;;;   index.scm update --store STORE.sqlite --index DIR [--rebuild] [--quiet]
;;;   index.scm search --index DIR [--limit N] [--or] QUERY...
;;;   index.scm count  --index DIR
;;;
;;; Documents
;;;   reasoning:<id>   one per reasoning entry (session, kind, commit)
;;;   commit:<hash>    one per commit (subject + body, author date)
;;;
;;; Terms (boolean, case-folded on both sides, so prefix values are compared
;;; exactly rather than stemmed)
;;;   K<key>       identity: the replace key, never queried by a user
;;;   T<type>      reasoning | commit
;;;   R<repo>      repository name
;;;   S<session>   session id
;;;   O<topic>     one per word of the session id, so `topic:pve' is an entry
;;;                point into the pve story
;;;   C<hash>      the commit an entry is attached to (7 characters)
;;;   D<yyyymmdd>  author date
;;;
;;;; Value slots
;;;   0  date (yyyymmdd; lexicographic order is chronological)
;;;   1  digest of the indexed text, for the incremental compare
;;;   2  the document's JSON record -- so a hit is one value read and needs
;;;      no second query
;;;
;;; `update' is incremental: each row's text is digested and compared with
;;; slot 1 of the document already in the index; only new or changed rows are
;;; replaced. Deletions are not propagated -- the store is append-only, so a
;;; row that disappears is a `--rebuild'.

(use-modules (ice-9 format)
             (ice-9 match)
             (ice-9 getopt-long)
             (rnrs arithmetic bitwise)
             (srfi srfi-1)
             (srfi srfi-13)
             (sqlite3)
             (xapian wrap))

;;; ---------------------------------------------------------------------
;;; Schema. One place, so the indexer and the query parser cannot drift.

(define SCHEMA-VERSION 1)

(define SLOT-DATE 0)
(define SLOT-DIGEST 1)
(define SLOT-DOCDATA 2)

;; User-facing prefix -> boolean term prefix.
(define PREFIXES
  '(("type:"    . "T")
    ("repo:"    . "R")
    ("session:" . "S")
    ("topic:"   . "O")
    ("commit:"  . "C")
    ("date:"    . "D")))

(define (term prefix value)
  (string-append prefix (string-downcase value)))

;;; ---------------------------------------------------------------------
;;; Helpers

(define CURRENT-OPTIONS (make-parameter (list)))

(define (quiet?)
  (option-ref (CURRENT-OPTIONS) 'quiet #f))

(define (die fmt . args)
  (format (current-error-port) "index.scm: ~a\n" (apply format #f fmt args))
  (exit 1))

(define (note fmt . args)
  (unless (quiet?)
    (apply format (current-error-port) fmt args)
    (newline (current-error-port))))

;; FNV-1a over the text: the only property needed is "changes when the text
;; changes", and this avoids a crypto dependency for it.
(define (digest str)
  (let loop ((i 0) (h 2166136261))
    (if (>= i (string-length str))
        (number->string h)
        (loop (+ i 1)
              (modulo (* (bitwise-xor h (char->integer (string-ref str i)))
                         16777619)
                      4294967296)))))

(define (yyyymmdd epoch)
  (let* ((t (gmtime epoch))
         (y (+ 1900 (tm:year t)))
         (m (+ 1 (tm:mon t)))
         (d (tm:mday t)))
    (+ (* y 10000) (* m 100) d)))

;; The store knows its repository only as a directory; the basename is the
;; name a reader uses ("genenetwork4"), so that is the R term.
(define (repo-name store)
  (basename (dirname (canonicalize-path store))))

(define (alist-set table key value)
  (cons (cons key value) table))

(define (non-empty-string? value)
  (and (string? value) (not (string-null? value))))

;;; ---------------------------------------------------------------------
;;; Database handles

(define (open-writable index rebuild)
  ;; Xapian creates its files inside DIR, but not DIR -- and only the plain
  ;; `update' path used to create it, so `update --rebuild' on a store whose
  ;; index directory was deleted (the documented way to recover, and the state
  ;; of every store that has never been indexed) died in new-WritableDatabase
  ;; with "mkdir failed". `mkdir -p' is a no-op when the directory is there.
  (unless (zero? (system* "mkdir" "-p" index))
    (die "could not create ~a" index))
  (new-WritableDatabase index
                        (if rebuild (DB-CREATE-OR-OVERWRITE) (DB-CREATE-OR-OPEN))))

(define (open-readable index)
  (unless (file-exists? index)
    (die "no index at ~a: run `update' first" index))
  (new-Database index))


;;; ---------------------------------------------------------------------
;;; JSON, emitted rather than parsed
;;;
;;; The index stores a JSON record per document (slot 2) and hands it back
;;; verbatim, so nothing here ever parses JSON -- which is why this is a
;;; writer of thirty lines instead of a library dependency. Objects are
;;; alists of dotted pairs; values are strings, numbers, or lists of
;;; strings; nothing else is stored.

(define (json-escape text)
  (let ((out (open-output-string)))
    (string-for-each
     (lambda (ch)
       (case ch
         ((#\") (display "\\\"" out))
         ((#\\) (display "\\\\" out))
         ((#\newline) (display "\\n" out))
         ((#\tab) (display "\\t" out))
         ((#\return) (display "\\r" out))
         (else
          (if (< (char->integer ch) 32)
              (format out "\\u~4,'0x" (char->integer ch))
              (write-char ch out)))))
     text)
    (get-output-string out)))

(define (json-value value)
  (cond ((string? value) (string-append "\"" (json-escape value) "\""))
        ((number? value) (number->string value))
        ((boolean? value) (if value "true" "false"))
        ((list? value) (string-append "["
                                      (string-join (map json-value value) ",")
                                      "]"))
        ((null? value) "null")
        (else (json-value (format #f "~a" value)))))

(define (json-object record)
  (string-append "{"
                 (string-join (map (lambda (pair)
                                     (string-append (json-value (symbol->string (car pair)))
                                                    ":"
                                                    (json-value (second pair))))
                                   record)
                              ",")
                 "}"))

;;; ---------------------------------------------------------------------
;;; Records -> documents
;;;
;;; A record is an alist with symbol keys, so it is both what we index and
;;; what slot 2 carries: the caller renders exactly what the indexer stored.
;;; guile-json wants object entries as two-element lists (key value).

(define (field key value) (list key value))

;; An entry is (key value), so the cdr is a one-element list -- assq-ref would
;; hand back the wrapping list rather than the value.
(define (lookup record key)
  (let ((entry (assq key record)))
    (and entry (second entry))))

;;; The title is the entry's own gist -- the assessed one-line claim, which is
;;; what `list' and `why' show. It is deliberately NOT the subject of the
;;; commit the entry is attached to: that subject is already the title of the
;;; commit document, and an entry recorded with `--commit none' has no commit
;;; to borrow from, which left it indexed with an empty title and its gist
;;; unsearchable.
(define (reasoning-record repo row)
  (match row
    ((id session step kind body commit_hash created_at gist)
     (let* ((key (format #f "reasoning:~a" id))
            (date (yyyymmdd created_at))
            (title (if (string? gist) (string-trim-both gist) ""))
            (topics (if (non-empty-string? session)
                        (map (lambda (word) (term "O" word))
                             (string-split session #\-))
                        (list)))
            (terms (append (list (term "T" "reasoning")
                                 (term "R" repo)
                                 (term "D" (number->string date)))
                           (if (non-empty-string? session)
                               (cons (term "S" session) topics)
                               (list))
                           (if (non-empty-string? commit_hash)
                               (list (term "C" (substring commit_hash 0 7)))
                               (list)))))
       (list key
             (list (field 'type "reasoning")
                   (field 'key key)
                   (field 'repo repo)
                   (field 'session (or session ""))
                   (field 'kind (or kind ""))
                   (field 'step (or step 0))
                   (field 'date date)
                   (field 'commit (or commit_hash ""))
                   (field 'title title)
                   (field 'terms terms)
                   (field 'body (or body ""))
                   (field 'text (string-append title " " (or body "")))))))
    (else (die "unexpected reasoning row: ~s" row))))

(define (commit-record repo row sessions)
  (match row
    ((hash subject body author_ts)
     (let* ((key (format #f "commit:~a" hash))
            (date (yyyymmdd author_ts))
            (subject (or subject ""))
            ;; A commit inherits the topics of the reasoning attached to it:
            ;; otherwise the pve story answers with its reasoning and not with
            ;; the commits that carry it, which is exactly backwards for a
            ;; reader following it.
            (topics (append-map (lambda (session)
                                  (map (lambda (word) (term "O" word))
                                       (string-split session #\-)))
                                sessions)))
       (list key
             (list (field 'type "commit")
                   (field 'key key)
                   (field 'repo repo)
                   (field 'date date)
                   (field 'commit hash)
                   (field 'title subject)
                   (field 'session (if (null? sessions) "" (car sessions)))
                   (field 'terms (append (list (term "T" "commit")
                                               (term "R" repo)
                                               (term "D" (number->string date))
                                               (term "C" (substring hash 0 7)))
                                         (map (lambda (session) (term "S" session))
                                              sessions)
                                         topics))
                   (field 'body (or body ""))
                   (field 'text (string-append subject " " (or body "")))))))
    (else (die "unexpected commit row: ~s" row))))

(define (sessions-by-commit attached)
  (fold (lambda (row table)
          (match row
            ((commit_hash sessions)
             (alist-set table commit_hash
                        (if sessions (string-split sessions #\,) (list))))
            (else table)))
        '()
        attached))

(define (records repo reasoning commits attached)
  (let ((sessions (sessions-by-commit attached)))
    (append (map (lambda (row) (reasoning-record repo row)) reasoning)
            (map (lambda (row)
                   (commit-record repo row
                                  (or (assoc-ref sessions (first row)) (list))))
                 commits))))

;;; ---------------------------------------------------------------------
;;; Writing

(define (build-document stemmer record)
  (let ((doc (new-Document))
        (text (lookup record 'text))
        (date (lookup record 'date)))
    (Document-set-data doc (json-object record))
    ;; The identity term first: it is what the next run deletes by, so a
    ;; document without it can never be replaced -- only duplicated.
    (Document-add-boolean-term doc (string-append "K" (lookup record 'key)))
    (for-each (lambda (t) (Document-add-boolean-term doc t))
              (lookup record 'terms))
    (Document-add-value doc SLOT-DATE (number->string date))
    (Document-add-value doc SLOT-DIGEST (digest text))
    (Document-add-value doc SLOT-DOCDATA (json-object record))
    (let ((tg (new-TermGenerator)))
      (TermGenerator-set-stemmer tg stemmer)
      (TermGenerator-set-document tg doc)
      (TermGenerator-index-text tg text))
    doc))

;; Every document is re-created: deleted by its unique term, then added.
;; Xapian applies that as a delete plus an insert of the changed postings, so
;; the on-disk work is incremental; this program merely re-tokenises every
;; body (~1 s at a few thousand documents), which buys not having to remember
;; what was indexed last time -- state that can go stale.
(define (index-record! db stemmer key record)
  ;; delete-then-add rather than replace-document: `delete_document' by
  ;; unique term works through this binding, while `replace_document' with a
  ;; term silently appends (it lands on the docid overload), which duplicates
  ;; the whole store on the second run. Deleting a term that is not there is
  ;; a no-op, so this is also the first-index case.
  (let ((term (string-append "K" key)))
    (false-if-exception (WritableDatabase-delete-document db term))
    (WritableDatabase-add-document db (build-document stemmer record))))

;;; ---------------------------------------------------------------------
;;; Reading the store

;; guile-sqlite3 gives results through a prepared statement: sqlite-map
;; runs a proc per row and returns them as a list.
(define (query db sql)
  (let ((statement (sqlite-prepare db sql)))
    (let ((rows (sqlite-map vector->list statement)))
      (sqlite-finalize statement)
      rows)))

(define (store-rows store)
  (let ((db (sqlite-open store SQLITE_OPEN_READONLY)))
    (let ((reasoning (query db "SELECT r.id, r.session_id, r.step, r.kind, r.body,
                                      COALESCE(r.commit_hash, ''), r.created_at,
                                      COALESCE(r.gist, '')
                               FROM reasoning r
                               ORDER BY r.id"))
          (commits (query db "SELECT hash, subject, COALESCE(body, ''), author_ts
                              FROM commits ORDER BY author_ts"))
          ;; Which sessions' reasoning is attached to which commit, so a commit
          ;; document can carry those sessions' topics.
          (attached (query db "SELECT commit_hash, GROUP_CONCAT(DISTINCT session_id)
                               FROM reasoning
                               WHERE commit_hash IS NOT NULL AND session_id <> ''
                               GROUP BY commit_hash")))
      (sqlite-close db)
      (values reasoning commits attached))))

;;; ---------------------------------------------------------------------
;;; Commands

(define (cmd-update opts)
  (let ((store (option-ref opts 'store #f))
        (index (option-ref opts 'index #f))
        (rebuild (option-ref opts 'rebuild #f)))
    (unless store (die "update needs --store STORE.sqlite"))
    (unless index (die "update needs --index DIR"))
    (unless (file-exists? store) (die "no store at ~a" store))
    (call-with-values (lambda () (store-rows store))
      (lambda (reasoning commits attached)
        (let* ((repo (repo-name store))
               (records (records repo reasoning commits attached))
               (db (open-writable index rebuild))
               (stemmer (new-Stem "english"))
               (written 0))
          (for-each (lambda (entry)
                      (index-record! db stemmer (first entry) (second entry))
                      (set! written (+ written 1)))
                    records)
          (WritableDatabase-commit db)
          (delete-WritableDatabase db)
          (note "~a: ~a documents (~a reasoning, ~a commits)"
                (if rebuild "rebuilt" "updated") written
                (length reasoning) (length commits)))))))

(define (make-query-parser db default-op)
  (let ((qp (new-QueryParser)))
    (QueryParser-set-stemmer qp (new-Stem "english"))
    (QueryParser-set-database qp db)
    (QueryParser-set-default-op qp default-op)
    (for-each (lambda (pair)
                (QueryParser-add-boolean-prefix qp (car pair) (cdr pair)))
              PREFIXES)
    qp))

(define (cmd-search opts words)
  (let ((index (option-ref opts 'index #f))
        (limit (string->number (option-ref opts 'limit "10")))
        (query-string (string-join words " ")))
    (unless index (die "search needs --index DIR"))
    (when (string-null? (string-trim-both query-string))
      (die "search needs a query"))
    (let* ((db (open-readable index))
           (qp (make-query-parser db (if (option-ref opts 'or #f)
                                         (Query-OP-OR)
                                         (Query-OP-AND))))
           (enq (new-Enquire db)))
      (Enquire-set-query enq (QueryParser-parse-query qp query-string))
      (Enquire-set-sort-by-value enq SLOT-DATE #t)
      (let* ((mset (Enquire-get-mset enq 0 limit))
             (n (MSet-size mset))
             (hits (let loop ((i 0) (acc (list)))
                     (if (>= i n)
                         acc
                         (let* ((item (MSet-get-hit mset i))
                                (doc (MSetIterator-get-document item))
                                (record (Document-get-value doc SLOT-DOCDATA)))
                           (loop (+ i 1)
                                 (cons (string-append
                                        "{\"docid\":" (number->string (MSetIterator-get-docid item))
                                        ",\"percent\":" (number->string (MSetIterator-get-percent item))
                                        ",\"record\":" record "}")
                                       acc)))))))
        (display
         (string-append
          "{\"query\":" (json-value query-string)
          ",\"count\":" (number->string (length hits))
          ",\"matches\":" (number->string (MSet-get-matches-estimated mset))
          ",\"hits\":["
          (string-join (reverse hits) ",")
          "]}"))
        (newline))
      (delete-Database db))))

(define (cmd-count opts)
  (let ((db (open-readable (option-ref opts 'index #f))))
    (format #t "~a\n" (Database-get-doccount db))
    (delete-Database db)))

(define (usage)
  (format #t "usage: index.scm update --store STORE.sqlite --index DIR [--rebuild]~@
              \x20      index.scm search --index DIR [--limit N] [--or] QUERY...~@
              \x20      index.scm count  --index DIR~%"))

(define OPTION-SPEC
  '((store (value #t))
    (index (value #t))
    (limit (value #t))
    (rebuild)
    (or)
    (quiet)
    (help (single-char #\h))))

(define (main args)
  (let* ((opts (getopt-long args OPTION-SPEC))
         (rest (option-ref opts '() '())))
    (parameterize ((CURRENT-OPTIONS opts))
      (match rest
        (("update" . _) (cmd-update opts))
        (("search" . words) (cmd-search opts words))
        (("count" . _) (cmd-count opts))
        (("schema" . _) (format #t "~a\n" SCHEMA-VERSION))
        (("--help" . _) (usage))
        (((? (lambda (a) (member a '("-h" "--help"))) _) . _) (usage))
        ((cmd . _) (die "unknown command: ~a" cmd))
        (() (usage))))))

(main (command-line))