1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
|
package main
import "core:c"
import "core:fmt"
import "core:strings"
import sqlite3 "sqlite3:sqlite3"
// The database is a file next to the sources. It must not be called
// `ai-memory': that is the name of the binary `odin build -out:ai-memory'
// produces.
DEFAULT_DB_PATH :: "ai-memory.sqlite"
SCHEMA :: #load("schema.sql", string)
// schema_version is the version SCHEMA declares, so the number exists in one
// place only: the binary and the schema it creates cannot disagree.
schema_version :: proc() -> int {
// A copy up front: SCHEMA is a compile-time constant and Odin only allows
// constant indices into one.
schema: string = SCHEMA
marker :: "pragma user_version = "
index := strings.index(schema, marker)
if index < 0 {
return 0
}
// SCHEMA is a compile-time constant, so it is walked by index rather than
// sliced: Odin will not slice a constant with a runtime offset.
version := 0
at := index + len(marker)
for at < len(schema) {
c := schema[at]
if c < '0' || c > '9' {
break
}
version = version * 10 + int(c - '0')
at += 1
}
return version
}
Db :: struct {
handle: ^sqlite3.Connection,
}
Stmt :: struct {
db: ^Db,
handle: ^sqlite3.Statement,
}
db_open :: proc(path: string) -> (db: Db, ok: bool) {
cpath := strings.clone_to_cstring(path, context.temp_allocator)
if rc := sqlite3.open(cpath, &db.handle); rc != .Ok {
fmt.eprintfln("ai-memory: cannot open %q: %v", path, rc)
return {}, false
}
// Foreign keys are off by default in SQLite; WAL keeps readers (the
// assistant) out of the writer's way.
if !db_exec(&db, "pragma journal_mode = wal") ||
!db_exec(&db, "pragma foreign_keys = on") ||
!db_exec(&db, "pragma busy_timeout = 5000") {
db_close(&db)
return {}, false
}
return db, true
}
db_close :: proc(db: ^Db) {
if db.handle != nil {
sqlite3.close(db.handle)
db.handle = nil
}
}
db_exec :: proc(db: ^Db, sql: string) -> bool {
csql := strings.clone_to_cstring(sql, context.temp_allocator)
err: cstring
if rc := sqlite3.exec(db.handle, csql, nil, nil, &err); rc != .Ok {
db_report(db, rc, err)
return false
}
return true
}
// db_report prints a SQLite failure. The extended code says *which* constraint
// failed, which the primary code hides.
db_report :: proc(db: ^Db, rc: sqlite3.Result_Code, err: cstring) {
msg := ""
if err != nil {
msg = strings.clone(string(err), context.temp_allocator)
sqlite3.free(cast(rawptr)err)
} else if db.handle != nil {
msg = string(sqlite3.errmsg(db.handle))
}
extended := rc
if db.handle != nil {
extended = sqlite3.extended_errcode(db.handle)
}
if extended != rc {
fmt.eprintfln("ai-memory: sql error (%v/%v): %s", rc, extended, msg)
} else {
fmt.eprintfln("ai-memory: sql error (%v): %s", rc, msg)
}
if (extended == .Error && len(msg) == 0) ||
strings.contains(msg, "no such column") ||
strings.contains(msg, "no such table") {
// SQLITE_ERROR with no message is what a *statement* failure looks like
// when the schema is older than this binary: a column the code writes
// does not exist yet. The store is fine; the schema needs upgrading.
fmt.eprintln("ai-memory: the store's schema is older than this binary -- " +
"run `ai-memory init' to upgrade it")
}
#partial switch extended {
case .Constraint_ForeignKey:
fmt.eprintln("ai-memory: the row this refers to does not exist yet " +
"(for commits: run `sync')")
case .Constraint_Unique:
fmt.eprintln("ai-memory: that entry already exists")
case:
}
}
stmt :: proc(db: ^Db, sql: string) -> (s: Stmt, ok: bool) {
s.db = db
csql := strings.clone_to_cstring(sql, context.temp_allocator)
if rc := sqlite3.prepare(db.handle, csql, c.int(len(sql)), &s.handle, nil); rc != .Ok {
db_report(db, rc, nil)
return {}, false
}
return s, true
}
stmt_close :: proc(s: ^Stmt) {
if s.handle != nil {
sqlite3.finalize(s.handle)
s.handle = nil
}
}
// bind_text copies VALUE into SQLite's own buffer, so the caller's string may
// be temporary.
stmt_bind_text :: proc(s: ^Stmt, idx: int, value: string) -> bool {
cvalue := strings.clone_to_cstring(value, context.temp_allocator)
rc := sqlite3.bind_text(s.handle, c.int(idx), cvalue, c.int(len(value)),
sqlite3.Destructor{behaviour = .Transient})
if rc != .Ok {
db_report(s.db, rc, nil)
return false
}
return true
}
stmt_bind_i64 :: proc(s: ^Stmt, idx: int, value: i64) -> bool {
rc := sqlite3.bind_int64(s.handle, c.int(idx), c.int64_t(value))
if rc != .Ok {
db_report(s.db, rc, nil)
return false
}
return true
}
stmt_bind_null :: proc(s: ^Stmt, idx: int) -> bool {
rc := sqlite3.bind_null(s.handle, c.int(idx))
if rc != .Ok {
db_report(s.db, rc, nil)
return false
}
return true
}
// stmt_step returns the raw result code: .Row, .Done or an error.
stmt_step :: proc(s: ^Stmt) -> sqlite3.Result_Code {
rc := sqlite3.step(s.handle)
if rc != .Row && rc != .Done {
db_report(s.db, rc, nil)
}
return rc
}
stmt_col_text :: proc(s: ^Stmt, idx: int, allocator := context.temp_allocator) -> string {
c := sqlite3.column_text(s.handle, c.int(idx))
if c == nil {
return ""
}
return strings.clone(string(c), allocator)
}
stmt_col_i64 :: proc(s: ^Stmt, idx: int) -> i64 {
return i64(sqlite3.column_int64(s.handle, c.int(idx)))
}
// db_migrate creates the schema and upgrades older stores in place.
db_migrate :: proc(db: ^Db) -> bool {
// SCHEMA's `create table if not exists' never alters an existing table, and
// it creates indexes over the columns added since version 1, so the column
// additions have to happen before SCHEMA runs. A fresh store is version 0
// and simply gets everything from SCHEMA.
//
// Each step tests the schema rather than trusting user_version: a store can
// carry a newer column under an older number (another writer, a hand ALTER),
// and an unconditional ALTER then fails with "duplicate column name" -- which
// reads as corruption when it is only version skew.
version := db_user_version(db)
// A newer ai-memory may have written this store, and this binary does not
// know its schema. Refuse before touching anything: the alternative is what
// a stale binary did here -- it silently rewrote user_version from 5 to 3,
// because its migration chain ends at the version *it* knows. The data was
// fine (SQLite cannot drop a column, so gist_source survived), but the stamp
// lied, and the next newer binary re-ran data steps it thought were pending.
if version > schema_version() {
fmt.eprintfln("ai-memory: this store is schema v%d; this binary knows v%d",
version, schema_version())
fmt.eprintln("ai-memory: it was written by a newer ai-memory -- upgrade " +
"this binary (scripts/install.sh). Nothing was changed.")
return false
}
started_at := version
// Version 2: basis and supersedes on reasoning.
if version == 1 {
if !db_add_column(db, "reasoning", "basis", "text") {
return false
}
if !db_add_column(db, "reasoning", "supersedes",
"integer references reasoning(id)") {
return false
}
version = 2
}
// Version 3: symbols.basename, so an unqualified identifier is an index seek
// rather than a LIKE scan.
if version == 2 {
if db_add_column(db, "symbols", "basename", "text") {
if !db_backfill_basename(db) {
return false
}
}
version = 3
}
// The gist columns are ensured before any data step above uses them, because
// db_add_column is inspection-guarded and the backfills below depend on both
// existing whatever the store's recorded version says. A store that sat at
// version 3 while a newer binary wrote gists into it failed exactly here: the
// version 3 -> 4 backfill referenced gist_source before version 5 had added
// it, and every command then died on "no such column: gist_source", not just
// the first. Guarded by the table's existence: a fresh store is version 0
// with no tables at all, and gets both columns from SCHEMA at the end.
if db_has_table(db, "reasoning") {
if !db_add_column(db, "reasoning", "gist", "text") ||
!db_add_column(db, "reasoning", "gist_source", "text") {
return false
}
}
// Version 4: reasoning.gist, the one-line claim. Entries written before the
// column existed get a first-sentence gist.
if version == 3 {
if !db_backfill_gist(db) {
return false
}
version = 4
}
// Version 5: reasoning.gist_source, so a derived gist (the body's first
// sentence) can be recomputed while an assessed one is frozen. A gist
// written before this column existed cannot be classified, and freezing is
// the safe answer: marking it 'assessed' means no later migration may touch
// it. Rows with no gist at all are left alone for the backfill to fill.
if version == 4 {
if !db_label_existing_gists(db) {
return false
}
version = 5
}
// Version 6: the blobs table, the C/C++ declaration cache. SCHEMA creates it
// with `create table if not exists', so this step only moves the stamp -- but
// it still has to exist, or an older binary would go on writing a store that
// a newer one had already added the table to, without ever saying so.
if version == 5 {
version = 6
}
// Only ever move the stamp forward. Writing it unconditionally is how the
// downgrade happened: a binary that knows no steps past v3 still stamped 3.
if version > started_at {
if !db_exec(db, fmt.tprintf("pragma user_version = %d", version)) {
return false
}
}
return db_exec(db, SCHEMA)
}
// db_add_column adds a column unless it is already there, and reports whether
// the table now has it.
db_add_column :: proc(db: ^Db, table, column, decl: string) -> bool {
if db_has_column(db, table, column) {
return true
}
if !db_exec(db, fmt.tprintf("alter table %s add column %s %s", table, column, decl)) {
return false
}
return true
}
// db_has_table reports whether the store has TABLE. A fresh store has none:
// `init' creates them from SCHEMA after the migration steps have run.
db_has_table :: proc(db: ^Db, table: string) -> bool {
s, ok := stmt(db, "select 1 from sqlite_master where type = 'table' and name = ?1")
if !ok {
return false
}
stmt_bind_text(&s, 1, table)
found := stmt_step(&s) == .Row
stmt_close(&s)
return found
}
// db_has_column reports whether TABLE already has COLUMN.
db_has_column :: proc(db: ^Db, table, column: string) -> bool {
s, ok := stmt(db, fmt.tprintf("pragma table_info(%s)", table))
if !ok {
return false
}
defer stmt_close(&s)
for stmt_step(&s) == .Row {
if stmt_col_text(&s, 1) == column {
return true
}
}
return false
}
// db_backfill_basename fills symbols.basename for stores created before
// version 3. The value is the identifier after the last dot of the qualified
// name; SQLite has no right-to-left string functions, so this walks the rows.
db_backfill_basename :: proc(db: ^Db) -> bool {
Rows :: struct {
id: i64,
name: string,
}
rows := make([dynamic]Rows, 0, 1024)
defer delete(rows)
s, ok := stmt(db, "select id, name from symbols")
if !ok {
return false
}
for stmt_step(&s) == .Row {
append(&rows, Rows{
id = stmt_col_i64(&s, 0),
name = strings.clone(stmt_col_text(&s, 1), context.temp_allocator),
})
}
stmt_close(&s)
if !db_exec(db, "begin") {
return false
}
// One statement per row: after SQLITE_DONE a statement must be reset before
// it can be re-bound, and this binding set does not expose sqlite3_reset.
// Preparing again is cheap enough for a one-off migration.
for row in rows {
base := row.name
if dot := strings.last_index_byte(base, '.'); dot >= 0 {
base = base[dot + 1:]
}
u, uok := stmt(db, "update symbols set basename = ?1 where id = ?2")
if !uok {
db_exec(db, "rollback")
return false
}
stmt_bind_text(&u, 1, base)
stmt_bind_i64(&u, 2, row.id)
done := stmt_step(&u) == .Done
stmt_close(&u)
if !done {
db_exec(db, "rollback")
return false
}
}
return db_exec(db, "commit")
}
// db_user_version reads the store's schema version (0 for an empty database).
db_user_version :: proc(db: ^Db) -> int {
s, ok := stmt(db, "pragma user_version")
if !ok {
return 0
}
defer stmt_close(&s)
if stmt_step(&s) != .Row {
return 0
}
return int(stmt_col_i64(&s, 0))
}
// sqlite_changes reports rows modified by the most recent statement.
sqlite_changes :: proc(db: ^Db) -> i64 {
return i64(sqlite3.changes(db.handle))
}
// db_label_existing_gists classifies gists written before gist_source existed.
// Nothing in the store says whether such a gist was assessed or derived, and the
// two errors are not equal: calling an assessed gist 'derived' lets a later
// re-derivation overwrite a judgement, while calling a derived one 'assessed'
// only freezes a first sentence. So everything that is there becomes assessed,
// and only rows with no gist at all are left for the backfill.
db_label_existing_gists :: proc(db: ^Db) -> bool {
if !db_exec(db, "update reasoning set gist_source = 'assessed' " +
"where gist is not null and gist_source is null") {
return false
}
return db_exec(db, "update reasoning set gist_source = 'derived' " +
"where gist is null and gist_source is null")
}
// db_backfill_gist gives entries written before version 4 a first-sentence
// gist. It is a floor, not a substitute: a recorded --gist says what the entry
// claims, where the first sentence often only says what the entry is about.
db_backfill_gist :: proc(db: ^Db) -> bool {
Rows :: struct {
id: i64,
body: string,
}
rows := make([dynamic]Rows, 0, 256)
defer delete(rows)
s, ok := stmt(db, "select id, body from reasoning")
if !ok {
return false
}
for stmt_step(&s) == .Row {
append(&rows, Rows{
id = stmt_col_i64(&s, 0),
body = strings.clone(stmt_col_text(&s, 1), context.temp_allocator),
})
}
stmt_close(&s)
if !db_exec(db, "begin") {
return false
}
// One statement per row: the binding has no sqlite3_reset (see the note in
// db_backfill_basename).
//
// `where gist is null or gist_source = 'derived'` is the whole point of
// gist_source: this used to rewrite every row, so an assessed --gist was one
// migration away from being replaced by the body's first sentence.
for row in rows {
u, uok := stmt(db, "update reasoning set gist = ?1, " +
"gist_source = coalesce(gist_source, 'derived') " +
"where id = ?2 and (gist is null or gist_source = 'derived')")
if !uok {
db_exec(db, "rollback")
return false
}
stmt_bind_text(&u, 1, gist_of(row.body))
stmt_bind_i64(&u, 2, row.id)
done := stmt_step(&u) == .Done
stmt_close(&u)
if !done {
db_exec(db, "rollback")
return false
}
}
return db_exec(db, "commit")
}
// gist_of takes the first sentence of BODY, capped, as a fallback gist.
gist_of :: proc(body: string) -> string {
flat := body
end := len(flat)
for i in 0 ..< len(flat) {
if flat[i] != '.' && flat[i] != '?' && flat[i] != '!' && flat[i] != ';' {
continue
}
if i + 1 < len(flat) && flat[i + 1] != ' ' && flat[i + 1] != '\n' && flat[i + 1] != '\t' {
continue
}
// Skip "e.g." and "i.e.", where the dot is not the sentence's end.
if flat[i] == '.' && i >= 3 {
if (flat[i - 1] == 'g' || flat[i - 1] == 'e') && flat[i - 2] == '.' {
continue
}
}
end = i
break
}
out := flat[:end]
if len(out) > 120 {
out = out[:117]
return fmt.tprintf("%s...", out)
}
return out
}
|