package main import "core:c" import "core:fmt" import "core:strings" import sqlite3 "sqlite3:sqlite3" // The database is a file next to the sources. It must not be called // `ai-memory': that is the name of the binary `odin build -out:ai-memory' // produces. DEFAULT_DB_PATH :: "ai-memory.sqlite" SCHEMA :: #load("schema.sql", string) // schema_version is the version SCHEMA declares, so the number exists in one // place only: the binary and the schema it creates cannot disagree. schema_version :: proc() -> int { // A copy up front: SCHEMA is a compile-time constant and Odin only allows // constant indices into one. schema: string = SCHEMA marker :: "pragma user_version = " index := strings.index(schema, marker) if index < 0 { return 0 } // SCHEMA is a compile-time constant, so it is walked by index rather than // sliced: Odin will not slice a constant with a runtime offset. version := 0 at := index + len(marker) for at < len(schema) { c := schema[at] if c < '0' || c > '9' { break } version = version * 10 + int(c - '0') at += 1 } return version } Db :: struct { handle: ^sqlite3.Connection, } Stmt :: struct { db: ^Db, handle: ^sqlite3.Statement, } db_open :: proc(path: string) -> (db: Db, ok: bool) { cpath := strings.clone_to_cstring(path, context.temp_allocator) if rc := sqlite3.open(cpath, &db.handle); rc != .Ok { fmt.eprintfln("ai-memory: cannot open %q: %v", path, rc) return {}, false } // Foreign keys are off by default in SQLite; WAL keeps readers (the // assistant) out of the writer's way. if !db_exec(&db, "pragma journal_mode = wal") || !db_exec(&db, "pragma foreign_keys = on") || !db_exec(&db, "pragma busy_timeout = 5000") { db_close(&db) return {}, false } return db, true } db_close :: proc(db: ^Db) { if db.handle != nil { sqlite3.close(db.handle) db.handle = nil } } db_exec :: proc(db: ^Db, sql: string) -> bool { csql := strings.clone_to_cstring(sql, context.temp_allocator) err: cstring if rc := sqlite3.exec(db.handle, csql, nil, nil, &err); rc != .Ok { db_report(db, rc, err) return false } return true } // db_report prints a SQLite failure. The extended code says *which* constraint // failed, which the primary code hides. db_report :: proc(db: ^Db, rc: sqlite3.Result_Code, err: cstring) { msg := "" if err != nil { msg = strings.clone(string(err), context.temp_allocator) sqlite3.free(cast(rawptr)err) } else if db.handle != nil { msg = string(sqlite3.errmsg(db.handle)) } extended := rc if db.handle != nil { extended = sqlite3.extended_errcode(db.handle) } if extended != rc { fmt.eprintfln("ai-memory: sql error (%v/%v): %s", rc, extended, msg) } else { fmt.eprintfln("ai-memory: sql error (%v): %s", rc, msg) } if (extended == .Error && len(msg) == 0) || strings.contains(msg, "no such column") || strings.contains(msg, "no such table") { // SQLITE_ERROR with no message is what a *statement* failure looks like // when the schema is older than this binary: a column the code writes // does not exist yet. The store is fine; the schema needs upgrading. fmt.eprintln("ai-memory: the store's schema is older than this binary -- " + "run `ai-memory init' to upgrade it") } #partial switch extended { case .Constraint_ForeignKey: fmt.eprintln("ai-memory: the row this refers to does not exist yet " + "(for commits: run `sync')") case .Constraint_Unique: fmt.eprintln("ai-memory: that entry already exists") case: } } stmt :: proc(db: ^Db, sql: string) -> (s: Stmt, ok: bool) { s.db = db csql := strings.clone_to_cstring(sql, context.temp_allocator) if rc := sqlite3.prepare(db.handle, csql, c.int(len(sql)), &s.handle, nil); rc != .Ok { db_report(db, rc, nil) return {}, false } return s, true } stmt_close :: proc(s: ^Stmt) { if s.handle != nil { sqlite3.finalize(s.handle) s.handle = nil } } // bind_text copies VALUE into SQLite's own buffer, so the caller's string may // be temporary. stmt_bind_text :: proc(s: ^Stmt, idx: int, value: string) -> bool { cvalue := strings.clone_to_cstring(value, context.temp_allocator) rc := sqlite3.bind_text(s.handle, c.int(idx), cvalue, c.int(len(value)), sqlite3.Destructor{behaviour = .Transient}) if rc != .Ok { db_report(s.db, rc, nil) return false } return true } stmt_bind_i64 :: proc(s: ^Stmt, idx: int, value: i64) -> bool { rc := sqlite3.bind_int64(s.handle, c.int(idx), c.int64_t(value)) if rc != .Ok { db_report(s.db, rc, nil) return false } return true } stmt_bind_null :: proc(s: ^Stmt, idx: int) -> bool { rc := sqlite3.bind_null(s.handle, c.int(idx)) if rc != .Ok { db_report(s.db, rc, nil) return false } return true } // stmt_step returns the raw result code: .Row, .Done or an error. stmt_step :: proc(s: ^Stmt) -> sqlite3.Result_Code { rc := sqlite3.step(s.handle) if rc != .Row && rc != .Done { db_report(s.db, rc, nil) } return rc } stmt_col_text :: proc(s: ^Stmt, idx: int, allocator := context.temp_allocator) -> string { c := sqlite3.column_text(s.handle, c.int(idx)) if c == nil { return "" } return strings.clone(string(c), allocator) } stmt_col_i64 :: proc(s: ^Stmt, idx: int) -> i64 { return i64(sqlite3.column_int64(s.handle, c.int(idx))) } // db_migrate creates the schema and upgrades older stores in place. db_migrate :: proc(db: ^Db) -> bool { // SCHEMA's `create table if not exists' never alters an existing table, and // it creates indexes over the columns added since version 1, so the column // additions have to happen before SCHEMA runs. A fresh store is version 0 // and simply gets everything from SCHEMA. // // Each step tests the schema rather than trusting user_version: a store can // carry a newer column under an older number (another writer, a hand ALTER), // and an unconditional ALTER then fails with "duplicate column name" -- which // reads as corruption when it is only version skew. version := db_user_version(db) // A newer ai-memory may have written this store, and this binary does not // know its schema. Refuse before touching anything: the alternative is what // a stale binary did here -- it silently rewrote user_version from 5 to 3, // because its migration chain ends at the version *it* knows. The data was // fine (SQLite cannot drop a column, so gist_source survived), but the stamp // lied, and the next newer binary re-ran data steps it thought were pending. if version > schema_version() { fmt.eprintfln("ai-memory: this store is schema v%d; this binary knows v%d", version, schema_version()) fmt.eprintln("ai-memory: it was written by a newer ai-memory -- upgrade " + "this binary (scripts/install.sh). Nothing was changed.") return false } started_at := version // Version 2: basis and supersedes on reasoning. if version == 1 { if !db_add_column(db, "reasoning", "basis", "text") { return false } if !db_add_column(db, "reasoning", "supersedes", "integer references reasoning(id)") { return false } version = 2 } // Version 3: symbols.basename, so an unqualified identifier is an index seek // rather than a LIKE scan. if version == 2 { if db_add_column(db, "symbols", "basename", "text") { if !db_backfill_basename(db) { return false } } version = 3 } // The gist columns are ensured before any data step above uses them, because // db_add_column is inspection-guarded and the backfills below depend on both // existing whatever the store's recorded version says. A store that sat at // version 3 while a newer binary wrote gists into it failed exactly here: the // version 3 -> 4 backfill referenced gist_source before version 5 had added // it, and every command then died on "no such column: gist_source", not just // the first. Guarded by the table's existence: a fresh store is version 0 // with no tables at all, and gets both columns from SCHEMA at the end. if db_has_table(db, "reasoning") { if !db_add_column(db, "reasoning", "gist", "text") || !db_add_column(db, "reasoning", "gist_source", "text") { return false } } // Version 4: reasoning.gist, the one-line claim. Entries written before the // column existed get a first-sentence gist. if version == 3 { if !db_backfill_gist(db) { return false } version = 4 } // Version 5: reasoning.gist_source, so a derived gist (the body's first // sentence) can be recomputed while an assessed one is frozen. A gist // written before this column existed cannot be classified, and freezing is // the safe answer: marking it 'assessed' means no later migration may touch // it. Rows with no gist at all are left alone for the backfill to fill. if version == 4 { if !db_label_existing_gists(db) { return false } version = 5 } // Version 6: the blobs table, the C/C++ declaration cache. SCHEMA creates it // with `create table if not exists', so this step only moves the stamp -- but // it still has to exist, or an older binary would go on writing a store that // a newer one had already added the table to, without ever saying so. if version == 5 { version = 6 } // Only ever move the stamp forward. Writing it unconditionally is how the // downgrade happened: a binary that knows no steps past v3 still stamped 3. if version > started_at { if !db_exec(db, fmt.tprintf("pragma user_version = %d", version)) { return false } } return db_exec(db, SCHEMA) } // db_add_column adds a column unless it is already there, and reports whether // the table now has it. db_add_column :: proc(db: ^Db, table, column, decl: string) -> bool { if db_has_column(db, table, column) { return true } if !db_exec(db, fmt.tprintf("alter table %s add column %s %s", table, column, decl)) { return false } return true } // db_has_table reports whether the store has TABLE. A fresh store has none: // `init' creates them from SCHEMA after the migration steps have run. db_has_table :: proc(db: ^Db, table: string) -> bool { s, ok := stmt(db, "select 1 from sqlite_master where type = 'table' and name = ?1") if !ok { return false } stmt_bind_text(&s, 1, table) found := stmt_step(&s) == .Row stmt_close(&s) return found } // db_has_column reports whether TABLE already has COLUMN. db_has_column :: proc(db: ^Db, table, column: string) -> bool { s, ok := stmt(db, fmt.tprintf("pragma table_info(%s)", table)) if !ok { return false } defer stmt_close(&s) for stmt_step(&s) == .Row { if stmt_col_text(&s, 1) == column { return true } } return false } // db_backfill_basename fills symbols.basename for stores created before // version 3. The value is the identifier after the last dot of the qualified // name; SQLite has no right-to-left string functions, so this walks the rows. db_backfill_basename :: proc(db: ^Db) -> bool { Rows :: struct { id: i64, name: string, } rows := make([dynamic]Rows, 0, 1024) defer delete(rows) s, ok := stmt(db, "select id, name from symbols") if !ok { return false } for stmt_step(&s) == .Row { append(&rows, Rows{ id = stmt_col_i64(&s, 0), name = strings.clone(stmt_col_text(&s, 1), context.temp_allocator), }) } stmt_close(&s) if !db_exec(db, "begin") { return false } // One statement per row: after SQLITE_DONE a statement must be reset before // it can be re-bound, and this binding set does not expose sqlite3_reset. // Preparing again is cheap enough for a one-off migration. for row in rows { base := row.name if dot := strings.last_index_byte(base, '.'); dot >= 0 { base = base[dot + 1:] } u, uok := stmt(db, "update symbols set basename = ?1 where id = ?2") if !uok { db_exec(db, "rollback") return false } stmt_bind_text(&u, 1, base) stmt_bind_i64(&u, 2, row.id) done := stmt_step(&u) == .Done stmt_close(&u) if !done { db_exec(db, "rollback") return false } } return db_exec(db, "commit") } // db_user_version reads the store's schema version (0 for an empty database). db_user_version :: proc(db: ^Db) -> int { s, ok := stmt(db, "pragma user_version") if !ok { return 0 } defer stmt_close(&s) if stmt_step(&s) != .Row { return 0 } return int(stmt_col_i64(&s, 0)) } // sqlite_changes reports rows modified by the most recent statement. sqlite_changes :: proc(db: ^Db) -> i64 { return i64(sqlite3.changes(db.handle)) } // db_label_existing_gists classifies gists written before gist_source existed. // Nothing in the store says whether such a gist was assessed or derived, and the // two errors are not equal: calling an assessed gist 'derived' lets a later // re-derivation overwrite a judgement, while calling a derived one 'assessed' // only freezes a first sentence. So everything that is there becomes assessed, // and only rows with no gist at all are left for the backfill. db_label_existing_gists :: proc(db: ^Db) -> bool { if !db_exec(db, "update reasoning set gist_source = 'assessed' " + "where gist is not null and gist_source is null") { return false } return db_exec(db, "update reasoning set gist_source = 'derived' " + "where gist is null and gist_source is null") } // db_backfill_gist gives entries written before version 4 a first-sentence // gist. It is a floor, not a substitute: a recorded --gist says what the entry // claims, where the first sentence often only says what the entry is about. db_backfill_gist :: proc(db: ^Db) -> bool { Rows :: struct { id: i64, body: string, } rows := make([dynamic]Rows, 0, 256) defer delete(rows) s, ok := stmt(db, "select id, body from reasoning") if !ok { return false } for stmt_step(&s) == .Row { append(&rows, Rows{ id = stmt_col_i64(&s, 0), body = strings.clone(stmt_col_text(&s, 1), context.temp_allocator), }) } stmt_close(&s) if !db_exec(db, "begin") { return false } // One statement per row: the binding has no sqlite3_reset (see the note in // db_backfill_basename). // // `where gist is null or gist_source = 'derived'` is the whole point of // gist_source: this used to rewrite every row, so an assessed --gist was one // migration away from being replaced by the body's first sentence. for row in rows { u, uok := stmt(db, "update reasoning set gist = ?1, " + "gist_source = coalesce(gist_source, 'derived') " + "where id = ?2 and (gist is null or gist_source = 'derived')") if !uok { db_exec(db, "rollback") return false } stmt_bind_text(&u, 1, gist_of(row.body)) stmt_bind_i64(&u, 2, row.id) done := stmt_step(&u) == .Done stmt_close(&u) if !done { db_exec(db, "rollback") return false } } return db_exec(db, "commit") } // gist_of takes the first sentence of BODY, capped, as a fallback gist. gist_of :: proc(body: string) -> string { flat := body end := len(flat) for i in 0 ..< len(flat) { if flat[i] != '.' && flat[i] != '?' && flat[i] != '!' && flat[i] != ';' { continue } if i + 1 < len(flat) && flat[i + 1] != ' ' && flat[i + 1] != '\n' && flat[i + 1] != '\t' { continue } // Skip "e.g." and "i.e.", where the dot is not the sentence's end. if flat[i] == '.' && i >= 3 { if (flat[i - 1] == 'g' || flat[i - 1] == 'e') && flat[i - 2] == '.' { continue } } end = i break } out := flat[:end] if len(out) > 120 { out = out[:117] return fmt.tprintf("%s...", out) } return out }