package main import "core:fmt" import "core:os" import "core:strconv" import "core:strings" // Field/record separators for git log parsing; these control characters do not // occur in commit messages. REC_SEP :: 0x1e FLD_SEP :: 0x1f // REPO_DIR is the repository git commands run in; empty means the current // directory. Set from --repo PATH. REPO_DIR: string Git_Commit :: struct { hash: string, author_ts: i64, subject: string, body: string, } // A symbol a commit touched. `added'/`removed' come from declarations that // appear or disappear in the diff; `in_context' marks the enclosing item named in // a hunk header, i.e. "the change happened inside this function". Decl :: struct { name: string, // unqualified identifier kind: string, // proc | fn | type | const | var file: string, added: bool, removed: bool, in_context: bool, } // git runs `git ARGS' in REPO_DIR and captures its stdout. git :: proc(args: []string, allocator := context.temp_allocator, report := true) -> (out: string, ok: bool) { cmd := make([dynamic]string, 0, len(args) + 1, context.temp_allocator) append(&cmd, "git") append(&cmd, ..args) desc := os.Process_Desc{command = cmd[:]} if len(REPO_DIR) > 0 { desc.working_dir = REPO_DIR } state, stdout, stderr, err := os.process_exec(desc, context.temp_allocator) if err != nil { fmt.eprintfln("ai-memory: cannot run git: %v", err) return "", false } if state.exit_code != 0 { if report { msg := strings.trim_space(string(stderr)) fmt.eprintfln("ai-memory: git %s failed: %s", args[0], msg) } return "", false } return strings.clone(string(stdout), allocator), true } // git_log reads the history, newest first. LIMIT of 0 means everything. git_log :: proc(limit: int) -> ([]Git_Commit, bool) { // A fresh repository has no commits yet; that is not an error here, so the // probe stays quiet. if _, has_head := git([]string{"rev-parse", "--verify", "--quiet", "HEAD"}, context.temp_allocator, false); !has_head { return nil, true } args := make([dynamic]string, 0, 8, context.temp_allocator) append(&args, "log", "--format=%H%x1f%at%x1f%s%x1f%b%x1e") if limit > 0 { append(&args, fmt.tprintf("-n%d", limit)) } raw, ok := git(args[:]) if !ok { return nil, false } commits := make([dynamic]Git_Commit, 0, 256) for raw_part in split_byte(raw, REC_SEP) { part := strings.trim_space(raw_part) if len(part) == 0 { continue } fields := split_byte(part, FLD_SEP) if len(fields) < 3 { continue } c: Git_Commit c.hash = fields[0] c.author_ts = parse_i64(fields[1]) c.subject = fields[2] if len(fields) > 3 { c.body = join_fields(fields[3:]) } append(&commits, c) } return commits[:], true } // git_touched reports the symbols commit HASH touched, from three sources: // declarations that appear or disappear on +/- lines (Odin, Rust), declarations // diffed between file blobs by ctags (C, C++), and the enclosing item named in // each hunk header (git's userdiff patterns, so no parsing needed). DB is the // store whose blob cache the ctags pass reads and fills; nil disables it. git_touched :: proc(hash: string, db: ^Db = nil) -> ([]Decl, bool) { raw, ok := git([]string{"show", "--unified=0", "--no-color", "--no-renames", "--format=", "--no-ext-diff", hash}) if !ok { return nil, false } found := make(map[string]Decl) defer delete(found) file := "" for line in split_byte(raw, '\n') { switch { case strings.has_prefix(line, "+++ b/"): file = line[len("+++ b/"):] if i := strings.index_byte(file, '\t'); i >= 0 { file = file[:i] } case strings.has_prefix(line, "@@ "): // "@@ -412,6 +412,7 @@ fn hash_pve(state: &PveState) -> Result<()>" if ctx, has_ctx := hunk_context(line); has_ctx { if name, kind, is_decl := symbol_in_context(file, ctx); is_decl { merge_decl(&found, Decl{name = name, kind = kind, file = file, in_context = true}) } } case is_change_line(line): name, kind, is_decl := parse_decl(file, line[1:]) if is_decl { merge_decl(&found, Decl{ name = name, kind = kind, file = file, added = line[0] == '+', removed = line[0] == '-', }) } } } if db != nil { // C-family declarations are not readable off a line, so they come from // the file blobs -- see c_family_decls. cdecls, cok := c_family_decls(db, hash) if !cok { return nil, false } for d in cdecls { merge_decl(&found, d) } } decls := make([dynamic]Decl, 0, len(found)) for _, d in found { append(&decls, d) } return decls[:], true } // ---- C and C++ declarations, from universal-ctags --------------------------- // // A C-family declaration cannot be read off a diff line: inside a function body // `foo(bar);' calls something, inside a class body the same bytes declare a // constructor, and both are indented the same way. So the file itself is handed // to ctags -- a real parser -- and the *sets* of declarations in the parent blob // and the new blob are compared; what is only in the new one was added, what is // only in the old one was removed. A name whose tag survives both is left to // hunk context, which is the honest answer for "the change was inside this". // // The cost is bounded by caching: a blob's tags are stored under its git object // id, and a blob never changes, so each file version is parsed once per store // however many commits mention it. Re-syncing an imported history runs no ctags // at all. // CTAGS_DEFAULT is the ctags this binary looks for when the environment says // nothing. guix.scm compiles the store path in (-define:CTAGS_PATH=...), so an // installed ai-memory does not depend on PATH; a plain `odin build' leaves // `ctags', which does. CTAGS_DEFAULT :: #config(CTAGS_PATH, "ctags") // ctags_path returns the ctags to run. $AI_MEMORY_CTAGS wins over the compiled // default, so a user can point at their own build -- which is why the default is // compiled in rather than exported by a wrapper script, where it would overwrite // the user's value instead of standing behind it. ctags_path :: proc() -> string { if env := os.get_env("AI_MEMORY_CTAGS", context.temp_allocator); len(env) > 0 { return env } return CTAGS_DEFAULT } // CTAGS_MISSING keeps the warning to one line: ctags is optional, and a missing // binary must not turn every sync into a wall of text. CTAGS_MISSING: bool // Blob_Change is one file in `git show --raw': the path, and the object ids // before and after. An all-zero id means the file did not exist on that side. Blob_Change :: struct { path: string, old: string, new: string, } // raw_changes lists the file-level changes of HASH. raw_changes :: proc(hash: string) -> ([]Blob_Change, bool) { raw, ok := git([]string{"show", "--raw", "--no-renames", "--no-abbrev", "--no-color", "--format=", hash}) if !ok { return nil, false } changes := make([dynamic]Blob_Change, 0, 8) for line in split_byte(raw, '\n') { // ":100644 100644 M\tpath" if len(line) == 0 || line[0] != ':' { continue } fields := split_byte(line, ' ') if len(fields) < 5 { continue } tab := strings.index_byte(fields[4], '\t') if tab < 0 { continue } append(&changes, Blob_Change{ path = fields[4][tab + 1:], old = fields[2], new = fields[3], }) } return changes[:], true } // sha_absent reports whether an object id names nothing, i.e. the file was added // (old side) or deleted (new side). sha_absent :: proc(sha: string) -> bool { if len(sha) == 0 { return true } for c in sha { if c != '0' { return false } } return true } // Blob_Source is one blob to parse, with the suffix EXT so ctags can tell a C // file from a C++ one. Blob_Source :: struct { blob: string, ext: string, } // c_family_decls reports the C-family declarations HASH added and removed. // // A commit that touches no C-family file costs one `git show --raw' and returns // nothing. ctags itself is only run for blobs the cache does not have, and if // ctags cannot be run at all the store keeps working the way it did before this // pass existed: hunk context, no add/remove for C and C++. c_family_decls :: proc(db: ^Db, hash: string) -> (decls: []Decl, ok: bool) { changes, ok_changes := raw_changes(hash) if !ok_changes { return nil, false } // Which blobs this commit needs, and the suffix to parse each under. One // blob can appear twice (a file changed in two commits); the map keeps one. needed := make(map[string]string) defer delete(needed) n := 0 for change in changes { if lang_of(change.path) != .C_Family { continue } n += 1 ext := c_family_ext(change.path) if !sha_absent(change.old) { needed[change.old] = ext } if !sha_absent(change.new) { needed[change.new] = ext } } if n == 0 { return nil, true } if len(needed) == 0 { return nil, true } // The store's cache first; whatever is left is parsed in one ctags run. tags := make(map[string]string) defer delete(tags) sources := make([dynamic]Blob_Source, 0, len(needed)) for blob, ext in needed { if cached, has := cached_blob_tags(db, blob); has { tags[blob] = cached } else { append(&sources, Blob_Source{blob = blob, ext = ext}) } } if len(sources) > 0 { fresh, ok_parse := parse_blob_tags(sources[:]) if !ok_parse { // Caches stay valid; the commit simply gets no declaration-level // attribution for its C-family files. Not an error: imports must // not stop because a helper is missing. return nil, true } for blob, t in fresh { tags[blob] = t store_blob_tags(db, blob, t) } } out := make([dynamic]Decl, 0, 8) for change in changes { if lang_of(change.path) != .C_Family { continue } old_tags := tags[change.old] or_else "" new_tags := tags[change.new] or_else "" old_set := make(map[string]bool) defer delete(old_set) for line in split_byte(old_tags, '\n') { if len(line) > 0 { old_set[line] = true } } new_set := make(map[string]bool) defer delete(new_set) for line in split_byte(new_tags, '\n') { if len(line) > 0 { new_set[line] = true } } for line in split_byte(new_tags, '\n') { if len(line) == 0 || old_set[line] { continue } if d, is_decl := decl_of_tag(line, change.path); is_decl { d.added = true append(&out, d) } } for line in split_byte(old_tags, '\n') { if len(line) == 0 || new_set[line] { continue } if d, is_decl := decl_of_tag(line, change.path); is_decl { d.removed = true append(&out, d) } } } return out[:], true } // decl_of_tag turns a cached tag line, "", into a Decl. decl_of_tag :: proc(tag, file: string) -> (d: Decl, ok: bool) { tab := strings.index_byte(tag, '\t') if tab < 0 { return {}, false } tag_kind := tag[:tab] kind, is_decl := c_family_kind(tag_kind) if !is_decl { return {}, false } name := tag[tab + 1:] if len(name) == 0 { return {}, false } // `#define CALC_H' and `#define __CALC_H__' guard a header; they are // bookkeeping, not something a reader asks why about. if tag_kind == "macro" && is_include_guard(name) { return {}, false } return Decl{name = name, kind = kind, file = file}, true } // is_include_guard reports whether NAME looks like a header guard. is_include_guard :: proc(name: string) -> bool { if strings.has_prefix(name, "__") { return true } return strings.has_suffix(name, "_H") || strings.has_suffix(name, "_H__") || strings.has_suffix(name, "_HPP") || strings.has_suffix(name, "_HH") || strings.has_suffix(name, "_INCLUDED") } // c_family_ext returns the suffix of PATH, which is what ctags keys the language // on; the temporary file keeps it. c_family_ext :: proc(path: string) -> string { for ext in C_EXTENSIONS { if strings.has_suffix(path, ext) { return ext } } return "" } // cached_blob_tags reads the tags of BLOB from the store, if they were parsed // before. cached_blob_tags :: proc(db: ^Db, blob: string) -> (tags: string, ok: bool) { s, sok := stmt(db, "select tags from blobs where blob = ?1") if !sok { return "", false } defer stmt_close(&s) stmt_bind_text(&s, 1, blob) if stmt_step(&s) != .Row { return "", false } return stmt_col_text(&s, 0), true } // store_blob_tags adds a parsed blob to the cache. `on conflict do nothing': // a blob's tags are a function of its content, so an existing row is already // right, and re-writing it would only churn the file. store_blob_tags :: proc(db: ^Db, blob, tags: string) -> bool { s, ok := stmt(db, `insert into blobs (blob, tags) values (?1, ?2) on conflict (blob) do nothing`) if !ok { return false } defer stmt_close(&s) stmt_bind_text(&s, 1, blob) stmt_bind_text(&s, 2, tags) return stmt_step(&s) == .Done } // parse_blob_tags parses each source blob with one ctags run and returns, per // blob, its tags as "" lines. The raw ctags kind is kept // in the cache and mapped to the store's vocabulary when the tag is used, so a // changed mapping does not need the blobs re-parsed. // // The blobs have to be real files: ctags reads the language from the name. They // are written into one temporary directory and removed again. parse_blob_tags :: proc(sources: []Blob_Source) -> (tags: map[string]string, ok: bool) { tmp_root, tmp_err := os.temp_directory(context.temp_allocator) if tmp_err != nil { fmt.eprintfln("ai-memory: no temporary directory: %v", tmp_err) return nil, false } dir := fmt.tprintf("%s/ai-memory-tags-%d", tmp_root, os.get_pid()) if err := os.make_directory_all(dir); err != nil { fmt.eprintfln("ai-memory: cannot create %s: %v", dir, err) return nil, false } defer os.remove_all(dir) paths := make([dynamic]string, 0, len(sources)) by_path := make(map[string]string) defer delete(by_path) for src, i in sources { content, cat_ok := git([]string{"cat-file", "blob", src.blob}) if !cat_ok { continue } path := fmt.tprintf("%s/%d%s", dir, i, src.ext) if err := os.write_entire_file_from_bytes(path, transmute([]u8)content); err != nil { fmt.eprintfln("ai-memory: cannot write %s: %v", path, err) continue } append(&paths, path) by_path[path] = src.blob } if len(paths) == 0 { return make(map[string]string), true } args := make([dynamic]string, 0, len(paths) + 8, context.temp_allocator) append(&args, "--sort=no", "--fields=+K", "-o", "-") // Lambdas are tagged by default, under names ctags derives from a hash of // the file (`__anon426fc91f0102'): they change whenever the line moves, so // every commit would report a handful of functions added and removed. They // are not declarations a reader asks about; the enclosing function is what // hunk context already gives. append(&args, "--extras=-{anonymous}") // Prototypes are off by default, and without them a declaration in a header // is invisible: only definitions in .cpp files would ever be `added'. append(&args, "--kinds-C=+p", "--kinds-C++=+p") append(&args, ..paths[:]) out, run_ok := run_capture(ctags_path(), args[:]) if !run_ok { if !CTAGS_MISSING { CTAGS_MISSING = true fmt.eprintln("ai-memory: ctags is not runnable, so C and C++ declarations " + "keep hunk context only (set $AI_MEMORY_CTAGS)") } return nil, false } tags = make(map[string]string) for line in split_byte(out, '\n') { if len(line) == 0 || line[0] == '!' { continue } // "namepath/^pattern$/;"kind[extra fields]" fields := split_byte(line, '\t') if len(fields) < 4 { continue } name := fields[0] blob := by_path[fields[1]] or_else "" if len(blob) == 0 || len(name) == 0 { continue } tag := fmt.tprintf("%s\t%s", fields[3], name) tags[blob] = strings.concatenate({tags[blob] or_else "", tag, "\n"}, context.temp_allocator) } return tags, true } // run_capture runs EXE with ARGS and returns its stdout. Unlike `git', a failure // here is not fatal: ctags is an optional helper. run_capture :: proc(exe: string, args: []string) -> (out: string, ok: bool) { cmd := make([dynamic]string, 0, len(args) + 1, context.temp_allocator) append(&cmd, exe) append(&cmd, ..args) state, stdout, _, err := os.process_exec({command = cmd[:]}, context.temp_allocator) if err != nil { return "", false } if state.exit_code != 0 { return "", false } return string(stdout), true } // is_change_line reports an added or removed line, excluding file headers. is_change_line :: proc(line: string) -> bool { if len(line) < 2 { return false } if line[0] != '+' && line[0] != '-' { return false } return !strings.has_prefix(line, "+++") && !strings.has_prefix(line, "---") } // merge_decl adds D to M, keeping the strongest evidence seen for the symbol. merge_decl :: proc(m: ^map[string]Decl, decl: Decl) { d := decl // Keyed by file *and* name, not by name: `main.open_store' and `db.open' // in one commit are two symbols, and merging them threw one away and // gave the survivor the other file's attribution. The separator cannot // occur in a path or an identifier. key := fmt.tprintf("%s\x1f%s", d.file, d.name) if old, exists := m[key]; exists { d.added = d.added || old.added d.removed = d.removed || old.removed d.in_context = d.in_context || old.in_context } m[key] = d } // hunk_context returns the text after the closing "@@" of a hunk header. hunk_context :: proc(line: string) -> (ctx: string, ok: bool) { rest := line[len("@@ "):] i := strings.index(rest, " @@") if i < 0 { return "", false } ctx = strings.trim_space(rest[i + len(" @@"):]) return ctx, len(ctx) > 0 } // symbol_in_context names the enclosing item from a hunk header. Our language // parsers handle Odin and Rust signatures; everything else falls back to "the // identifier in front of the parameter list". symbol_in_context :: proc(file, ctx: string) -> (name: string, kind: string, ok: bool) { if name, kind, ok = parse_decl(file, ctx); ok { return name, kind, true } if name, ok = signature_name(ctx); ok { return name, "fn", true } return "", "", false } // signature_name picks the identifier in front of the first parameter list that // is not a keyword, e.g. "static void hash_pve(int a)" -> "hash_pve". signature_name :: proc(ctx: string) -> (name: string, ok: bool) { for i in 0 ..< len(ctx) { if ctx[i] != '(' { continue } start := i for start > 0 && is_ident_byte(ctx[start - 1]) { start -= 1 } if start == i { continue } candidate := ctx[start:i] if !is_keyword(candidate) { return candidate, true } } return "", false } is_ident_byte :: proc(c: u8) -> bool { return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') } is_keyword :: proc(s: string) -> bool { keywords := [?]string{ "if", "for", "while", "switch", "match", "return", "when", "else", "sizeof", "alignof", "offsetof", "typeof", "catch", "assert", "cast", "transmute", "in", "do", "and", "or", "not", "unsafe", "async", } for k in keywords { if s == k { return true } } return false } // symbol_name qualifies a declaration with the file it lives in, so that // `main.open_store' and `db.open' do not collide. symbol_name :: proc(file, name: string) -> string { base := file if i := strings.last_index_byte(base, '/'); i >= 0 { base = base[i + 1:] } // A C-family name keeps its extension (lmm.cpp.mdb_analyze) because a header // declaration and the definition in the source are different rows; the others // are single-file languages, where the base name is enough. extensions := [?]string{".odin", ".rs", ".py", ".rb"} for ext in extensions { if strings.has_suffix(base, ext) { base = base[:len(base) - len(ext)] break } } if len(base) == 0 { return name } return fmt.tprintf("%s.%s", base, name) } // change_of collapses "saw it added and removed" into a single "mod", and // anything only seen in hunk headers into "context". change_of :: proc(d: Decl) -> string { switch { case d.added && d.removed: return "mod" case d.removed: return "remove" case d.added: return "add" } return "context" } // split_byte splits s on a single-byte separator, borrowing from s. split_byte :: proc(s: string, sep: u8) -> []string { parts := make([dynamic]string, 0, 16, context.temp_allocator) start := 0 for i in 0 ..< len(s) { if s[i] == sep { append(&parts, s[start:i]) start = i + 1 } } append(&parts, s[start:]) return parts[:] } // parse_i64 parses an i64, defaulting to 0 (timestamps only). parse_i64 :: proc(s: string) -> i64 { if v, ok := strconv.parse_i64(s, 10); ok { return v } return 0 } // join_fields re-assembles a commit body that happened to contain the field // separator. join_fields :: proc(fields: []string) -> string { if len(fields) == 0 { return "" } out := fields[0] for f in fields[1:] { out = fmt.tprintf("%s\x1f%s", out, f) } return out } // git_commit reads a single commit's metadata, for commits that `record' or // `attach' refer to before the next `sync'. git_commit :: proc(hash: string) -> (c: Git_Commit, ok: bool) { raw, good := git([]string{"show", "--no-patch", "--no-color", "--format=%H%x1f%at%x1f%s%x1f%b%x1e", hash}) if !good { return {}, false } for raw_part in split_byte(raw, REC_SEP) { part := strings.trim_space(raw_part) if len(part) == 0 { continue } fields := split_byte(part, FLD_SEP) if len(fields) < 3 { continue } c.hash = fields[0] c.author_ts = parse_i64(fields[1]) c.subject = fields[2] if len(fields) > 3 { c.body = join_fields(fields[3:]) } return c, true } return {}, false }