about summary refs log tree commit diff
path: root/lang.odin
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-11-17 10:18:26 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-11-17 10:18:26 +0000
commitb7234d2514b4a693b37cf4968b267a5fafe16678 (patch)
tree0691588a6d1507230f9298ac18b855675c4d2eeb /lang.odin
parentb861693dc26a8f9944c3464a34c101af5a81d33c (diff)
downloadai-memory-master.tar.gz
symbols: scan Python and Ruby declarations, and make why FILEBASE resolve HEAD master
A commit that only adds a file recorded nothing: git has no before-side for
the new file, so its hunks carry no context ("@@ -0,0 +1,207 @@"), and Python
and Ruby had no declaration scanner to fall back on. Adding
bin/geno2mdb.py to gemma-wrapper left one commit row and two reasoning
entries with no symbol at all, so `why` could not reach them while `search`
and `list` could.

Both languages are now scanned line by line, which is the right tool here:
in Python and Ruby a declaration is the first token on its line, so the
keyword decides -- `def foo(bar):` declares, `foo(bar)` calls -- the
distinction no C++ line scanner can make.

  Python   def / async def -> fn, class -> type, and at column 0 only
           NAME = value -> const, NAME: Type -> var (an indented assignment
           is a local inside a def)
  Ruby     def / def self.NAME (with ? ! = suffixes) -> fn, class / module ->
           type, and a column-0 capitalised NAME = value -> const

An added file needs no special case: every one of its lines is a '+' line,
which git_touched already walks, so the same path that records an added line
in an existing file now records a new file's declarations. Names strip .py
and .rb like .odin and .rs (script.helper); the C family keeps its extension
because a header declaration and the source definition are different rows.

The lookup's last resort was `s.name like '%name'` -- a suffix match -- so
`why geno2mdb` could never find a file's own symbols even though SKILL.md
documents a LIKE '%...%' scan. It is a substring match now, so the file base
works, and the doc is true again.

The check phase commits script.py and script.rb into its throwaway
repository and asserts both land as `add` rows and that `why script`
resolves; measured on gemma-wrapper, sync adds 344 touches over 319 commits
and `why read_snps` reaches the reasoning attached to the commit that
introduced it.
Diffstat (limited to 'lang.odin')
-rw-r--r--lang.odin129
1 files changed, 129 insertions, 0 deletions
diff --git a/lang.odin b/lang.odin
index 58d2f34..8a56013 100644
--- a/lang.odin
+++ b/lang.odin
@@ -8,6 +8,8 @@ Lang :: enum {
 	Odin,
 	Rust,
 	C_Family, // .c .h .cc .cpp .cxx .hpp .hh .hxx .ipp .tpp .inl
+	Python,   // .py
+	Ruby,     // .rb
 	Other,
 }
 
@@ -36,6 +38,10 @@ lang_of :: proc(file: string) -> Lang {
 		return .Odin
 	case strings.has_suffix(file, ".rs"):
 		return .Rust
+	case strings.has_suffix(file, ".py"):
+		return .Python
+	case strings.has_suffix(file, ".rb"):
+		return .Ruby
 	}
 	for ext in C_EXTENSIONS {
 		if strings.has_suffix(file, ext) {
@@ -53,6 +59,10 @@ parse_decl :: proc(file, line: string) -> (name: string, kind: string, ok: bool)
 		return parse_odin_decl(line)
 	case .Rust:
 		return parse_rust_decl(line)
+	case .Python:
+		return parse_python_decl(line)
+	case .Ruby:
+		return parse_ruby_decl(line)
 	case .C_Family:
 		// Deliberately no lexical scan.  In C and C++ a diff line cannot say
 		// whether it declares anything: inside a body `foo(bar);' is a call and
@@ -212,6 +222,125 @@ parse_rust_decl :: proc(line: string) -> (name: string, kind: string, ok: bool)
 	return "", "", false
 }
 
+// -- Python -----------------------------------------------------------------
+//
+//	def NAME(...):        -> fn   (and `async def')
+//	class NAME:           -> type
+//	NAME = value          -> const  (module level only; indented is inside a def)
+//	NAME: Type            -> var    (module level only)
+//
+// A line scanner is the right tool here, unlike C and C++: a Python
+// declaration is the first thing on its line, so there is no statement that
+// can look like one -- `foo(bar)` is a call and `def foo(bar):` is not, and the
+// keyword settles it.  What it cannot see is a definition built at run time
+// (`f = lambda x: x`, a `def` inside an `exec`), which is why the gist of a
+// symbol is a name and the file it lives in, not a promise about the AST.
+parse_python_decl :: proc(line: string) -> (name: string, kind: string, ok: bool) {
+	indented := len(line) > 0 && (line[0] == ' ' || line[0] == '\t')
+	s := strings.trim_left_space(line)
+	if len(s) == 0 || s[0] == '#' || s[0] == '@' || s[0] == '"' || s[0] == '\'' {
+		return "", "", false
+	}
+	if strings.has_prefix(s, "async def ") {
+		if n, nok := ident_after(s, "async def "); nok {
+			return n, "fn", true
+		}
+		return "", "", false
+	}
+	if strings.has_prefix(s, "def ") {
+		if n, nok := ident_after(s, "def "); nok {
+			return n, "fn", true
+		}
+		return "", "", false
+	}
+	if strings.has_prefix(s, "class ") {
+		if n, nok := ident_after(s, "class "); nok {
+			return n, "type", true
+		}
+		return "", "", false
+	}
+	if indented {
+		// Inside a def or a class body: an assignment there is a local, not a
+		// declaration a reader asks "why" about.
+		return "", "", false
+	}
+	if n, aok := assignment_name(s); aok {
+		return n, "const", true
+	}
+	if sep := strings.index_byte(s, ':'); sep > 0 {
+		if lhs := strings.trim_space(s[:sep]); is_ident(lhs) && !strings.contains(s, "=") {
+			return lhs, "var", true
+		}
+	}
+	return "", "", false
+}
+
+// -- Ruby -------------------------------------------------------------------
+//
+//	def NAME / def self.NAME / def NAME?  -> fn
+//	class NAME / module NAME             -> type
+//	NAME = value                         -> const  (module level, capitalised)
+//
+// Ruby's declarations are line-anchored in the same way, with one trap the
+// callers of this store care about: `def` and `class` also appear inside
+// strings and heredocs, and no line scanner can tell.  Anchoring on the first
+// token keeps that rare.
+parse_ruby_decl :: proc(line: string) -> (name: string, kind: string, ok: bool) {
+	indented := len(line) > 0 && (line[0] == ' ' || line[0] == '\t')
+	s := strings.trim_left_space(line)
+	if len(s) == 0 || s[0] == '#' || s[0] == '=' {
+		return "", "", false
+	}
+	if strings.has_prefix(s, "def ") {
+		rest := s[len("def "):]
+		if strings.has_prefix(rest, "self.") {
+			rest = rest[len("self."):]
+		}
+		if n, nok := ident_after(rest, ""); nok {
+			// `ready?', `save!', `name=' are one identifier to Ruby.
+			return strings.trim_right(n, "?!=:"), "fn", true
+		}
+		return "", "", false
+	}
+	if strings.has_prefix(s, "class ") || strings.has_prefix(s, "module ") {
+		prefix := strings.has_prefix(s, "class ") ? "class " : "module "
+		if n, nok := ident_after(s, prefix); nok {
+			return n, "type", true
+		}
+		return "", "", false
+	}
+	if indented {
+		return "", "", false
+	}
+	// A Ruby constant is capitalised; `x = 1` at column 0 is a local in a
+	// script and not worth a symbol.
+	if n, aok := assignment_name(s); aok && n[0] >= 'A' && n[0] <= 'Z' {
+		return n, "const", true
+	}
+	return "", "", false
+}
+
+// assignment_name reports NAME from `NAME = value' (not `NAME == value').
+assignment_name :: proc(s: string) -> (name: string, ok: bool) {
+	for i in 0 ..< len(s) {
+		if s[i] != '=' {
+			continue
+		}
+		if i + 1 < len(s) && s[i + 1] == '=' {
+			return "", false
+		}
+		if i > 0 && (s[i - 1] == '!' || s[i - 1] == '<' || s[i - 1] == '>' || s[i - 1] == '=') {
+			return "", false
+		}
+		lhs := strings.trim_space(s[:i])
+		if is_ident(lhs) {
+			return lhs, true
+		}
+		return "", false
+	}
+	return "", false
+}
+
 // ident_after reads the identifier that follows PREFIX in S.
 ident_after :: proc(s, prefix: string) -> (name: string, ok: bool) {
 	rest := strings.trim_left_space(s[len(prefix):])