feat: tokenizer

2025-12-23 14:17:43 -05:00
parent 4845acea73
commit 61bb622dcd
8 changed files with 227 additions and 0 deletions
--- a/pkg/tokenizer/token.go
+++ b/pkg/tokenizer/token.go
@@ -0,0 +1,17 @@
+package tokenizer
+
+type TokenType int
+
+const (
+	TokenOpenParen TokenType = iota
+	TokenCloseParen
+	TokenAtom
+	TokenSlash
+	TokenDot
+)
+
+type Token struct {
+	Index int
+	Type TokenType
+	Value string
+}