feat: new parser strategy

pkg/token/scan.go

@@ -0,0 +1,74 @@
+package token
+
+import (
+	"errors"
+	"fmt"
+	"unicode"
+
+	"git.maximhutz.com/max/lambda/pkg/iterator"
+)
+
+// IsVariable determines whether a rune can be a valid variable character.
+func IsVariable(r rune) bool {
+	return unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_'
+}
+
+// ScanRune consumes the next rune from the iterator if it satisfies the
+// predicate.
+// Returns an error if the iterator is exhausted or the rune does not match.
+func ScanRune(i *iterator.Iterator[rune], expected func(rune) bool) (rune, error) {
+	return iterator.Try(i, func(i *iterator.Iterator[rune]) (rune, error) {
+		if r, err := i.Next(); err != nil {
+			return r, err
+		} else if !expected(r) {
+			return r, fmt.Errorf("got unexpected rune %v'", r)
+		} else {
+			return r, nil
+		}
+	})
+}
+
+// ScanCharacter consumes the next rune from the iterator if it matches the
+// expected rune exactly.
+// Returns an error if the iterator is exhausted or the rune does not match.
+func ScanCharacter(i *iterator.Iterator[rune], expected rune) (rune, error) {
+	return ScanRune(i, func(r rune) bool { return r == expected })
+}
+
+// ScanAtom scans a contiguous sequence of variable characters into a single
+// atom token.
+// The first rune has already been consumed and is passed in.
+func ScanAtom[T Type](i *iterator.Iterator[rune], first rune, typ T, column int) *Token[T] {
+	atom := []rune{first}
+
+	for {
+		if r, err := ScanRune(i, IsVariable); err != nil {
+			break
+		} else {
+			atom = append(atom, r)
+		}
+	}
+
+	return NewAtom(typ, string(atom), column)
+}
+
+// Scan tokenizes an input string using a language-specific scanToken function.
+// The scanToken function is called repeatedly until the input is exhausted.
+// It returns nil (no token, no error) for skippable input like whitespace.
+// Errors are accumulated and returned joined at the end.
+func Scan[T Type](input string, scanToken func(*iterator.Iterator[rune]) (*Token[T], error)) ([]Token[T], error) {
+	i := iterator.Of([]rune(input))
+	tokens := []Token[T]{}
+	errorList := []error{}
+
+	for !i.Done() {
+		token, err := scanToken(i)
+		if err != nil {
+			errorList = append(errorList, err)
+		} else if token != nil {
+			tokens = append(tokens, *token)
+		}
+	}
+
+	return tokens, errors.Join(errorList...)
+}