feat: better recursive descent

pkg/saccharine/tokenizer.go

@@ -3,9 +3,11 @@ package saccharine
 import (
 	"errors"
 	"fmt"
+	"log/slog"
 	"unicode"
 
 	"git.maximhutz.com/max/lambda/pkg/iterator"
+	"git.maximhutz.com/max/lambda/pkg/saccharine/token"
 )
 
 // isVariables determines whether a rune can be a valid variable.
@@ -13,60 +15,77 @@ func isVariable(r rune) bool {
 	return unicode.IsLetter(r) || unicode.IsNumber(r)
 }
 
+func parseRune(i *iterator.Iterator[rune], expected func(rune) bool) (rune, error) {
+	i2 := i.Copy()
+
+	if r, err := i2.Next(); err != nil {
+		return r, err
+	} else if !expected(r) {
+		return r, fmt.Errorf("got unexpected rune %v'", r)
+	} else {
+		i.Sync(i2)
+		return r, nil
+	}
+}
+
 // Pulls the next token from an iterator over runes. If it cannot, it will
 // return nil. If an error occurs, it will return that.
-func getToken(i *iterator.Iterator[rune]) (*Token, error) {
+func getToken(i *iterator.Iterator[rune]) (*token.Token, error) {
 	index := i.Index()
 
-	if i.IsDone() {
+	if i.Done() {
 		return nil, nil
 	}
 
-	letter, err := i.Pop()
+	letter, err := i.Next()
 	if err != nil {
 		return nil, fmt.Errorf("cannot produce next token: %w", err)
 	}
 
 	switch {
 	case letter == '(':
-		// The opening deliminator of an application.
-		return &Token{Type: TokenOpenParen, Index: index, Value: string(letter)}, nil
+		return token.NewOpenParen(index), nil
 	case letter == ')':
-		// The terminator of an application.
-		return &Token{Type: TokenCloseParen, Index: index, Value: string(letter)}, nil
+		return token.NewCloseParen(index), nil
 	case letter == '.':
-		// The terminator of the parameters in an abstraction.
-		return &Token{Type: TokenDot, Index: index, Value: string(letter)}, nil
+		return token.NewDot(index), nil
 	case letter == '\\':
-		// The opening deliminator of an abstraction.
-		return &Token{Type: TokenSlash, Index: index, Value: string(letter)}, nil
+		return token.NewSlash(index), nil
 	case unicode.IsSpace(letter):
-		// If there is a space character, ignore it.
 		return nil, nil
 	case isVariable(letter):
-		rest := i.PopWhile(isVariable)
-		atom := string(append([]rune{letter}, rest...))
+		atom := []rune{letter}
 
-		return &Token{Index: index, Type: TokenVariable, Value: atom}, nil
+		for {
+			if r, err := parseRune(i, isVariable); err != nil {
+				break
+			} else {
+				atom = append(atom, r)
+			}
+		}
+
+		return token.NewAtom(string(atom), index), nil
 	}
 
 	return nil, fmt.Errorf("unknown character '%v'", letter)
 }
 
 // Parses a list of runes into tokens. All error encountered are returned, as well.
-func GetTokens(input []rune) ([]Token, error) {
-	i := iterator.New(input)
-	tokens := []Token{}
+func GetTokens(input []rune) (*iterator.Iterator[token.Token], error) {
+	i := iterator.Of(input)
+	tokens := []token.Token{}
 	errorList := []error{}
 
-	for !i.IsDone() {
+	for !i.Done() {
 		token, err := getToken(i)
 		if err != nil {
+			slog.Info("token error", "error", err)
 			errorList = append(errorList, err)
 		} else if token != nil {
+			slog.Info("token parsed", "token", token)
 			tokens = append(tokens, *token)
 		}
 	}
 
-	return tokens, errors.Join(errorList...)
+	return iterator.Of(tokens), errors.Join(errorList...)
 }