feat: stuff

pkg/tokenizer/tokenizer.go

@@ -1,95 +1,72 @@
 package tokenizer
 
 import (
+	"errors"
 	"fmt"
-	"strings"
 	"unicode"
 
 	"git.maximhutz.com/max/lambda/pkg/iterator"
 )
 
+// isVariables determines whether a rune can be a valid variable.
 func isVariable(r rune) bool {
 	return unicode.IsLetter(r) || unicode.IsNumber(r)
 }
 
+// Pulls the next token from an iterator over runes. If it cannot, it will
+// return nil. If an error occurs, it will return that.
 func getToken(i *iterator.Iterator[rune]) (*Token, error) {
+	index := i.Index()
+
 	if i.IsDone() {
 		return nil, nil
 	}
 
-	letter, err := i.Next()
+	letter, err := i.Pop()
 	if err != nil {
 		return nil, fmt.Errorf("cannot produce next token: %w", err)
 	}
 
-	// If it is an operand.
-	switch letter {
-	case '(':
-		return &Token{
-			Type:  TokenOpenParen,
-			Index: i.Index(),
-			Value: string(letter),
-		}, nil
-	case ')':
-		return &Token{
-			Type:  TokenCloseParen,
-			Index: i.Index(),
-			Value: string(letter),
-		}, nil
-	case '.':
-		return &Token{
-			Type:  TokenDot,
-			Index: i.Index(),
-			Value: string(letter),
-		}, nil
-	case '\\':
-		return &Token{
-			Type:  TokenSlash,
-			Index: i.Index(),
-			Value: string(letter),
-		}, nil
-	}
-
-	// If it is a space.
-	if unicode.IsSpace(letter) {
+	switch {
+	case letter == '(':
+		// The opening deliminator of an application.
+		return &Token{Type: TokenOpenParen, Index: index, Value: string(letter)}, nil
+	case letter == ')':
+		// The terminator of an application.
+		return &Token{Type: TokenCloseParen, Index: index, Value: string(letter)}, nil
+	case letter == '.':
+		// The terminator of the parameters in an abstraction.
+		return &Token{Type: TokenDot, Index: index, Value: string(letter)}, nil
+	case letter == '\\':
+		// The opening deliminator of an abstraction.
+		return &Token{Type: TokenSlash, Index: index, Value: string(letter)}, nil
+	case unicode.IsSpace(letter):
+		// If there is a space character, ignore it.
 		return nil, nil
+	case isVariable(letter):
+		rest := i.PopWhile(isVariable)
+		atom := string(append([]rune{letter}, rest...))
+
+		return &Token{Index: index, Type: TokenVariable, Value: atom}, nil
 	}
 
-	// Otherwise, it is an atom.
-	atom := strings.Builder{}
-	index := i.Index()
-	for !i.IsDone() {
-		pop, err := i.Peek()
-		if err != nil || !isVariable(pop) {
-			break
-		}
-
-		atom.WriteRune(pop)
-		if _, err := i.Next(); err != nil {
-			break
-		}
-	}
-
-	return &Token{
-		Index: index,
-		Type:  TokenVariable,
-		Value: atom.String(),
-	}, nil
+	return nil, fmt.Errorf("unknown character '%v'", letter)
 }
 
-func GetTokens(input []rune) ([]Token, []error) {
+// Parses a list of runes into tokens. All error encountered are returned, as well.
+func GetTokens(input []rune) ([]Token, error) {
 	i := iterator.New(input)
 	tokens := []Token{}
-	errors := []error{}
+	errorList := []error{}
 
 	for !i.IsDone() {
-		token, err := getToken(&i)
+		token, err := getToken(i)
 		if err != nil {
-			errors = append(errors, err)
+			errorList = append(errorList, err)
 		} else if token != nil {
 			tokens = append(tokens, *token)
 		}
 	}
 
-	return tokens, errors
+	return tokens, errors.Join(errorList...)
 }