wip: new folder structure, overhaul language

pkg/saccharine/tokenizer.go

@@ -0,0 +1,72 @@
+package saccharine
+
+import (
+	"errors"
+	"fmt"
+	"unicode"
+
+	"git.maximhutz.com/max/lambda/pkg/iterator"
+)
+
+// isVariables determines whether a rune can be a valid variable.
+func isVariable(r rune) bool {
+	return unicode.IsLetter(r) || unicode.IsNumber(r)
+}
+
+// Pulls the next token from an iterator over runes. If it cannot, it will
+// return nil. If an error occurs, it will return that.
+func getToken(i *iterator.Iterator[rune]) (*Token, error) {
+	index := i.Index()
+
+	if i.IsDone() {
+		return nil, nil
+	}
+
+	letter, err := i.Pop()
+	if err != nil {
+		return nil, fmt.Errorf("cannot produce next token: %w", err)
+	}
+
+	switch {
+	case letter == '(':
+		// The opening deliminator of an application.
+		return &Token{Type: TokenOpenParen, Index: index, Value: string(letter)}, nil
+	case letter == ')':
+		// The terminator of an application.
+		return &Token{Type: TokenCloseParen, Index: index, Value: string(letter)}, nil
+	case letter == '.':
+		// The terminator of the parameters in an abstraction.
+		return &Token{Type: TokenDot, Index: index, Value: string(letter)}, nil
+	case letter == '\\':
+		// The opening deliminator of an abstraction.
+		return &Token{Type: TokenSlash, Index: index, Value: string(letter)}, nil
+	case unicode.IsSpace(letter):
+		// If there is a space character, ignore it.
+		return nil, nil
+	case isVariable(letter):
+		rest := i.PopWhile(isVariable)
+		atom := string(append([]rune{letter}, rest...))
+
+		return &Token{Index: index, Type: TokenVariable, Value: atom}, nil
+	}
+
+	return nil, fmt.Errorf("unknown character '%v'", letter)
+}
+
+// Parses a list of runes into tokens. All error encountered are returned, as well.
+func GetTokens(input []rune) ([]Token, error) {
+	i := iterator.New(input)
+	tokens := []Token{}
+	errorList := []error{}
+
+	for !i.IsDone() {
+		token, err := getToken(i)
+		if err != nil {
+			errorList = append(errorList, err)
+		} else if token != nil {
+			tokens = append(tokens, *token)
+		}
+	}
+
+	return tokens, errors.Join(errorList...)
+}