feat: scanner added

pkg/saccharine/scan.go

@@ -1,64 +1,23 @@
 package saccharine
 
-import (
-	"fmt"
-	"unicode"
+import "git.maximhutz.com/max/lambda/pkg/token"
 
-	"git.maximhutz.com/max/lambda/pkg/iterator"
-	"git.maximhutz.com/max/lambda/pkg/token"
-)
+// scanner is the declarative lexer for the Saccharine language.
+var scanner = token.NewScanner[TokenType]().
+	On(`:=`, TokenAssign).
+	On(`\(`, TokenOpenParen).
+	On(`\)`, TokenCloseParen).
+	On(`\{`, TokenOpenBrace).
+	On(`\}`, TokenCloseBrace).
+	On(`;`, TokenHardBreak).
+	On(`\n`, TokenSoftBreak).
+	On(`\\`, TokenSlash).
+	On(`\.`, TokenDot).
+	On(`[a-zA-Z0-9_]+`, TokenAtom).
+	Skip(`#[^\n]*`).
+	Skip(`[^\S\n]+`)
 
-// Pulls the next token from an iterator over runes. If it cannot, it will
-// return nil. If an error occurs, it will return that.
-func scanToken(i *iterator.Iterator[rune]) (*Token, error) {
-	index := i.Index()
-
-	if i.Done() {
-		return nil, nil
-	}
-
-	letter, err := i.Next()
-	if err != nil {
-		return nil, fmt.Errorf("cannot produce next token: %w", err)
-	}
-
-	switch {
-	case letter == '(':
-		return token.New(TokenOpenParen, index), nil
-	case letter == ')':
-		return token.New(TokenCloseParen, index), nil
-	case letter == '.':
-		return token.New(TokenDot, index), nil
-	case letter == '\\':
-		return token.New(TokenSlash, index), nil
-	case letter == '\n':
-		return token.New(TokenSoftBreak, index), nil
-	case letter == '{':
-		return token.New(TokenOpenBrace, index), nil
-	case letter == '}':
-		return token.New(TokenCloseBrace, index), nil
-	case letter == ':':
-		if _, err := token.ScanCharacter(i, '='); err != nil {
-			return nil, err
-		} else {
-			return token.New(TokenAssign, index), nil
-		}
-	case letter == ';':
-		return token.New(TokenHardBreak, index), nil
-	case letter == '#':
-		// Skip everything until the next newline or EOF.
-		i.While(func(r rune) bool { return r != '\n' })
-		return nil, nil
-	case unicode.IsSpace(letter):
-		return nil, nil
-	case token.IsVariable(letter):
-		return token.ScanAtom(i, letter, TokenAtom, index), nil
-	}
-
-	return nil, fmt.Errorf("unknown character '%v'", string(letter))
-}
-
-// scan a string into tokens.
+// scan tokenizes a string into Saccharine tokens.
 func scan(input string) ([]Token, error) {
-	return token.Scan(input, scanToken)
+	return scanner.Scan(input)
 }