refactor: extract shared token package (#46)

## Description

Both the `saccharine` and `lambda` packages need tokenizing and parsing primitives.
This PR extracts shared token infrastructure into a new `pkg/token` package, then wires both languages up to use it.

- Add `pkg/token` with a generic `Token[T]` type, `Scan`, `ScanAtom`, `ScanRune`, `ScanCharacter`, `IsVariable`, `ParseRawToken`, and `ParseList`.
- Refactor `pkg/saccharine` to delegate to `pkg/token`, removing duplicated scanning and parsing helpers.
- Implement `Codec.Decode` for `pkg/lambda` (scanner + parser) using the shared token package.
- Add `iterator.While` for predicate-driven iteration.
- Rename `iterator.Do` to `iterator.Try` to better describe its rollback semantics.

### Decisions

- The `Type` constraint (`comparable` + `Name() string`) keeps the generic token flexible while ensuring every token type can produce readable error messages.
- `iterator.Do` was renamed to `iterator.Try` since it describes a try/rollback operation, not a side-effecting "do".

## Benefits

- Eliminates duplicated token, scanning, and parsing code between languages.
- Enables the `lambda` package to decode (parse) lambda calculus strings, which was previously unimplemented.
- Makes it straightforward to add new languages by reusing `pkg/token` primitives.

## Checklist

- [x] Code follows conventional commit format.
- [x] Branch follows naming convention (`<type>/<description>`). Always use underscores.
- [x] Tests pass (if applicable).
- [ ] Documentation updated (if applicable).

Reviewed-on: #46
Co-authored-by: M.V. Hutz <git@maximhutz.me>
Co-committed-by: M.V. Hutz <git@maximhutz.me>

pkg/saccharine/scan.go

@@ -1,44 +1,13 @@
 package saccharine
 
 import (
-	"errors"
 	"fmt"
 	"unicode"
 
 	"git.maximhutz.com/max/lambda/pkg/iterator"
+	"git.maximhutz.com/max/lambda/pkg/token"
 )
 
-// isVariables determines whether a rune can be a valid variable.
-func isVariable(r rune) bool {
-	return unicode.IsLetter(r) || unicode.IsNumber(r)
-}
-
-func scanRune(i *iterator.Iterator[rune], expected func(rune) bool) (rune, error) {
-	i2 := i.Copy()
-
-	if r, err := i2.Next(); err != nil {
-		return r, err
-	} else if !expected(r) {
-		return r, fmt.Errorf("got unexpected rune %v'", r)
-	} else {
-		i.Sync(i2)
-		return r, nil
-	}
-}
-
-func scanCharacter(i *iterator.Iterator[rune], expected rune) (rune, error) {
-	i2 := i.Copy()
-
-	if r, err := i2.Next(); err != nil {
-		return r, err
-	} else if r != expected {
-		return r, fmt.Errorf("got unexpected rune %v'", r)
-	} else {
-		i.Sync(i2)
-		return r, nil
-	}
-}
-
 // Pulls the next token from an iterator over runes. If it cannot, it will
 // return nil. If an error occurs, it will return that.
 func scanToken(i *iterator.Iterator[rune]) (*Token, error) {
@@ -55,56 +24,35 @@ func scanToken(i *iterator.Iterator[rune]) (*Token, error) {
 
 	switch {
 	case letter == '(':
-		return NewToken(TokenOpenParen, index), nil
+		return token.New(TokenOpenParen, index), nil
 	case letter == ')':
-		return NewToken(TokenCloseParen, index), nil
+		return token.New(TokenCloseParen, index), nil
 	case letter == '.':
-		return NewToken(TokenDot, index), nil
+		return token.New(TokenDot, index), nil
 	case letter == '\\':
-		return NewToken(TokenSlash, index), nil
+		return token.New(TokenSlash, index), nil
 	case letter == '\n':
-		return NewToken(TokenSoftBreak, index), nil
+		return token.New(TokenSoftBreak, index), nil
 	case letter == '{':
-		return NewToken(TokenOpenBrace, index), nil
+		return token.New(TokenOpenBrace, index), nil
 	case letter == '}':
-		return NewToken(TokenCloseBrace, index), nil
+		return token.New(TokenCloseBrace, index), nil
 	case letter == ':':
-		if _, err := scanCharacter(i, '='); err != nil {
+		if _, err := token.ScanCharacter(i, '='); err != nil {
 			return nil, err
 		} else {
-			return NewToken(TokenAssign, index), nil
+			return token.New(TokenAssign, index), nil
 		}
 	case letter == ';':
-		return NewToken(TokenHardBreak, index), nil
+		return token.New(TokenHardBreak, index), nil
 	case letter == '#':
 		// Skip everything until the next newline or EOF.
-		for !i.Done() {
-			r, err := i.Next()
-			if err != nil {
-				return nil, fmt.Errorf("error while parsing comment: %w", err)
-			}
-
-			if r == '\n' {
-				// Put the newline back so it can be processed as a soft break.
-				i.Back()
-				break
-			}
-		}
+		i.While(func(r rune) bool { return r != '\n' })
 		return nil, nil
 	case unicode.IsSpace(letter):
 		return nil, nil
-	case isVariable(letter):
-		atom := []rune{letter}
-
-		for {
-			if r, err := scanRune(i, isVariable); err != nil {
-				break
-			} else {
-				atom = append(atom, r)
-			}
-		}
-
-		return NewTokenAtom(string(atom), index), nil
+	case token.IsVariable(letter):
+		return token.ScanAtom(i, letter, TokenAtom, index), nil
 	}
 
 	return nil, fmt.Errorf("unknown character '%v'", string(letter))
@@ -112,18 +60,5 @@ func scanToken(i *iterator.Iterator[rune]) (*Token, error) {
 
 // scan a string into tokens.
 func scan(input string) ([]Token, error) {
-	i := iterator.Of([]rune(input))
-	tokens := []Token{}
-	errorList := []error{}
-
-	for !i.Done() {
-		token, err := scanToken(i)
-		if err != nil {
-			errorList = append(errorList, err)
-		} else if token != nil {
-			tokens = append(tokens, *token)
-		}
-	}
-
-	return tokens, errors.Join(errorList...)
+	return token.Scan(input, scanToken)
 }