M.V. Hutz
aca197ef51
## Description `iterator.Try` previously copied the entire iterator and synced it back on success, causing an unnecessary heap allocation on every call. This PR simplifies `Try` to save and restore the index directly, and removes the now-unused `Copy` and `Sync` methods. - Rewrite `ScanRune` and `ParseRawToken` as peek-then-advance, eliminating the need for `Try` at leaf level. - Remove redundant `Try` wrappers from `parseExpression`, `parseAbstraction`, `parseApplication`, `parseLet`, and `parseToken`, which are already disambiguated by their callers. - Keep `Try` only where true backtracking is needed: `parseStatement`, which must choose between `parseLet` and `parseDeclare`. - Fix pre-existing panic in saccharine `parseExpression` when the iterator is exhausted (added `Done()` guard). ### Decisions - `Try` now operates on the original iterator instead of a copy, removing the confusing pattern where the callback's `i` was a different object than the caller's `i`. - Removed `parseSoftBreak` and `parseHardBreak` helper functions since `ParseRawToken` no longer needs `Try` wrapping. ## Benefits - Eliminates a heap allocation per `Try` call. - Reduces nesting and indirection in all parse functions. - Makes the code easier to follow by removing the shadow-`i` pattern. - `Try` is now only used at genuine choice points in the grammar. ## Checklist - [x] Code follows conventional commit format. - [x] Branch follows naming convention (`<type>/<description>`). Always use underscores. - [x] Tests pass (if applicable). - [x] Documentation updated (if applicable). Reviewed-on: #47 Co-authored-by: M.V. Hutz <git@maximhutz.me> Co-committed-by: M.V. Hutz <git@maximhutz.me>
75 lines
2.1 KiB
Go
75 lines
2.1 KiB
Go
package token
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"unicode"
|
|
|
|
"git.maximhutz.com/max/lambda/pkg/iterator"
|
|
)
|
|
|
|
// IsVariable determines whether a rune can be a valid variable character.
|
|
func IsVariable(r rune) bool {
|
|
return unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_'
|
|
}
|
|
|
|
// ScanRune consumes the next rune from the iterator if it satisfies the
|
|
// predicate.
|
|
// Returns an error if the iterator is exhausted or the rune does not match.
|
|
func ScanRune(i *iterator.Iterator[rune], expected func(rune) bool) (rune, error) {
|
|
r, err := i.Get()
|
|
if err != nil {
|
|
return r, err
|
|
}
|
|
if !expected(r) {
|
|
return r, fmt.Errorf("got unexpected rune %v'", r)
|
|
}
|
|
i.Forward()
|
|
return r, nil
|
|
}
|
|
|
|
// ScanCharacter consumes the next rune from the iterator if it matches the
|
|
// expected rune exactly.
|
|
// Returns an error if the iterator is exhausted or the rune does not match.
|
|
func ScanCharacter(i *iterator.Iterator[rune], expected rune) (rune, error) {
|
|
return ScanRune(i, func(r rune) bool { return r == expected })
|
|
}
|
|
|
|
// ScanAtom scans a contiguous sequence of variable characters into a single
|
|
// atom token.
|
|
// The first rune has already been consumed and is passed in.
|
|
func ScanAtom[T Type](i *iterator.Iterator[rune], first rune, typ T, column int) *Token[T] {
|
|
atom := []rune{first}
|
|
|
|
for {
|
|
if r, err := ScanRune(i, IsVariable); err != nil {
|
|
break
|
|
} else {
|
|
atom = append(atom, r)
|
|
}
|
|
}
|
|
|
|
return NewAtom(typ, string(atom), column)
|
|
}
|
|
|
|
// Scan tokenizes an input string using a language-specific scanToken function.
|
|
// The scanToken function is called repeatedly until the input is exhausted.
|
|
// It returns nil (no token, no error) for skippable input like whitespace.
|
|
// Errors are accumulated and returned joined at the end.
|
|
func Scan[T Type](input string, scanToken func(*iterator.Iterator[rune]) (*Token[T], error)) ([]Token[T], error) {
|
|
i := iterator.Of([]rune(input))
|
|
tokens := []Token[T]{}
|
|
errorList := []error{}
|
|
|
|
for !i.Done() {
|
|
token, err := scanToken(i)
|
|
if err != nil {
|
|
errorList = append(errorList, err)
|
|
} else if token != nil {
|
|
tokens = append(tokens, *token)
|
|
}
|
|
}
|
|
|
|
return tokens, errors.Join(errorList...)
|
|
}
|