refactor: extract shared token package (#46)
## Description Both the `saccharine` and `lambda` packages need tokenizing and parsing primitives. This PR extracts shared token infrastructure into a new `pkg/token` package, then wires both languages up to use it. - Add `pkg/token` with a generic `Token[T]` type, `Scan`, `ScanAtom`, `ScanRune`, `ScanCharacter`, `IsVariable`, `ParseRawToken`, and `ParseList`. - Refactor `pkg/saccharine` to delegate to `pkg/token`, removing duplicated scanning and parsing helpers. - Implement `Codec.Decode` for `pkg/lambda` (scanner + parser) using the shared token package. - Add `iterator.While` for predicate-driven iteration. - Rename `iterator.Do` to `iterator.Try` to better describe its rollback semantics. ### Decisions - The `Type` constraint (`comparable` + `Name() string`) keeps the generic token flexible while ensuring every token type can produce readable error messages. - `iterator.Do` was renamed to `iterator.Try` since it describes a try/rollback operation, not a side-effecting "do". ## Benefits - Eliminates duplicated token, scanning, and parsing code between languages. - Enables the `lambda` package to decode (parse) lambda calculus strings, which was previously unimplemented. - Makes it straightforward to add new languages by reusing `pkg/token` primitives. ## Checklist - [x] Code follows conventional commit format. - [x] Branch follows naming convention (`<type>/<description>`). Always use underscores. - [x] Tests pass (if applicable). - [ ] Documentation updated (if applicable). Reviewed-on: #46 Co-authored-by: M.V. Hutz <git@maximhutz.me> Co-committed-by: M.V. Hutz <git@maximhutz.me>
This commit was merged in pull request #46.
This commit is contained in:
41
pkg/token/parse.go
Normal file
41
pkg/token/parse.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package token
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.maximhutz.com/max/lambda/pkg/iterator"
|
||||
)
|
||||
|
||||
// ParseRawToken consumes the next token from the iterator if its type matches
|
||||
// the expected type.
|
||||
// Uses [iterator.Try] for automatic backtracking on failure.
|
||||
func ParseRawToken[T Type](i *iterator.Iterator[Token[T]], expected T) (*Token[T], error) {
|
||||
return iterator.Try(i, func(i *iterator.Iterator[Token[T]]) (*Token[T], error) {
|
||||
if tok, err := i.Next(); err != nil {
|
||||
return nil, err
|
||||
} else if tok.Type != expected {
|
||||
return nil, fmt.Errorf("expected token %v, got %v'", expected.Name(), tok.Value)
|
||||
} else {
|
||||
return &tok, nil
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ParseList repeatedly applies a parse function, collecting results into a
|
||||
// slice.
|
||||
// Stops when the parse function returns an error.
|
||||
// Returns an error if fewer than minimum results are collected.
|
||||
func ParseList[T Type, U any](i *iterator.Iterator[Token[T]], fn func(*iterator.Iterator[Token[T]]) (U, error), minimum int) ([]U, error) {
|
||||
results := []U{}
|
||||
|
||||
for {
|
||||
if u, err := fn(i); err != nil {
|
||||
if len(results) < minimum {
|
||||
return nil, fmt.Errorf("expected at least '%v' items, got only '%v': %w", minimum, len(results), err)
|
||||
}
|
||||
return results, nil
|
||||
} else {
|
||||
results = append(results, u)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user