mvhutz/lambda
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: extract shared token package (#46)

Browse Source 
## Description

Both the `saccharine` and `lambda` packages need tokenizing and parsing primitives.
This PR extracts shared token infrastructure into a new `pkg/token` package, then wires both languages up to use it.

- Add `pkg/token` with a generic `Token[T]` type, `Scan`, `ScanAtom`, `ScanRune`, `ScanCharacter`, `IsVariable`, `ParseRawToken`, and `ParseList`.
- Refactor `pkg/saccharine` to delegate to `pkg/token`, removing duplicated scanning and parsing helpers.
- Implement `Codec.Decode` for `pkg/lambda` (scanner + parser) using the shared token package.
- Add `iterator.While` for predicate-driven iteration.
- Rename `iterator.Do` to `iterator.Try` to better describe its rollback semantics.

### Decisions

- The `Type` constraint (`comparable` + `Name() string`) keeps the generic token flexible while ensuring every token type can produce readable error messages.
- `iterator.Do` was renamed to `iterator.Try` since it describes a try/rollback operation, not a side-effecting "do".

## Benefits

- Eliminates duplicated token, scanning, and parsing code between languages.
- Enables the `lambda` package to decode (parse) lambda calculus strings, which was previously unimplemented.
- Makes it straightforward to add new languages by reusing `pkg/token` primitives.

## Checklist

- [x] Code follows conventional commit format.
- [x] Branch follows naming convention (`<type>/<description>`). Always use underscores.
- [x] Tests pass (if applicable).
- [ ] Documentation updated (if applicable).

Reviewed-on: #46
Co-authored-by: M.V. Hutz <git@maximhutz.me>
Co-committed-by: M.V. Hutz <git@maximhutz.me>
This commit was merged in pull request #46.
This commit is contained in:
Maxim Hutz
2026-02-12 00:25:18 +00:00
committed by Maxim Hutz
parent 361f529bdc
commit da3da70855
11 changed files with 392 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
pkg/saccharine/parse.go
View File

@@ -5,37 +5,26 @@ import (
"fmt"
"git.maximhutz.com/max/lambda/pkg/iterator"
"git.maximhutz.com/max/lambda/pkg/token"
)
type tokenIterator = iterator.Iterator[Token]
func parseRawToken(i *tokenIterator, expected TokenType) (*Token, error) {
return iterator.Do(i, func(i *tokenIterator) (*Token, error) {
if tok, err := i.Next(); err != nil {
return nil, err
} else if tok.Type != expected {
return nil, fmt.Errorf("expected token %v, got %v'", expected.Name(), tok.Value)
} else {
return &tok, nil
}
})
}
func passSoftBreaks(i *tokenIterator) {
for {
if _, err := parseRawToken(i, TokenSoftBreak); err != nil {
if _, err := token.ParseRawToken(i, TokenSoftBreak); err != nil {
return
}
}
}
func parseToken(i *tokenIterator, expected TokenType, ignoreSoftBreaks bool) (*Token, error) {
return iterator.Do(i, func(i *tokenIterator) (*Token, error) {
return iterator.Try(i, func(i *tokenIterator) (*Token, error) {
if ignoreSoftBreaks {
passSoftBreaks(i)
}
return parseRawToken(i, expected)
return token.ParseRawToken(i, expected)
})
}
@@ -48,35 +37,20 @@ func parseString(i *tokenIterator) (string, error) {
}
func parseBreak(i *tokenIterator) (*Token, error) {
if tok, softErr := parseRawToken(i, TokenSoftBreak); softErr == nil {
if tok, softErr := token.ParseRawToken(i, TokenSoftBreak); softErr == nil {
return tok, nil
} else if tok, hardErr := parseRawToken(i, TokenHardBreak); hardErr == nil {
} else if tok, hardErr := token.ParseRawToken(i, TokenHardBreak); hardErr == nil {
return tok, nil
} else {
return nil, errors.Join(softErr, hardErr)
}
}
func parseList[U any](i *tokenIterator, fn func(*tokenIterator) (U, error), minimum int) ([]U, error) {
results := []U{}
for {
if u, err := fn(i); err != nil {
if len(results) < minimum {
return nil, fmt.Errorf("expected at least '%v' items, got only '%v': %w", minimum, len(results), err)
}
return results, nil
} else {
results = append(results, u)
}
}
}
func parseAbstraction(i *tokenIterator) (*Abstraction, error) {
return iterator.Do(i, func(i *tokenIterator) (*Abstraction, error) {
return iterator.Try(i, func(i *tokenIterator) (*Abstraction, error) {
if _, err := parseToken(i, TokenSlash, true); err != nil {
return nil, fmt.Errorf("no function slash (col %d): %w", i.MustGet().Column, err)
} else if parameters, err := parseList(i, parseString, 0); err != nil {
} else if parameters, err := token.ParseList(i, parseString, 0); err != nil {
return nil, err
} else if _, err = parseToken(i, TokenDot, true); err != nil {
return nil, fmt.Errorf("no function dot (col %d): %w", i.MustGet().Column, err)
@@ -89,10 +63,10 @@ func parseAbstraction(i *tokenIterator) (*Abstraction, error) {
}
func parseApplication(i *tokenIterator) (*Application, error) {
return iterator.Do(i, func(i *tokenIterator) (*Application, error) {
return iterator.Try(i, func(i *tokenIterator) (*Application, error) {
if _, err := parseToken(i, TokenOpenParen, true); err != nil {
return nil, fmt.Errorf("no openning brackets (col %d): %w", i.MustGet().Column, err)
} else if expressions, err := parseList(i, parseExpression, 1); err != nil {
} else if expressions, err := token.ParseList(i, parseExpression, 1); err != nil {
return nil, err
} else if _, err := parseToken(i, TokenCloseParen, true); err != nil {
return nil, fmt.Errorf("no closing brackets (col %d): %w", i.MustGet().Column, err)
@@ -114,12 +88,12 @@ func parseStatements(i *tokenIterator) ([]Statement, error) {
statements := []Statement{}
//nolint:errcheck
parseList(i, parseBreak, 0)
token.ParseList(i, parseBreak, 0)
for {
if statement, err := parseStatement(i); err != nil {
break
} else if _, err := parseList(i, parseBreak, 1); err != nil && !i.Done() {
} else if _, err := token.ParseList(i, parseBreak, 1); err != nil && !i.Done() {
break
} else {
statements = append(statements, statement)
@@ -159,7 +133,7 @@ func parseClause(i *tokenIterator, braces bool) (*Clause, error) {
}
func parseExpression(i *tokenIterator) (Expression, error) {
return iterator.Do(i, func(i *tokenIterator) (Expression, error) {
return iterator.Try(i, func(i *tokenIterator) (Expression, error) {
passSoftBreaks(i)
switch peek := i.MustGet(); peek.Type {
@@ -178,8 +152,8 @@ func parseExpression(i *tokenIterator) (Expression, error) {
}
func parseLet(i *tokenIterator) (*LetStatement, error) {
return iterator.Do(i, func(i *tokenIterator) (*LetStatement, error) {
if parameters, err := parseList(i, parseString, 1); err != nil {
return iterator.Try(i, func(i *tokenIterator) (*LetStatement, error) {
if parameters, err := token.ParseList(i, parseString, 1); err != nil {
return nil, err
} else if _, err := parseToken(i, TokenAssign, true); err != nil {
return nil, err