diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 3ff9390..bc7af89 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -72,11 +72,11 @@ func ParseExpression(i *iterator.Iterator[tokenizer.Token]) (lambda.Expression, args = append(args, arg) } - close, closeErr := i.Next() - if closeErr != nil { - return nil, fmt.Errorf("could not parse call terminating parenthesis: %w", closeErr) - } else if close.Type != tokenizer.TokenCloseParen { - return nil, fmt.Errorf("expected call terminating parenthesis, got '%v' (column %v)", close.Value, close.Index) + closing, closingErr := i.Next() + if closingErr != nil { + return nil, fmt.Errorf("could not parse call terminating parenthesis: %w", closingErr) + } else if closing.Type != tokenizer.TokenCloseParen { + return nil, fmt.Errorf("expected call terminating parenthesis, got '%v' (column %v)", closing.Value, closing.Index) } // Construction. diff --git a/pkg/tokenizer/tokenizer.go b/pkg/tokenizer/tokenizer.go index 9161e57..bdd73f6 100644 --- a/pkg/tokenizer/tokenizer.go +++ b/pkg/tokenizer/tokenizer.go @@ -2,11 +2,16 @@ package tokenizer import ( "fmt" + "strings" "unicode" "git.maximhutz.com/max/lambda/pkg/iterator" ) +func isVariable(r rune) bool { + return unicode.IsLetter(r) || unicode.IsNumber(r) +} + func getToken(i *iterator.Iterator[rune]) (*Token, error) { if i.IsDone() { return nil, nil @@ -51,29 +56,25 @@ func getToken(i *iterator.Iterator[rune]) (*Token, error) { } // Otherwise, it is an atom. - atom := string(letter) + atom := strings.Builder{} index := i.Index() - for { - if i.IsDone() { - return &Token{ - Index: index, - Type: TokenVariable, - Value: atom, - }, nil - } - + for !i.IsDone() { pop, err := i.Peek() - if err != nil || unicode.IsSpace(pop) || unicode.IsPunct(pop) { - return &Token{ - Index: index, - Type: TokenVariable, - Value: atom, - }, nil + if err != nil || !isVariable(pop) { + break } - i.Next() - atom += string(pop) + atom.WriteRune(pop) + if _, err := i.Next(); err != nil { + break + } } + + return &Token{ + Index: index, + Type: TokenVariable, + Value: atom.String(), + }, nil } func GetTokens(input []rune) ([]Token, []error) {