96 lines
1.6 KiB
Go
96 lines
1.6 KiB
Go
package tokenizer
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"unicode"
|
|
|
|
"git.maximhutz.com/max/lambda/pkg/iterator"
|
|
)
|
|
|
|
func isVariable(r rune) bool {
|
|
return unicode.IsLetter(r) || unicode.IsNumber(r)
|
|
}
|
|
|
|
func getToken(i *iterator.Iterator[rune]) (*Token, error) {
|
|
if i.IsDone() {
|
|
return nil, nil
|
|
}
|
|
|
|
letter, err := i.Next()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot produce next token: %w", err)
|
|
}
|
|
|
|
// If it is an operand.
|
|
switch letter {
|
|
case '(':
|
|
return &Token{
|
|
Type: TokenOpenParen,
|
|
Index: i.Index(),
|
|
Value: string(letter),
|
|
}, nil
|
|
case ')':
|
|
return &Token{
|
|
Type: TokenCloseParen,
|
|
Index: i.Index(),
|
|
Value: string(letter),
|
|
}, nil
|
|
case '.':
|
|
return &Token{
|
|
Type: TokenDot,
|
|
Index: i.Index(),
|
|
Value: string(letter),
|
|
}, nil
|
|
case '\\':
|
|
return &Token{
|
|
Type: TokenSlash,
|
|
Index: i.Index(),
|
|
Value: string(letter),
|
|
}, nil
|
|
}
|
|
|
|
// If it is a space.
|
|
if unicode.IsSpace(letter) {
|
|
return nil, nil
|
|
}
|
|
|
|
// Otherwise, it is an atom.
|
|
atom := strings.Builder{}
|
|
index := i.Index()
|
|
for !i.IsDone() {
|
|
pop, err := i.Peek()
|
|
if err != nil || !isVariable(pop) {
|
|
break
|
|
}
|
|
|
|
atom.WriteRune(pop)
|
|
if _, err := i.Next(); err != nil {
|
|
break
|
|
}
|
|
}
|
|
|
|
return &Token{
|
|
Index: index,
|
|
Type: TokenVariable,
|
|
Value: atom.String(),
|
|
}, nil
|
|
}
|
|
|
|
func GetTokens(input []rune) ([]Token, []error) {
|
|
i := iterator.New(input)
|
|
tokens := []Token{}
|
|
errors := []error{}
|
|
|
|
for !i.IsDone() {
|
|
token, err := getToken(&i)
|
|
if err != nil {
|
|
errors = append(errors, err)
|
|
} else if token != nil {
|
|
tokens = append(tokens, *token)
|
|
}
|
|
}
|
|
|
|
return tokens, errors
|
|
}
|