package tokenizer import ( "fmt" "unicode" "git.maximhutz.com/max/lambda/pkg/iterator" ) func getToken(i *iterator.Iterator[rune]) (*Token, error) { if i.IsDone() { return nil, nil } letter, err := i.Next() if err != nil { return nil, fmt.Errorf("Cannot produce next token: %w", err) } // If it is an operand. switch letter { case '(': return &Token{ Type: TokenOpenParen, Index: i.Index(), Value: string(letter), }, nil case ')': return &Token{ Type: TokenCloseParen, Index: i.Index(), Value: string(letter), }, nil case '.': return &Token{ Type: TokenDot, Index: i.Index(), Value: string(letter), }, nil case '\\': return &Token{ Type: TokenSlash, Index: i.Index(), Value: string(letter), }, nil } // If it is a space. if unicode.IsSpace(letter) { return nil, nil } // Otherwise, it is an atom. atom := string(letter) index := i.Index() for { if i.IsDone() { return &Token{ Index: index, Type: TokenVariable, Value: atom, }, nil } pop, err := i.Peek() if err != nil || unicode.IsSpace(pop) || unicode.IsPunct(pop) { return &Token{ Index: index, Type: TokenVariable, Value: atom, }, nil } i.Next() atom += string(pop) } } func GetTokens(input []rune) ([]Token, []error) { i := iterator.New(input) tokens := []Token{} errors := []error{} for !i.IsDone() { token, err := getToken(&i) if err != nil { errors = append(errors, err) } else if token != nil { tokens = append(tokens, *token) } } return tokens, errors }