package token import ( "errors" "fmt" "unicode" "git.maximhutz.com/max/lambda/pkg/iterator" ) // IsVariable determines whether a rune can be a valid variable character. func IsVariable(r rune) bool { return unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_' } // ScanRune consumes the next rune from the iterator if it satisfies the // predicate. // Returns an error if the iterator is exhausted or the rune does not match. func ScanRune(i *iterator.Iterator[rune], expected func(rune) bool) (rune, error) { return iterator.Try(i, func(i *iterator.Iterator[rune]) (rune, error) { if r, err := i.Next(); err != nil { return r, err } else if !expected(r) { return r, fmt.Errorf("got unexpected rune %v'", r) } else { return r, nil } }) } // ScanCharacter consumes the next rune from the iterator if it matches the // expected rune exactly. // Returns an error if the iterator is exhausted or the rune does not match. func ScanCharacter(i *iterator.Iterator[rune], expected rune) (rune, error) { return ScanRune(i, func(r rune) bool { return r == expected }) } // ScanAtom scans a contiguous sequence of variable characters into a single // atom token. // The first rune has already been consumed and is passed in. func ScanAtom[T Type](i *iterator.Iterator[rune], first rune, typ T, column int) *Token[T] { atom := []rune{first} for { if r, err := ScanRune(i, IsVariable); err != nil { break } else { atom = append(atom, r) } } return NewAtom(typ, string(atom), column) } // Scan tokenizes an input string using a language-specific scanToken function. // The scanToken function is called repeatedly until the input is exhausted. // It returns nil (no token, no error) for skippable input like whitespace. // Errors are accumulated and returned joined at the end. func Scan[T Type](input string, scanToken func(*iterator.Iterator[rune]) (*Token[T], error)) ([]Token[T], error) { i := iterator.Of([]rune(input)) tokens := []Token[T]{} errorList := []error{} for !i.Done() { token, err := scanToken(i) if err != nil { errorList = append(errorList, err) } else if token != nil { tokens = append(tokens, *token) } } return tokens, errors.Join(errorList...) }