lambda/pkg/saccharine/scan.go

package saccharine

import (
	"errors"
	"fmt"
	"unicode"

	"git.maximhutz.com/max/lambda/pkg/iterator"
)

// isVariables determines whether a rune can be a valid variable.
func isVariable(r rune) bool {
	return unicode.IsLetter(r) || unicode.IsNumber(r)
}

func scanRune(i *iterator.Iterator[rune], expected func(rune) bool) (rune, error) {
	i2 := i.Copy()

	if r, err := i2.Next(); err != nil {
		return r, err
	} else if !expected(r) {
		return r, fmt.Errorf("got unexpected rune %v'", r)
	} else {
		i.Sync(i2)
		return r, nil
	}
}

func scanCharacter(i *iterator.Iterator[rune], expected rune) (rune, error) {
	i2 := i.Copy()

	if r, err := i2.Next(); err != nil {
		return r, err
	} else if r != expected {
		return r, fmt.Errorf("got unexpected rune %v'", r)
	} else {
		i.Sync(i2)
		return r, nil
	}
}

// Pulls the next token from an iterator over runes. If it cannot, it will
// return nil. If an error occurs, it will return that.
func scanToken(i *iterator.Iterator[rune]) (*Token, error) {
	index := i.Index()

	if i.Done() {
		return nil, nil
	}

	letter, err := i.Next()
	if err != nil {
		return nil, fmt.Errorf("cannot produce next token: %w", err)
	}

	switch {
	case letter == '(':
		return NewToken(TokenOpenParen, index), nil
	case letter == ')':
		return NewToken(TokenCloseParen, index), nil
	case letter == '.':
		return NewToken(TokenDot, index), nil
	case letter == '\\':
		return NewToken(TokenSlash, index), nil
	case letter == '\n':
		return NewToken(TokenSoftBreak, index), nil
	case letter == '{':
		return NewToken(TokenOpenBrace, index), nil
	case letter == '}':
		return NewToken(TokenCloseBrace, index), nil
	case letter == ':':
		if _, err := scanCharacter(i, '='); err != nil {
			return nil, err
		} else {
			return NewToken(TokenAssign, index), nil
		}
	case letter == ';':
		return NewToken(TokenHardBreak, index), nil
	case letter == '#':
		// Skip everything until the next newline or EOF.
		for !i.Done() {
			r, err := i.Next()
			if err != nil {
				return nil, fmt.Errorf("error while parsing comment: %w", err)
			}

			if r == '\n' {
				// Put the newline back so it can be processed as a soft break.
				i.Back()
				break
			}
		}
		return nil, nil
	case unicode.IsSpace(letter):
		return nil, nil
	case isVariable(letter):
		atom := []rune{letter}

		for {
			if r, err := scanRune(i, isVariable); err != nil {
				break
			} else {
				atom = append(atom, r)
			}
		}

		return NewTokenAtom(string(atom), index), nil
	}

	return nil, fmt.Errorf("unknown character '%v'", string(letter))
}

// scan a string into tokens.
func scan(input string) ([]Token, error) {
	i := iterator.Of([]rune(input))
	tokens := []Token{}
	errorList := []error{}

	for !i.Done() {
		token, err := scanToken(i)
		if err != nil {
			errorList = append(errorList, err)
		} else if token != nil {
			tokens = append(tokens, *token)
		}
	}

	return tokens, errors.Join(errorList...)
}