feat: better recursive descent

pkg/saccharine/token/token.go

@@ -0,0 +1,47 @@
+package token
+
+// All tokens in the pseudo-lambda language.
+type Type int
+
+const (
+	// Denotes the '(' token.
+	OpenParen Type = iota
+	// Denotes the ')' token.
+	CloseParen
+	// Denotes an alpha-numeric variable.
+	Atom
+	// Denotes the '/' token.
+	Slash
+	// Denotes the '.' token.
+	Dot
+)
+
+// A representation of a token in source code.
+type Token struct {
+	// Where the token begins in the source text.
+	Index int
+	// What type the token is.
+	Type Type
+	// The value of the token.
+	Value string
+}
+
+func NewOpenParen(index int) *Token {
+	return &Token{Type: OpenParen, Index: index, Value: "("}
+}
+
+func NewCloseParen(index int) *Token {
+	return &Token{Type: CloseParen, Index: index, Value: ")"}
+}
+
+func NewDot(index int) *Token {
+	return &Token{Type: Dot, Index: index, Value: "."}
+}
+
+func NewSlash(index int) *Token {
+	return &Token{Type: Slash, Index: index, Value: "\\"}
+}
+
+func NewAtom(name string, index int) *Token {
+	return &Token{Type: Atom, Index: index, Value: name}
+}