From d24cbd9d860084dea6364e44b75490af4d80794b Mon Sep 17 00:00:00 2001
From: "M.V. Hutz" <git@maximhutz.me>
Date: Mon, 9 Feb 2026 18:40:35 -0500
Subject: [PATCH] docs: token

---
 pkg/saccharine/scan.go  | 18 ++++-----
 pkg/saccharine/token.go | 90 ++++++++++++++++++-----------------------
 2 files changed, 49 insertions(+), 59 deletions(-)

diff --git a/pkg/saccharine/scan.go b/pkg/saccharine/scan.go
index e190b9d..762900b 100644
--- a/pkg/saccharine/scan.go
+++ b/pkg/saccharine/scan.go
@@ -55,27 +55,27 @@ func scanToken(i *iterator.Iterator[rune]) (*Token, error) {
 
 	switch {
 	case letter == '(':
-		return NewTokenOpenParen(index), nil
+		return NewToken(TokenOpenParen, index), nil
 	case letter == ')':
-		return NewTokenCloseParen(index), nil
+		return NewToken(TokenCloseParen, index), nil
 	case letter == '.':
-		return NewTokenDot(index), nil
+		return NewToken(TokenDot, index), nil
 	case letter == '\\':
-		return NewTokenSlash(index), nil
+		return NewToken(TokenSlash, index), nil
 	case letter == '\n':
-		return NewTokenSoftBreak(index), nil
+		return NewToken(TokenSoftBreak, index), nil
 	case letter == '{':
-		return NewTokenOpenBrace(index), nil
+		return NewToken(TokenOpenBrace, index), nil
 	case letter == '}':
-		return NewTokenCloseBrace(index), nil
+		return NewToken(TokenCloseBrace, index), nil
 	case letter == ':':
 		if _, err := scanCharacter(i, '='); err != nil {
 			return nil, err
 		} else {
-			return NewTokenAssign(index), nil
+			return NewToken(TokenAssign, index), nil
 		}
 	case letter == ';':
-		return NewTokenHardBreak(index), nil
+		return NewToken(TokenHardBreak, index), nil
 	case letter == '#':
 		// Skip everything until the next newline or EOF.
 		for !i.Done() {
diff --git a/pkg/saccharine/token.go b/pkg/saccharine/token.go
index ee61dbf..c7a1ac6 100644
--- a/pkg/saccharine/token.go
+++ b/pkg/saccharine/token.go
@@ -2,90 +2,80 @@ package saccharine
 
 import "fmt"
 
-// All tokens in the pseudo-lambda language.
+// A TokenType is an identifier for any token in the Saccharine language.
 type TokenType int
 
+// All official tokens of the Saccharine language.
 const (
-	TokenOpenParen  TokenType = iota // Denotes the '(' token.
-	TokenCloseParen                  // Denotes the ')' token.
-	TokenOpenBrace                   // Denotes the '{' token.
-	TokenCloseBrace                  // Denotes the '}' token.
-	TokenHardBreak                   // Denotes the ';' token.
-	TokenAssign                      // Denotes the ':=' token.
-	TokenAtom                        // Denotes an alpha-numeric variable.
-	TokenSlash                       // Denotes the '/' token.
-	TokenDot                         // Denotes the '.' token.
-	TokenSoftBreak                   // Denotes a new-line.
+	// TokenOpenParen denotes the '(' token.
+	TokenOpenParen TokenType = iota
+	// TokenCloseParen denotes the ')' token.
+	TokenCloseParen
+	// TokenOpenBrace denotes the '{' token.
+	TokenOpenBrace
+	// TokenCloseBrace denotes the '}' token.
+	TokenCloseBrace
+	// TokenHardBreak denotes the ';' token.
+	TokenHardBreak
+	// TokenAssign denotes the ':=' token.
+	TokenAssign
+	// TokenAtom denotes an alpha-numeric variable.
+	TokenAtom
+	// TokenSlash denotes the '/' token.
+	TokenSlash
+	// TokenDot denotes the '.' token.
+	TokenDot
+	// TokenSoftBreak denotes a new-line.
+	TokenSoftBreak
 )
 
-// A representation of a token in source code.
+// A Token in the Saccharine language.
 type Token struct {
 	Column int       // Where the token begins in the source text.
 	Type   TokenType // What type the token is.
 	Value  string    // The value of the token.
 }
 
-func NewTokenOpenParen(column int) *Token {
-	return &Token{Type: TokenOpenParen, Column: column, Value: "("}
-}
-
-func NewTokenCloseParen(column int) *Token {
-	return &Token{Type: TokenCloseParen, Column: column, Value: ")"}
-}
-
-func NewTokenOpenBrace(column int) *Token {
-	return &Token{Type: TokenOpenBrace, Column: column, Value: "{"}
-}
-
-func NewTokenCloseBrace(column int) *Token {
-	return &Token{Type: TokenCloseBrace, Column: column, Value: "}"}
-}
-
-func NewTokenDot(column int) *Token {
-	return &Token{Type: TokenDot, Column: column, Value: "."}
-}
-
-func NewTokenHardBreak(column int) *Token {
-	return &Token{Type: TokenHardBreak, Column: column, Value: ";"}
-}
-
-func NewTokenAssign(column int) *Token {
-	return &Token{Type: TokenAssign, Column: column, Value: ":="}
-}
-
-func NewTokenSlash(column int) *Token {
-	return &Token{Type: TokenSlash, Column: column, Value: "\\"}
+// NewToken creates a [Token] of the given type at the given column.
+// The token's value is derived from its [TokenType].
+func NewToken(typ TokenType, column int) *Token {
+	return &Token{Type: typ, Column: column, Value: typ.Name()}
 }
 
+// NewTokenAtom creates a [TokenAtom] with the given name at the given column.
 func NewTokenAtom(name string, column int) *Token {
 	return &Token{Type: TokenAtom, Column: column, Value: name}
 }
 
-func NewTokenSoftBreak(column int) *Token {
-	return &Token{Type: TokenSoftBreak, Column: column, Value: "\\n"}
-}
-
+// Name returns the type of the TokenType, as a string.
 func (t TokenType) Name() string {
 	switch t {
 	case TokenOpenParen:
 		return "("
 	case TokenCloseParen:
 		return ")"
+	case TokenOpenBrace:
+		return "{"
+	case TokenCloseBrace:
+		return "}"
+	case TokenHardBreak:
+		return ";"
+	case TokenAssign:
+		return ":="
+	case TokenAtom:
+		return "ATOM"
 	case TokenSlash:
 		return "\\"
 	case TokenDot:
 		return "."
-	case TokenAtom:
-		return "ATOM"
 	case TokenSoftBreak:
 		return "\\n"
-	case TokenHardBreak:
-		return ";"
 	default:
 		panic(fmt.Errorf("unknown token type %v", t))
 	}
 }
 
+// Name returns the type of the Token, as a string.
 func (t Token) Name() string {
 	return t.Type.Name()
 }