docs: token

pkg/saccharine/token.go

@@ -2,90 +2,80 @@ package saccharine
 
 import "fmt"
 
-// All tokens in the pseudo-lambda language.
+// A TokenType is an identifier for any token in the Saccharine language.
 type TokenType int
 
+// All official tokens of the Saccharine language.
 const (
-	TokenOpenParen  TokenType = iota // Denotes the '(' token.
-	TokenCloseParen                  // Denotes the ')' token.
-	TokenOpenBrace                   // Denotes the '{' token.
-	TokenCloseBrace                  // Denotes the '}' token.
-	TokenHardBreak                   // Denotes the ';' token.
-	TokenAssign                      // Denotes the ':=' token.
-	TokenAtom                        // Denotes an alpha-numeric variable.
-	TokenSlash                       // Denotes the '/' token.
-	TokenDot                         // Denotes the '.' token.
-	TokenSoftBreak                   // Denotes a new-line.
+	// TokenOpenParen denotes the '(' token.
+	TokenOpenParen TokenType = iota
+	// TokenCloseParen denotes the ')' token.
+	TokenCloseParen
+	// TokenOpenBrace denotes the '{' token.
+	TokenOpenBrace
+	// TokenCloseBrace denotes the '}' token.
+	TokenCloseBrace
+	// TokenHardBreak denotes the ';' token.
+	TokenHardBreak
+	// TokenAssign denotes the ':=' token.
+	TokenAssign
+	// TokenAtom denotes an alpha-numeric variable.
+	TokenAtom
+	// TokenSlash denotes the '/' token.
+	TokenSlash
+	// TokenDot denotes the '.' token.
+	TokenDot
+	// TokenSoftBreak denotes a new-line.
+	TokenSoftBreak
 )
 
-// A representation of a token in source code.
+// A Token in the Saccharine language.
 type Token struct {
 	Column int       // Where the token begins in the source text.
 	Type   TokenType // What type the token is.
 	Value  string    // The value of the token.
 }
 
-func NewTokenOpenParen(column int) *Token {
-	return &Token{Type: TokenOpenParen, Column: column, Value: "("}
-}
-
-func NewTokenCloseParen(column int) *Token {
-	return &Token{Type: TokenCloseParen, Column: column, Value: ")"}
-}
-
-func NewTokenOpenBrace(column int) *Token {
-	return &Token{Type: TokenOpenBrace, Column: column, Value: "{"}
-}
-
-func NewTokenCloseBrace(column int) *Token {
-	return &Token{Type: TokenCloseBrace, Column: column, Value: "}"}
-}
-
-func NewTokenDot(column int) *Token {
-	return &Token{Type: TokenDot, Column: column, Value: "."}
-}
-
-func NewTokenHardBreak(column int) *Token {
-	return &Token{Type: TokenHardBreak, Column: column, Value: ";"}
-}
-
-func NewTokenAssign(column int) *Token {
-	return &Token{Type: TokenAssign, Column: column, Value: ":="}
-}
-
-func NewTokenSlash(column int) *Token {
-	return &Token{Type: TokenSlash, Column: column, Value: "\\"}
+// NewToken creates a [Token] of the given type at the given column.
+// The token's value is derived from its [TokenType].
+func NewToken(typ TokenType, column int) *Token {
+	return &Token{Type: typ, Column: column, Value: typ.Name()}
 }
 
+// NewTokenAtom creates a [TokenAtom] with the given name at the given column.
 func NewTokenAtom(name string, column int) *Token {
 	return &Token{Type: TokenAtom, Column: column, Value: name}
 }
 
-func NewTokenSoftBreak(column int) *Token {
-	return &Token{Type: TokenSoftBreak, Column: column, Value: "\\n"}
-}
-
+// Name returns the type of the TokenType, as a string.
 func (t TokenType) Name() string {
 	switch t {
 	case TokenOpenParen:
 		return "("
 	case TokenCloseParen:
 		return ")"
+	case TokenOpenBrace:
+		return "{"
+	case TokenCloseBrace:
+		return "}"
+	case TokenHardBreak:
+		return ";"
+	case TokenAssign:
+		return ":="
+	case TokenAtom:
+		return "ATOM"
 	case TokenSlash:
 		return "\\"
 	case TokenDot:
 		return "."
-	case TokenAtom:
-		return "ATOM"
 	case TokenSoftBreak:
 		return "\\n"
-	case TokenHardBreak:
-		return ";"
 	default:
 		panic(fmt.Errorf("unknown token type %v", t))
 	}
 }
 
+// Name returns the type of the Token, as a string.
 func (t Token) Name() string {
 	return t.Type.Name()
 }