feat: stuff

pkg/tokenizer/token.go

@@ -1,17 +1,27 @@
 package tokenizer
 
+// All tokens in the pseudo-lambda language.
 type TokenType int
 
 const (
+	// Denotes the '(' token.
 	TokenOpenParen TokenType = iota
+	// Denotes the ')' token.
 	TokenCloseParen
+	// Denotes an alpha-numeric variable.
 	TokenVariable
+	// Denotes the '/' token.
 	TokenSlash
+	// Denotes the '.' token.
 	TokenDot
 )
 
+// A representation of a token in source code.
 type Token struct {
+	// Where the token begins in the source text.
 	Index int
-	Type  TokenType
+	// What type the token is.
+	Type TokenType
+	// The value of the token.
 	Value string
 }