feat: new parser strategy

pkg/token/token.go

@@ -0,0 +1,36 @@
+// Package token provides generic token types and scanning/parsing primitives
+// for building language-specific lexers and parsers.
+package token
+
+// A Type is a constraint for language-specific token type enums.
+// It must be comparable (for equality checks) and must have a Name method
+// that returns a human-readable string for error messages.
+type Type interface {
+	comparable
+	// Name returns a human-readable name for this token type.
+	Name() string
+}
+
+// A Token is a lexical unit in a source language.
+type Token[T Type] struct {
+	Column int    // Where the token begins in the source text.
+	Type   T      // What type the token is.
+	Value  string // The value of the token.
+}
+
+// New creates a Token of the given type at the given column.
+// The token's value is derived from its type's Name method.
+func New[T Type](typ T, column int) *Token[T] {
+	return &Token[T]{Type: typ, Column: column, Value: typ.Name()}
+}
+
+// NewAtom creates a Token of the given type with a custom value at the given
+// column.
+func NewAtom[T Type](typ T, name string, column int) *Token[T] {
+	return &Token[T]{Type: typ, Column: column, Value: name}
+}
+
+// Name returns the type of the Token, as a string.
+func (t Token[T]) Name() string {
+	return t.Type.Name()
+}