// Package token provides generic token types and scanning/parsing primitives // for building language-specific lexers and parsers. package token // A Type is a constraint for language-specific token type enums. // It must be comparable (for equality checks) and must have a Name method // that returns a human-readable string for error messages. type Type interface { comparable // Name returns a human-readable name for this token type. Name() string } // A Token is a lexical unit in a source language. type Token[T Type] struct { Column int // Where the token begins in the source text. Type T // What type the token is. Value string // The value of the token. } // New creates a Token of the given type at the given column. // The token's value is derived from its type's Name method. func New[T Type](typ T, column int) *Token[T] { return &Token[T]{Type: typ, Column: column, Value: typ.Name()} } // NewAtom creates a Token of the given type with a custom value at the given // column. func NewAtom[T Type](typ T, name string, column int) *Token[T] { return &Token[T]{Type: typ, Column: column, Value: name} } // Name returns the type of the Token, as a string. func (t Token[T]) Name() string { return t.Type.Name() }