37 lines
1.2 KiB
Go
37 lines
1.2 KiB
Go
// Package token provides generic token types and scanning/parsing primitives
|
|
// for building language-specific lexers and parsers.
|
|
package token
|
|
|
|
// A Type is a constraint for language-specific token type enums.
|
|
// It must be comparable (for equality checks) and must have a Name method
|
|
// that returns a human-readable string for error messages.
|
|
type Type interface {
|
|
comparable
|
|
// Name returns a human-readable name for this token type.
|
|
Name() string
|
|
}
|
|
|
|
// A Token is a lexical unit in a source language.
|
|
type Token[T Type] struct {
|
|
Column int // Where the token begins in the source text.
|
|
Type T // What type the token is.
|
|
Value string // The value of the token.
|
|
}
|
|
|
|
// New creates a Token of the given type at the given column.
|
|
// The token's value is derived from its type's Name method.
|
|
func New[T Type](typ T, column int) *Token[T] {
|
|
return &Token[T]{Type: typ, Column: column, Value: typ.Name()}
|
|
}
|
|
|
|
// NewAtom creates a Token of the given type with a custom value at the given
|
|
// column.
|
|
func NewAtom[T Type](typ T, name string, column int) *Token[T] {
|
|
return &Token[T]{Type: typ, Column: column, Value: name}
|
|
}
|
|
|
|
// Name returns the type of the Token, as a string.
|
|
func (t Token[T]) Name() string {
|
|
return t.Type.Name()
|
|
}
|