feat: scanner added

pkg/token/token.go

@@ -18,18 +18,6 @@ type Token[T Type] struct {
 	Value  string // The value of the token.
 }
 
-// New creates a Token of the given type at the given column.
-// The token's value is derived from its type's Name method.
-func New[T Type](typ T, column int) *Token[T] {
-	return &Token[T]{Type: typ, Column: column, Value: typ.Name()}
-}
-
-// NewAtom creates a Token of the given type with a custom value at the given
-// column.
-func NewAtom[T Type](typ T, name string, column int) *Token[T] {
-	return &Token[T]{Type: typ, Column: column, Value: name}
-}
-
 // Name returns the type of the Token, as a string.
 func (t Token[T]) Name() string {
 	return t.Type.Name()