feat: parser

pkg/tokenizer/tokenizer.go

@@ -21,25 +21,25 @@ func getToken(i *iterator.Iterator[rune]) (*Token, error) {
 	switch letter {
 	case '(':
 		return &Token{
-			Type: TokenOpenParen,
+			Type:  TokenOpenParen,
 			Index: i.Index(),
 			Value: string(letter),
 		}, nil
 	case ')':
 		return &Token{
-			Type: TokenCloseParen,
+			Type:  TokenCloseParen,
 			Index: i.Index(),
 			Value: string(letter),
 		}, nil
 	case '.':
 		return &Token{
-			Type: TokenDot,
+			Type:  TokenDot,
 			Index: i.Index(),
 			Value: string(letter),
 		}, nil
 	case '\\':
 		return &Token{
-			Type: TokenSlash,
+			Type:  TokenSlash,
 			Index: i.Index(),
 			Value: string(letter),
 		}, nil
@@ -58,19 +58,16 @@ func getToken(i *iterator.Iterator[rune]) (*Token, error) {
 			return nil, nil
 		}
 
-		pop, err := i.Next()
-		if err != nil {
-			return nil, err
-		}
-
-		if unicode.IsSpace(pop) {
+		pop, err := i.Peek()
+		if err != nil || unicode.IsSpace(pop) || unicode.IsPunct(pop) {
 			return &Token{
 				Index: index,
-				Type: TokenAtom,
+				Type:  TokenAtom,
 				Value: atom,
 			}, nil
 		}
-		
+
+		i.Next()
 		atom += string(pop)
 	}
 }
@@ -90,4 +87,4 @@ func GetTokens(input []rune) ([]Token, []error) {
 	}
 
 	return tokens, errors
-}
+}