feat: tokenizer accepts braces, line terminator, and equal sign

.golangci.yml

@@ -25,7 +25,7 @@ formatters:
         # replace `interface{}` with `any` in the code on format
         - pattern: 'interface{}'
           replacement: 'any'
-
+    
     # make sure imports are always in a deterministic order
     # https://github.com/daixiang0/gci/
     gci:  # define the section orders for imports

pkg/convert/saccharine_to_lambda.go

@@ -1,18 +1,18 @@
 package convert
 
 import (
+	"fmt"
+
 	"git.maximhutz.com/max/lambda/pkg/lambda"
 	"git.maximhutz.com/max/lambda/pkg/saccharine/ast"
 )
 
-type compileVisitor struct{}
-
-func (v compileVisitor) VisitAtom(n *ast.Atom) lambda.Expression {
+func convertAtom(n *ast.Atom) lambda.Expression {
 	return lambda.NewVariable(n.Name)
 }
 
-func (v compileVisitor) VisitAbstraction(n *ast.Abstraction) lambda.Expression {
-	result := ast.Visit(v, n.Body)
+func convertAbstraction(n *ast.Abstraction) lambda.Expression {
+	result := SaccharineToLambda(n.Body)
 
 	parameters := n.Parameters
 
@@ -31,12 +31,12 @@ func (v compileVisitor) VisitAbstraction(n *ast.Abstraction) lambda.Expression {
 	return result
 }
 
-func (v compileVisitor) VisitApplication(n *ast.Application) lambda.Expression {
-	result := ast.Visit(v, n.Abstraction)
+func convertApplication(n *ast.Application) lambda.Expression {
+	result := SaccharineToLambda(n.Abstraction)
 
 	arguments := []lambda.Expression{}
 	for _, argument := range n.Arguments {
-		convertedArgument := ast.Visit(v, argument)
+		convertedArgument := SaccharineToLambda(argument)
 		arguments = append(arguments, convertedArgument)
 	}
 
@@ -48,5 +48,14 @@ func (v compileVisitor) VisitApplication(n *ast.Application) lambda.Expression {
 }
 
 func SaccharineToLambda(n ast.Expression) lambda.Expression {
-	return ast.Visit(&compileVisitor{}, n)
+	switch n := n.(type) {
+	case *ast.Atom:
+		return convertAtom(n)
+	case *ast.Abstraction:
+		return convertAbstraction(n)
+	case *ast.Application:
+		return convertApplication(n)
+	default:
+		panic(fmt.Errorf("unknown expression type: %v", n))
+	}
 }

pkg/lambda/stringify.go

@@ -26,7 +26,7 @@ func (v *stringifyVisitor) VisitApplication(c *Application) {
 }
 
 func Stringify(e Expression) string {
-	b := &stringifyVisitor{}
+	b := &stringifyVisitor{builder: strings.Builder{}}
 	e.Accept(b)
 	return b.builder.String()
 }

pkg/saccharine/ast/expression.go

@@ -20,9 +20,15 @@ type Atom struct {
 	Name string
 }
 
+type Clause struct {
+	Statements []Statement
+	Returns    Expression
+}
+
 func (Abstraction) IsExpression() {}
 func (Application) IsExpression() {}
 func (Atom) IsExpression()        {}
+func (Clause) IsExpression()      {}
 
 /** ------------------------------------------------------------------------- */
 
@@ -37,3 +43,7 @@ func NewApplication(abstraction Expression, arguments []Expression) *Application
 func NewAtom(name string) *Atom {
 	return &Atom{Name: name}
 }
+
+func NewClause(statements []Statement, returns Expression) *Clause {
+	return &Clause{Statements: statements, Returns: returns}
+}

pkg/saccharine/ast/program.go

@@ -1,5 +0,0 @@
-package ast
-
-type Program struct {
-	Statements []Statement
-}

pkg/saccharine/ast/statement.go

@@ -7,11 +7,6 @@ type Statement interface {
 /** ------------------------------------------------------------------------- */
 
 type LetStatement struct {
-	Variable string
-	Value    Expression
-}
-
-type MethodStatement struct {
 	Name       string
 	Parameters []string
 	Body       Expression
@@ -22,7 +17,6 @@ type DeclareStatement struct {
 }
 
 func (LetStatement) IsStatement()     {}
-func (MethodStatement) IsStatement()  {}
 func (DeclareStatement) IsStatement() {}
 
 /** ------------------------------------------------------------------------- */

pkg/saccharine/ast/visit.go

@@ -1,22 +0,0 @@
-package ast
-
-import "fmt"
-
-type Visitor[T any] interface {
-	VisitAtom(*Atom) T
-	VisitAbstraction(*Abstraction) T
-	VisitApplication(*Application) T
-}
-
-func Visit[T any](visitor Visitor[T], node Expression) T {
-	switch node := node.(type) {
-	case *Atom:
-		return visitor.VisitAtom(node)
-	case *Abstraction:
-		return visitor.VisitAbstraction(node)
-	case *Application:
-		return visitor.VisitApplication(node)
-	default:
-		panic(fmt.Sprintf("unknown node %t", node))
-	}
-}

pkg/saccharine/stringify.go

@@ -1,31 +1,39 @@
 package saccharine
 
 import (
+	"fmt"
 	"strings"
 
 	"git.maximhutz.com/max/lambda/pkg/saccharine/ast"
 )
 
-type stringifyVisitor struct{}
-
-func (v stringifyVisitor) VisitAtom(n *ast.Atom) string {
+func stringifyAtom(n *ast.Atom) string {
 	return n.Name
 }
 
-func (v stringifyVisitor) VisitAbstraction(n *ast.Abstraction) string {
-	return "\\" + strings.Join(n.Parameters, " ") + "." + ast.Visit(v, n.Body)
+func stringifyAbstraction(n *ast.Abstraction) string {
+	return "\\" + strings.Join(n.Parameters, " ") + "." + Stringify(n.Body)
 }
 
-func (v stringifyVisitor) VisitApplication(n *ast.Application) string {
-	arguments := []string{ast.Visit(v, n.Abstraction)}
+func stringifyApplication(n *ast.Application) string {
+	arguments := []string{Stringify(n.Abstraction)}
 
 	for _, argument := range n.Arguments {
-		arguments = append(arguments, ast.Visit(v, argument))
+		arguments = append(arguments, Stringify(argument))
 	}
 
 	return "(" + strings.Join(arguments, " ") + ")"
 }
 
 func Stringify(n ast.Expression) string {
-	return ast.Visit(&stringifyVisitor{}, n)
+	switch n := n.(type) {
+	case *ast.Atom:
+		return stringifyAtom(n)
+	case *ast.Abstraction:
+		return stringifyAbstraction(n)
+	case *ast.Application:
+		return stringifyApplication(n)
+	default:
+		panic(fmt.Errorf("unknown expression type: %v", n))
+	}
 }

pkg/saccharine/token/token.go

@@ -6,6 +6,10 @@ type Type int
 const (
 	OpenParen  Type = iota // Denotes the '(' token.
 	CloseParen             // Denotes the ')' token.
+	OpenBrace              // Denotes the '{' token.
+	CloseBrace             // Denotes the '}' token.
+	End                    // Denotes the ';' token.
+	Assign                 // Denotes the ':=' token.
 	Atom                   // Denotes an alpha-numeric variable.
 	Slash                  // Denotes the '/' token.
 	Dot                    // Denotes the '.' token.
@@ -27,10 +31,26 @@ func NewCloseParen(column int) *Token {
 	return &Token{Type: CloseParen, Column: column, Value: ")"}
 }
 
+func NewOpenBrace(column int) *Token {
+	return &Token{Type: OpenBrace, Column: column, Value: "{"}
+}
+
+func NewCloseBrace(column int) *Token {
+	return &Token{Type: CloseBrace, Column: column, Value: "}"}
+}
+
 func NewDot(column int) *Token {
 	return &Token{Type: Dot, Column: column, Value: "."}
 }
 
+func NewEnd(column int) *Token {
+	return &Token{Type: End, Column: column, Value: ";"}
+}
+
+func NewAssign(column int) *Token {
+	return &Token{Type: Assign, Column: column, Value: ":="}
+}
+
 func NewSlash(column int) *Token {
 	return &Token{Type: Slash, Column: column, Value: "\\"}
 }

pkg/saccharine/tokenizer.go

@@ -28,6 +28,19 @@ func parseRune(i *iterator.Iterator[rune], expected func(rune) bool) (rune, erro
 	}
 }
 
+func parseCharacter(i *iterator.Iterator[rune], expected rune) (rune, error) {
+	i2 := i.Copy()
+
+	if r, err := i2.Next(); err != nil {
+		return r, err
+	} else if r != expected {
+		return r, fmt.Errorf("got unexpected rune %v'", r)
+	} else {
+		i.Sync(i2)
+		return r, nil
+	}
+}
+
 // Pulls the next token from an iterator over runes. If it cannot, it will
 // return nil. If an error occurs, it will return that.
 func getToken(i *iterator.Iterator[rune]) (*token.Token, error) {
@@ -53,6 +66,18 @@ func getToken(i *iterator.Iterator[rune]) (*token.Token, error) {
 		return token.NewSlash(index), nil
 	case letter == '\n':
 		return token.NewNewline(index), nil
+	case letter == '{':
+		return token.NewNewline(index), nil
+	case letter == '}':
+		return token.NewNewline(index), nil
+	case letter == ':':
+		if _, err := parseCharacter(i, '='); err != nil {
+			return nil, err
+		} else {
+			return token.NewAssign(index), nil
+		}
+	case letter == ';':
+		return token.NewEnd(index), nil
 	case unicode.IsSpace(letter):
 		return nil, nil
 	case isVariable(letter):

samples/saccharine.txt

@@ -1,9 +1,16 @@
-0 := \f x.x
-(inc n) := \f x.(f (n f x))
-(add n m) := (m inc n)
-(mult n m) := (m (n f))
-(exp m n) := (m n)
+(true x y) := x
+(false x y) := y
 
-# This is the final output.
-5 := (inc (inc (inc (inc (inc 0)))))
-(exp 5 5)
+(pair a b) := \c.(c a b)
+(left p) := (p true)
+(false p) := (p false)
+
+zero 0 1 x = x
+
+inc n := \0 1 x.{
+  initial := (pair true x)
+  (on_zero p) := (pair false ((left p 1 0) (right p)))
+  (on_one p) := (pair (left p) (1 (right p)))
+  
+  (n on_zero on_one initial)
+}