mr/elysium
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix expression statements, Add modules (WIP), Support compilation of multiple files, Add stdlib

Browse Source
This commit is contained in:
MrLetsplay 2024-03-29 15:12:37 +01:00
parent 99372dbc6a
commit 77d3bc638d
Signed by: mr
SSH Key Fingerprint: SHA256:92jBH80vpXyaZHjaIl47pjRq+Yt7XGTArqQg1V7hSqg
6 changed files with 184 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
backend_wat.go
View File

@ -3,6 +3,7 @@ package main
import ( import (
"errors" "errors"
"strconv" "strconv"
"strings"
"unicode" "unicode"
) )
@ -319,7 +320,12 @@ func compileStatementWAT(stmt Statement, block *Block) (string, error) {
return "", err return "", err
} }
return wat + "drop\n", nil numItems := 1
if expr.Expression.ValueType.Type == Type_Tuple {
numItems = len(expr.Expression.ValueType.Value.(TupleType).Types)
}
return wat + strings.Repeat("drop\n", numItems), nil
case Statement_Block: case Statement_Block:
block := stmt.Value.(BlockStatement) block := stmt.Value.(BlockStatement)
wat, err := compileBlockWAT(block.Block) wat, err := compileBlockWAT(block.Block)
@ -426,7 +432,7 @@ func compileBlockWAT(block *Block) (string, error) {
} }
func compileFunctionWAT(function ParsedFunction) (string, error) { func compileFunctionWAT(function ParsedFunction) (string, error) {
funcWAT := "(func $" + safeASCIIIdentifier(function.Name) + "\n" funcWAT := "(func $" + safeASCIIIdentifier(function.FullName) + "\n"
for _, local := range function.Locals { for _, local := range function.Locals {
if !local.IsParameter { if !local.IsParameter {
@ -462,19 +468,21 @@ func compileFunctionWAT(function ParsedFunction) (string, error) {
funcWAT += wat funcWAT += wat
return funcWAT + ") (export \"" + function.Name + "\" (func $" + safeASCIIIdentifier(function.Name) + "))\n", nil return funcWAT + ") (export \"" + function.FullName + "\" (func $" + safeASCIIIdentifier(function.FullName) + "))\n", nil
} }
func backendWAT(file ParsedFile) (string, error) { func backendWAT(files []*ParsedFile) (string, error) {
module := "(module (memory 1)\n" module := "(module (memory 1)\n"
for _, function := range file.Functions { for _, file := range files {
wat, err := compileFunctionWAT(function) for _, function := range file.Functions {
if err != nil { wat, err := compileFunctionWAT(function)
return "", err if err != nil {
} return "", err
}
module += wat module += wat
}
} }
module += ")" module += ")"

34
lexer.go
View File

@ -22,10 +22,11 @@ const (
type Keyword uint32 type Keyword uint32
var Keywords []string = []string{"import", "void", "return", "true", "false", "if", "else"} var Keywords []string = []string{"import", "module", "void", "return", "true", "false", "if", "else"}
const ( const (
Keyword_Import Keyword = iota Keyword_Import Keyword = iota
Keyword_Module
Keyword_Void Keyword_Void
Keyword_Return Keyword_Return
Keyword_True Keyword_True
@ -36,7 +37,7 @@ const (
type Separator uint32 type Separator uint32
var Separators []rune = []rune{'(', ')', '{', '}', '[', ']', ';', ','} var Separators []rune = []rune{'(', ')', '{', '}', '[', ']', ';', ',', '.'}
const ( const (
Separator_OpenParen Separator = iota Separator_OpenParen Separator = iota
@ -47,6 +48,7 @@ const (
Separator_CloseSquare Separator_CloseSquare
Separator_Semicolon Separator_Semicolon
Separator_Comma Separator_Comma
Separator_Dot
) )
type Operator uint32 type Operator uint32
@ -84,10 +86,15 @@ const (
type LexToken struct { type LexToken struct {
Type LexType Type LexType
Position uint64 Position TokenPosition
Value any Value any
} }
type TokenPosition struct {
SourceFile string
Position uint64
}
type Literal struct { type Literal struct {
Type LiteralType Type LiteralType
Primitive PrimitiveType Primitive PrimitiveType
@ -97,11 +104,12 @@ type Literal struct {
type Lexer struct { type Lexer struct {
Runes []rune Runes []rune
LastTokenPosition uint64 LastTokenPosition uint64
SourceFile string
Position uint64 Position uint64
} }
func (l *Lexer) error(message string) error { func (l *Lexer) error(message string) error {
return CompilerError{Position: l.LastTokenPosition, Message: message} return CompilerError{Position: TokenPosition{SourceFile: l.SourceFile, Position: l.LastTokenPosition}, Message: message}
} }
func (l *Lexer) peekRune() *rune { func (l *Lexer) peekRune() *rune {
@ -210,12 +218,12 @@ func (l *Lexer) nextToken() (*LexToken, error) {
return nil, err return nil, err
} }
return &LexToken{Type: Type_Literal, Position: l.LastTokenPosition, Value: Literal{Type: Literal_String, Primitive: InvalidValue, Value: literal}}, nil return &LexToken{Type: Type_Literal, Position: TokenPosition{SourceFile: l.SourceFile, Position: l.LastTokenPosition}, Value: Literal{Type: Literal_String, Primitive: InvalidValue, Value: literal}}, nil
} }
op := l.tryOperator() op := l.tryOperator()
if op != InvalidValue { if op != InvalidValue {
return &LexToken{Type: Type_Operator, Position: l.LastTokenPosition, Value: op}, nil return &LexToken{Type: Type_Operator, Position: TokenPosition{SourceFile: l.SourceFile, Position: l.LastTokenPosition}, Value: op}, nil
} }
token := "" token := ""
@ -238,8 +246,6 @@ func (l *Lexer) nextToken() (*LexToken, error) {
runes := []rune(token) runes := []rune(token)
if unicode.IsDigit([]rune(token)[0]) { if unicode.IsDigit([]rune(token)[0]) {
// TODO: hexadecimal/binary/octal constants
var numberType PrimitiveType = InvalidValue var numberType PrimitiveType = InvalidValue
var rawNumber string = token var rawNumber string = token
for i, name := range PRIMITIVE_TYPE_NAMES { for i, name := range PRIMITIVE_TYPE_NAMES {
@ -268,20 +274,20 @@ func (l *Lexer) nextToken() (*LexToken, error) {
return nil, err return nil, err
} }
return &LexToken{Type: Type_Literal, Position: l.LastTokenPosition, Value: Literal{Type: Literal_Number, Primitive: numberType, Value: number}}, nil return &LexToken{Type: Type_Literal, Position: TokenPosition{SourceFile: l.SourceFile, Position: l.LastTokenPosition}, Value: Literal{Type: Literal_Number, Primitive: numberType, Value: number}}, nil
} }
if len(runes) == 1 { if len(runes) == 1 {
if idx := slices.Index(Separators, runes[0]); idx != -1 { if idx := slices.Index(Separators, runes[0]); idx != -1 {
return &LexToken{Type: Type_Separator, Position: l.LastTokenPosition, Value: Separator(idx)}, nil return &LexToken{Type: Type_Separator, Position: TokenPosition{SourceFile: l.SourceFile, Position: l.LastTokenPosition}, Value: Separator(idx)}, nil
} }
} }
if idx := slices.Index(Keywords, token); idx != -1 { if idx := slices.Index(Keywords, token); idx != -1 {
return &LexToken{Type: Type_Keyword, Position: l.LastTokenPosition, Value: Keyword(idx)}, nil return &LexToken{Type: Type_Keyword, Position: TokenPosition{SourceFile: l.SourceFile, Position: l.LastTokenPosition}, Value: Keyword(idx)}, nil
} }
return &LexToken{Type: Type_Identifier, Position: l.LastTokenPosition, Value: token}, nil return &LexToken{Type: Type_Identifier, Position: TokenPosition{SourceFile: l.SourceFile, Position: l.LastTokenPosition}, Value: token}, nil
} }
func (l *Lexer) parseNumber(raw string, numberType PrimitiveType) (any, error) { func (l *Lexer) parseNumber(raw string, numberType PrimitiveType) (any, error) {
@ -335,10 +341,10 @@ func (l *Lexer) parseNumber(raw string, numberType PrimitiveType) (any, error) {
panic(fmt.Sprintf("Unhandled type %s in parseNumber()", numberType)) panic(fmt.Sprintf("Unhandled type %s in parseNumber()", numberType))
} }
func lexer(program string) ([]LexToken, error) { func lexer(sourceFile string, source string) ([]LexToken, error) {
var tokens []LexToken var tokens []LexToken
lexer := Lexer{Runes: []rune(program)} lexer := Lexer{SourceFile: sourceFile, Runes: []rune(source)}
for { for {
token, err := lexer.nextToken() token, err := lexer.nextToken()

119
main.go
View File

@ -2,6 +2,7 @@
package main package main
import ( import (
"embed"
"log" "log"
"os" "os"
"strconv" "strconv"
@ -10,6 +11,9 @@ import (
const ERROR_LOG_LINES = 5 const ERROR_LOG_LINES = 5
//go:embed stdlib/*
var stdlib embed.FS
func countTabs(line string) int { func countTabs(line string) int {
tabs := 0 tabs := 0
for _, rune := range line { for _, rune := range line {
@ -20,13 +24,19 @@ func countTabs(line string) int {
return tabs return tabs
} }
func printCompilerError(file string, source string, err CompilerError) { func printCompilerError(sources map[string]string, err CompilerError) {
source, ok := sources[err.Position.SourceFile]
if !ok {
log.Println(err)
return
}
sourceRunes := []rune(source) sourceRunes := []rune(source)
lines := strings.Split(source, "\n") lines := strings.Split(source, "\n")
line := 0 line := 0
col := 0 col := 0
var i uint64 var i uint64
for i = 0; i < err.Position; i++ { for i = 0; i < err.Position.Position; i++ {
col++ col++
if sourceRunes[i] == '\n' { if sourceRunes[i] == '\n' {
line++ line++
@ -34,7 +44,7 @@ func printCompilerError(file string, source string, err CompilerError) {
} }
} }
log.Println("Failed to compile: " + err.Message + " (at " + file + ":" + strconv.Itoa(line+1) + ":" + strconv.Itoa(col+1) + ")") log.Println("Failed to compile: " + err.Message + " (at " + err.Position.SourceFile + ":" + strconv.Itoa(line+1) + ":" + strconv.Itoa(col+1) + ")")
linesStart := max(0, line-ERROR_LOG_LINES) linesStart := max(0, line-ERROR_LOG_LINES)
linesEnd := min(len(lines), line+ERROR_LOG_LINES+1) linesEnd := min(len(lines), line+ERROR_LOG_LINES+1)
@ -51,52 +61,91 @@ func printCompilerError(file string, source string, err CompilerError) {
} }
} }
func readEmbedDir(name string, files map[string]string) {
entries, err := stdlib.ReadDir(name)
if err != nil {
log.Fatalln(err)
}
for _, entry := range entries {
fullName := name + "/" + entry.Name()
if entry.IsDir() {
readEmbedDir(fullName, files)
} else {
bytes, err := stdlib.ReadFile(fullName)
if err != nil {
log.Fatalln(err)
}
files[fullName] = string(bytes)
}
}
}
func main() { func main() {
if len(os.Args) != 2 { if len(os.Args) < 2 {
log.Fatalln("Usage: " + os.Args[0] + " <file>") log.Fatalln("Usage: " + os.Args[0] + " <files...>")
} }
file := os.Args[1] files := os.Args[1:]
content, err := os.ReadFile(file)
if err != nil {
log.Fatalln("Cannot open input file.", err)
}
source := string(content) fileSources := make(map[string]string)
for _, file := range files {
tokens, err := lexer(source) content, err := os.ReadFile(file)
if err != nil { if err != nil {
if c, ok := err.(CompilerError); ok { log.Fatalln("Cannot open input file.", err)
printCompilerError(file, source, c)
} else {
log.Println(err)
} }
return fileSources[file] = string(content)
} }
log.Printf("Tokens:\n%+#v\n\n", tokens) stdlibFiles := make(map[string]string)
readEmbedDir("stdlib", stdlibFiles)
for path, file := range stdlibFiles {
fileSources["[embedded]/"+path] = file
}
parser := Parser{Tokens: tokens} fileTokens := make(map[string][]LexToken)
parsed, err := parser.parseFile() for file, source := range fileSources {
if err != nil { tokens, err := lexer(file, source)
if c, ok := err.(CompilerError); ok { if err != nil {
printCompilerError(file, source, c) if c, ok := err.(CompilerError); ok {
} else { printCompilerError(fileSources, c)
log.Println(err) } else {
log.Println(err)
}
return
} }
return // log.Printf("Tokens:\n%+#v\n\n", tokens)
fileTokens[file] = tokens
} }
log.Printf("Parsed:\n%+#v\n\n", parsed) var parsedFiles []*ParsedFile
for _, tokens := range fileTokens {
parser := Parser{Tokens: tokens}
parsed, err := parser.parseFile()
if err != nil {
if c, ok := err.(CompilerError); ok {
printCompilerError(fileSources, c)
} else {
log.Println(err)
}
validator := Validator{file: parsed} return
}
log.Printf("Parsed:\n%+#v\n\n", parsed)
parsedFiles = append(parsedFiles, parsed)
}
validator := Validator{files: parsedFiles}
errors := validator.validate() errors := validator.validate()
if len(errors) != 0 { if len(errors) != 0 {
for _, err = range errors { for _, err := range errors {
if c, ok := err.(CompilerError); ok { if c, ok := err.(CompilerError); ok {
printCompilerError(file, source, c) printCompilerError(fileSources, c)
} else { } else {
log.Println(err) log.Println(err)
} }
@ -107,12 +156,12 @@ func main() {
} }
} }
log.Printf("Validated:\n%+#v\n\n", parsed) // log.Printf("Validated:\n%+#v\n\n", parsedFiles)
wat, err := backendWAT(*parsed) wat, err := backendWAT(parsedFiles)
if err != nil { if err != nil {
if c, ok := err.(CompilerError); ok { if c, ok := err.(CompilerError); ok {
printCompilerError(file, source, c) printCompilerError(fileSources, c)
} else { } else {
log.Println(err) log.Println(err)
} }

35
parser.go
View File

@ -17,7 +17,7 @@ const (
type Type struct { type Type struct {
Type TypeType Type TypeType
Value any Value any
Position uint64 Position TokenPosition
} }
type NamedType struct { type NamedType struct {
@ -45,7 +45,7 @@ const (
type Statement struct { type Statement struct {
Type StatementType Type StatementType
Value any Value any
Position uint64 Position TokenPosition
} }
type ExpressionStatement struct { type ExpressionStatement struct {
@ -88,7 +88,7 @@ type Expression struct {
Type ExpressionType Type ExpressionType
Value any Value any
ValueType *Type ValueType *Type
Position uint64 Position TokenPosition
} }
type AssignmentExpression struct { type AssignmentExpression struct {
@ -161,6 +161,7 @@ type ParsedParameter struct {
type ParsedFunction struct { type ParsedFunction struct {
Name string Name string
FullName string // The fully-qualified name of the function, including the module name
Parameters []ParsedParameter Parameters []ParsedParameter
ReturnType *Type ReturnType *Type
Body *Block Body *Block
@ -172,13 +173,14 @@ type Import struct {
} }
type ParsedFile struct { type ParsedFile struct {
Module string
Imports []Import Imports []Import
Functions []ParsedFunction Functions []ParsedFunction
} }
type Parser struct { type Parser struct {
Tokens []LexToken Tokens []LexToken
Position uint64 Position TokenPosition
} }
func (p Parser) copy() Parser { func (p Parser) copy() Parser {
@ -481,6 +483,8 @@ func (p *Parser) tryUnaryExpression() (*Expression, error) {
if token.Type == Type_Identifier { if token.Type == Type_Identifier {
pCopy.nextToken() pCopy.nextToken()
// TODO: possible module name
next, err := pCopy.trySeparator(Separator_OpenParen) next, err := pCopy.trySeparator(Separator_OpenParen)
if err != nil { if err != nil {
return nil, err return nil, err
@ -921,8 +925,9 @@ func (p *Parser) expectFunction() (*ParsedFunction, error) {
func (p *Parser) parseFile() (*ParsedFile, error) { func (p *Parser) parseFile() (*ParsedFile, error) {
var err error var err error
var functions []ParsedFunction var module string
var imports []Import var imports []Import
var functions []ParsedFunction
for { for {
token := p.peekToken() token := p.peekToken()
@ -941,6 +946,24 @@ func (p *Parser) parseFile() (*ParsedFile, error) {
continue continue
} }
if token.Type == Type_Keyword && token.Value.(Keyword) == Keyword_Module {
p.nextToken()
if module != "" {
return nil, p.error("duplicate module declaration")
}
module, err = p.expectIdentifier()
if err != nil {
return nil, err
}
_, err := p.expectSeparator(Separator_Semicolon)
if err != nil {
return nil, err
}
}
var parsedFunction *ParsedFunction var parsedFunction *ParsedFunction
parsedFunction, err = p.expectFunction() parsedFunction, err = p.expectFunction()
if err != nil { if err != nil {
@ -950,5 +973,5 @@ func (p *Parser) parseFile() (*ParsedFile, error) {
functions = append(functions, *parsedFunction) functions = append(functions, *parsedFunction)
} }
return &ParsedFile{Imports: imports, Functions: functions}, nil return &ParsedFile{Module: module, Imports: imports, Functions: functions}, nil
} }

4
types.go
View File

@ -43,12 +43,12 @@ var STRING_TYPE = Type{Type: Type_Named, Value: STRING_TYPE_NAME}
const InvalidValue = 0xEEEEEE // Magic value const InvalidValue = 0xEEEEEE // Magic value
type CompilerError struct { type CompilerError struct {
Position uint64 Position TokenPosition
Message string Message string
} }
func (e CompilerError) Error() string { func (e CompilerError) Error() string {
return e.Message + " (at " + strconv.FormatUint(e.Position, 10) + ")" return e.Message + " (at " + e.Position.SourceFile + ", index " + strconv.FormatUint(e.Position.Position, 10) + ")"
} }
func isSignedInt(primitiveType PrimitiveType) bool { func isSignedInt(primitiveType PrimitiveType) bool {

46
validator.go
View File

@ -6,7 +6,8 @@ import (
) )
type Validator struct { type Validator struct {
file *ParsedFile files []*ParsedFile
allFunctions map[string]*ParsedFunction
currentBlock *Block currentBlock *Block
currentFunction *ParsedFunction currentFunction *ParsedFunction
@ -74,7 +75,7 @@ func isPrimitiveTypeExpandableTo(from PrimitiveType, to PrimitiveType) bool {
return false return false
} }
func (v *Validator) createError(message string, position uint64) error { func (v *Validator) createError(message string, position TokenPosition) error {
return CompilerError{Position: position, Message: message} return CompilerError{Position: position, Message: message}
} }
@ -234,15 +235,8 @@ func (v *Validator) validatePotentiallyVoidExpression(expr *Expression) []error
case Expression_FunctionCall: case Expression_FunctionCall:
fc := expr.Value.(FunctionCallExpression) fc := expr.Value.(FunctionCallExpression)
var calledFunc *ParsedFunction = nil calledFunc, ok := v.allFunctions[fc.Function]
for _, f := range v.file.Functions { if !ok {
if f.Name == fc.Function {
calledFunc = &f
break
}
}
if calledFunc == nil {
errors = append(errors, v.createError("call to undefined function '"+fc.Function+"'", expr.Position)) errors = append(errors, v.createError("call to undefined function '"+fc.Function+"'", expr.Position))
return errors return errors
} }
@ -435,12 +429,34 @@ func (v *Validator) validateFunction(function *ParsedFunction) []error {
func (v *Validator) validate() []error { func (v *Validator) validate() []error {
var errors []error var errors []error
for i := range v.file.Imports { v.allFunctions = make(map[string]*ParsedFunction)
errors = append(errors, v.validateImport(&v.file.Imports[i])...) for _, file := range v.files {
for i := range file.Functions {
function := &file.Functions[i]
fullFunctionName := function.Name
if file.Module != "" {
fullFunctionName = file.Module + "." + fullFunctionName
}
function.FullName = fullFunctionName
if _, exists := v.allFunctions[fullFunctionName]; exists {
errors = append(errors, v.createError("duplicate function "+fullFunctionName, function.ReturnType.Position))
}
v.allFunctions[fullFunctionName] = function
}
} }
for i := range v.file.Functions { for _, file := range v.files {
errors = append(errors, v.validateFunction(&v.file.Functions[i])...) for i := range file.Imports {
errors = append(errors, v.validateImport(&file.Imports[i])...)
}
for i := range file.Functions {
errors = append(errors, v.validateFunction(&file.Functions[i])...)
}
} }
return errors return errors