mr/elysium
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework parser structure

Browse Source
This commit is contained in:
MrLetsplay 2024-03-12 22:47:45 +01:00
parent f8ed70336d
commit f07754d79a
Signed by: mr
SSH Key Fingerprint: SHA256:92jBH80vpXyaZHjaIl47pjRq+Yt7XGTArqQg1V7hSqg
2 changed files with 97 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
main.go
View File

@ -23,7 +23,8 @@ func main() {
log.Printf("Tokens:\n%+#v\n\n", tokens)
parsed, err := parser(tokens)
parser := Parser{tokens}
parsed, err := parser.parseFile()
if err != nil {
log.Fatalln(err)
}

170
parser.go
View File

@ -2,6 +2,7 @@ package main
import (
"errors"
"log"
"slices"
"strings"
)
@ -129,88 +130,100 @@ type ParsedFile struct {
Functions []ParsedFunction
}
func expectSeparator(tokens []LexToken, separators ...Separator) (Separator, []LexToken, error) {
type Parser struct {
tokens []LexToken
}
func (p *Parser) peekToken() *LexToken {
if len(p.tokens) == 0 {
return nil
}
return &p.tokens[0]
}
func (p *Parser) nextToken() *LexToken {
if len(p.tokens) == 0 {
return nil
}
token := p.tokens[0]
p.tokens = p.tokens[1:]
return &token
}
func (p *Parser) expectSeparator(separators ...Separator) (Separator, error) {
var separatorNames []string
for _, sep := range separators {
separatorNames = append(separatorNames, string(Separators[sep]))
}
if len(tokens) == 0 {
return InvalidValue, nil, errors.New("expected one of " + strings.Join(separatorNames, " "))
separator := p.nextToken()
if separator == nil || separator.Type != Type_Separator || !slices.Contains(separators, separator.Value.(Separator)) {
return InvalidValue, errors.New("expected one of " + strings.Join(separatorNames, " "))
}
separator := tokens[0]
if separator.Type != Type_Separator || !slices.Contains(separators, separator.Value.(Separator)) {
return InvalidValue, nil, errors.New("expected one of " + strings.Join(separatorNames, " "))
return separator.Value.(Separator), nil
}
return separator.Value.(Separator), tokens[1:], nil
func (p *Parser) expectIdentifier() (string, error) {
identifier := p.nextToken()
if identifier == nil || identifier.Type != Type_Separator && identifier.Type != Type_Identifier {
return "", errors.New("expected identifier")
}
func expectIdentifier(tokens []LexToken) (string, []LexToken, error) {
if len(tokens) == 0 {
return "", nil, errors.New("expected identifier")
return identifier.Value.(string), nil
}
token := tokens[0]
if token.Type != Type_Separator && token.Type != Type_Identifier {
return "", nil, errors.New("expected identifier")
}
return token.Value.(string), tokens[1:], nil
}
func parseImport(tokens []LexToken) (*Import, []LexToken, error) {
func (p *Parser) expectImport() (*Import, error) {
var err error
if len(tokens) < 3 {
return nil, nil, errors.New("incomplete import")
importToken := p.nextToken()
if importToken == nil || importToken.Type != Type_Keyword || importToken.Value.(Keyword) != Keyword_Import {
return nil, errors.New("expected import")
}
// tokens[0] == import keyword
identifier := tokens[1]
if identifier.Type != Type_Identifier {
return nil, nil, errors.New("expected identifier")
identifier := p.nextToken()
if identifier == nil || identifier.Type != Type_Identifier {
return nil, errors.New("expected identifier")
}
_, tokens, err = expectSeparator(tokens[2:], Separator_Semicolon)
_, err = p.expectSeparator(Separator_Semicolon)
if err != nil {
return nil, nil, err
return nil, err
}
return &Import{Import: identifier.Value.(string)}, tokens, nil
return &Import{Import: identifier.Value.(string)}, nil
}
func parseType(tokens []LexToken) (*Type, []LexToken, error) {
func (p *Parser) expectType() (*Type, error) {
var err error
if len(tokens) == 0 {
return nil, nil, errors.New("expected type")
tok := p.nextToken()
if tok == nil {
return nil, errors.New("expected type")
}
tok := tokens[0]
if tok.Type == Type_Keyword && tok.Value.(Keyword) == Keyword_Void {
return &Type{Type: Type_Void, Value: nil}, tokens[1:], nil
return &Type{Type: Type_Void, Value: nil}, nil
}
if tok.Type == Type_Separator && tok.Value.(Separator) == Separator_OpenParen {
// Tuple type
var types []Type
tokens = tokens[1:]
for {
var parsedType *Type
parsedType, tokens, err = parseType(tokens)
parsedType, err = p.expectType()
if err != nil {
return nil, nil, err
return nil, err
}
types = append(types, *parsedType)
var sep Separator
sep, tokens, err = expectSeparator(tokens, Separator_Comma, Separator_CloseParen)
sep, err = p.expectSeparator(Separator_Comma, Separator_CloseParen)
if err != nil {
return nil, nil, err
return nil, err
}
if sep == Separator_CloseParen {
@ -219,20 +232,20 @@ func parseType(tokens []LexToken) (*Type, []LexToken, error) {
}
if len(types) == 0 {
return nil, nil, errors.New("empty tuple")
return nil, errors.New("empty tuple")
}
return &Type{Type: Type_Tuple, Value: TupleType{Types: types}}, tokens, nil
return &Type{Type: Type_Tuple, Value: TupleType{Types: types}}, nil
}
if tok.Type == Type_Identifier {
return &Type{Type: Type_Named, Value: tok.Value}, tokens[1:], nil
return &Type{Type: Type_Named, Value: tok.Value}, nil
}
return nil, nil, errors.New("expected type")
return nil, errors.New("expected type")
}
func parseFunction(tokens []LexToken) (*ParsedFunction, []LexToken, error) {
func (p *Parser) expectFunction() (*ParsedFunction, error) {
var err error
var name string
@ -240,79 +253,86 @@ func parseFunction(tokens []LexToken) (*ParsedFunction, []LexToken, error) {
var returnType *Type
var body Block
returnType, tokens, err = parseType(tokens)
returnType, err = p.expectType()
if err != nil {
return nil, nil, err
return nil, err
}
if len(tokens) < 2 {
return nil, nil, errors.New("incomplete function declaration")
name, err = p.expectIdentifier()
if err != nil {
return nil, err
}
name, tokens, err = expectIdentifier(tokens)
_, err = p.expectSeparator(Separator_OpenParen)
if err != nil {
return nil, nil, err
}
_, tokens, err = expectSeparator(tokens, Separator_OpenParen)
if err != nil {
return nil, nil, err
return nil, err
}
for {
if len(tokens) == 0 {
return nil, nil, errors.New("incomplete function declaration")
token := p.peekToken()
if token == nil {
return nil, errors.New("incomplete function declaration")
}
token := tokens[0]
if token.Type == Type_Separator && token.Value.(Separator) == Separator_CloseParen {
tokens = tokens[1:]
p.nextToken()
break
}
if len(parameters) > 0 {
if token.Type != Type_Separator && token.Value.(Separator) != Separator_Comma {
return nil, nil, errors.New("expected ,")
_, err := p.expectSeparator(Separator_Comma)
if err != nil {
return nil, err
}
tokens = tokens[1:]
}
var paramType *Type
paramType, tokens, err = parseType(tokens)
paramType, err = p.expectType()
if err != nil {
return nil, nil, err
return nil, err
}
var paramName string
paramName, tokens, err = expectIdentifier(tokens)
paramName, err = p.expectIdentifier()
if err != nil {
return nil, nil, err
return nil, err
}
parameters = append(parameters, ParsedParameter{Name: paramName, Type: *paramType})
}
_, tokens, err = expectSeparator(tokens, Separator_OpenCurly)
_, err = p.expectSeparator(Separator_OpenCurly)
if err != nil {
return nil, nil, err
return nil, err
}
// TODO: body, closing curly
return &ParsedFunction{Name: name, Parameters: parameters, ReturnType: *returnType, Body: body}, tokens, nil
_, err = p.expectSeparator(Separator_CloseCurly)
if err != nil {
return nil, err
}
func parser(tokens []LexToken) (*ParsedFile, error) {
return &ParsedFunction{Name: name, Parameters: parameters, ReturnType: *returnType, Body: body}, nil
}
func (p *Parser) parseFile() (*ParsedFile, error) {
var err error
var functions []ParsedFunction
var imports []Import
for len(tokens) > 0 {
if tokens[0].Type == Type_Keyword && tokens[0].Value.(Keyword) == Keyword_Import {
for {
token := p.peekToken()
if token == nil {
break
}
log.Printf("%+#v\n", token)
if token.Type == Type_Keyword && token.Value.(Keyword) == Keyword_Import {
var parsedImport *Import
parsedImport, tokens, err = parseImport(tokens)
parsedImport, err = p.expectImport()
if err != nil {
return nil, err
}
@ -322,7 +342,7 @@ func parser(tokens []LexToken) (*ParsedFile, error) {
}
var parsedFunction *ParsedFunction
parsedFunction, tokens, err = parseFunction(tokens)
parsedFunction, err = p.expectFunction()
if err != nil {
return nil, err
}