maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[refactoring] remove scanner token

Browse Source
This commit is contained in:
Vadym Slizov
2020-08-17 20:31:04 +03:00
parent 394092269a
commit 97747c5ac0
36 changed files with 6591 additions and 9813 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
internal/php7/parser.go
View File

@@ -12,7 +12,7 @@ import (
// Parser structure
type Parser struct {
Lexer *scanner.Lexer
currentToken *scanner.Token
currentToken *token.Token
rootNode ast.Vertex
errHandlerFunc func(*errors.Error)
}
@@ -39,8 +39,7 @@ func (p *Parser) Error(msg string) {
return
}
var pos = p.currentToken.Position
p.errHandlerFunc(errors.NewError(msg, &pos))
p.errHandlerFunc(errors.NewError(msg, p.currentToken.Position))
}
// Parse the php7 Parser entrypoint
@@ -82,7 +81,7 @@ func (p *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
delete(src.GetNode().Tokens, token.Start)
}
func (p *Parser) setFreeFloating(dst ast.Vertex, pos token.Position, tokens []token.Token) {
func (p *Parser) setFreeFloating(dst ast.Vertex, pos token.Position, tokens []*token.Token) {
if len(tokens) == 0 {
return
}
@@ -98,7 +97,7 @@ func (p *Parser) setFreeFloating(dst ast.Vertex, pos token.Position, tokens []to
}
}
func (p *Parser) setFreeFloatingTokens(dst ast.Vertex, pos token.Position, tokens []token.Token) {
func (p *Parser) setFreeFloatingTokens(dst ast.Vertex, pos token.Position, tokens []*token.Token) {
if len(tokens) == 0 {
return
}
@@ -108,14 +107,14 @@ func (p *Parser) setFreeFloatingTokens(dst ast.Vertex, pos token.Position, token
*dstCollection = make(token.Collection)
}
(*dstCollection)[pos] = make([]token.Token, 0)
(*dstCollection)[pos] = make([]*token.Token, 0)
for _, v := range tokens {
(*dstCollection)[pos] = append((*dstCollection)[pos], v)
}
}
func (p *Parser) setToken(dst ast.Vertex, pos token.Position, tokens []token.Token) {
func (p *Parser) setToken(dst ast.Vertex, pos token.Position, tokens []*token.Token) {
if len(tokens) == 0 {
return
}
@@ -141,7 +140,7 @@ func (p *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.
}
if semiColon[0].Value[0] == ';' {
p.setFreeFloatingTokens(prevNode, token.SemiColon, []token.Token{
p.setFreeFloatingTokens(prevNode, token.SemiColon, []*token.Token{
{
ID: token.ID(';'),
Value: semiColon[0].Value[0:1],
@@ -155,28 +154,18 @@ func (p *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.
tlen = 3
}
phpCloseTag := []token.Token{}
phpCloseTag := []*token.Token{}
if vlen-tlen > 1 {
phpCloseTag = append(phpCloseTag, token.Token{
phpCloseTag = append(phpCloseTag, &token.Token{
ID: token.T_WHITESPACE,
Value: semiColon[0].Value[1 : vlen-tlen],
})
}
phpCloseTag = append(phpCloseTag, token.Token{
phpCloseTag = append(phpCloseTag, &token.Token{
ID: T_CLOSE_TAG,
Value: semiColon[0].Value[vlen-tlen:],
})
p.setFreeFloatingTokens(htmlNode, token.Start, append(phpCloseTag, htmlNode.GetNode().Tokens[token.Start]...))
}
func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
for i := 1; i < len(yyDollar); i++ {
if yyDollar[i].token != nil {
p.Lexer.ReturnTokenToPool(yyDollar[i].token)
}
yyDollar[i].token = nil
}
yyVAL.token = nil
}

846
internal/php7/parser_test.go
View File

File diff suppressed because it is too large Load Diff

2890
internal/php7/php7.go generated
View File

File diff suppressed because it is too large Load Diff

1909
internal/php7/php7.y
View File

File diff suppressed because it is too large Load Diff

2
internal/php7/php7_bench_test.go
View File

@@ -382,7 +382,7 @@ CAD;
`
for n := 0; n < b.N; n++ {
lexer := scanner.NewLexer([]byte(src), "7.4", false, nil)
lexer := scanner.NewLexer([]byte(src), "7.4", nil)
php7parser := php7.NewParser(lexer, nil)
php7parser.Parse()
}

11
internal/php7/php7_test.go
View File

@@ -19595,11 +19595,12 @@ func TestPhp7(t *testing.T) {
},
}
lexer := scanner.NewLexer(src, "7.4", false, nil)
lexer := scanner.NewLexer(src, "7.4", nil)
php7parser := php7.NewParser(lexer, nil)
php7parser.Parse()
actual := php7parser.GetRootNode()
traverser.NewDFS(new(visitor.FilterParserNodes)).Traverse(actual)
traverser.NewDFS(new(visitor.FilterTokens)).Traverse(actual)
assert.DeepEqual(t, expected, actual)
}
@@ -19734,11 +19735,12 @@ func TestPhp5Strings(t *testing.T) {
},
}
lexer := scanner.NewLexer([]byte(src), "7.4", false, nil)
lexer := scanner.NewLexer([]byte(src), "7.4", nil)
php7parser := php7.NewParser(lexer, nil)
php7parser.Parse()
actual := php7parser.GetRootNode()
traverser.NewDFS(new(visitor.FilterParserNodes)).Traverse(actual)
traverser.NewDFS(new(visitor.FilterTokens)).Traverse(actual)
assert.DeepEqual(t, expected, actual)
}
@@ -19962,11 +19964,12 @@ CAD;
},
}
lexer := scanner.NewLexer([]byte(src), "7.4", false, nil)
lexer := scanner.NewLexer([]byte(src), "7.4", nil)
php7parser := php7.NewParser(lexer, nil)
php7parser.Parse()
actual := php7parser.GetRootNode()
traverser.NewDFS(new(visitor.FilterParserNodes)).Traverse(actual)
traverser.NewDFS(new(visitor.FilterTokens)).Traverse(actual)
assert.DeepEqual(t, expected, actual)
}
@@ -19989,7 +19992,7 @@ func TestPhp7ControlCharsErrors(t *testing.T) {
parserErrors = append(parserErrors, e)
}
lexer := scanner.NewLexer([]byte(src), "7.4", false, errorHandlerFunc)
lexer := scanner.NewLexer([]byte(src), "7.4", errorHandlerFunc)
php7parser := php7.NewParser(lexer, errorHandlerFunc)
php7parser.Parse()
assert.DeepEqual(t, expected, parserErrors)