[refactoring] remove lexer config struct
This commit is contained in:
Vadym Slizov
parent
7eff83624e
commit
424f7a132c
@ -25,14 +25,9 @@ func NewParser(src []byte, v string, withTokens bool) *Parser {
|
||||
withTokens: withTokens,
|
||||
}
|
||||
|
||||
scannerConfig := scanner.Config{
|
||||
WithHiddenTokens: withTokens,
|
||||
ErrHandlerFunc: func(e *errors.Error) {
|
||||
parser.errors = append(parser.errors, e)
|
||||
},
|
||||
}
|
||||
|
||||
lexer := scanner.NewLexer(src, v, scannerConfig)
|
||||
lexer := scanner.NewLexer(src, v, withTokens, func(e *errors.Error) {
|
||||
parser.errors = append(parser.errors, e)
|
||||
})
|
||||
parser.Lexer = lexer
|
||||
|
||||
return parser
|
||||
|
||||
@ -24,53 +24,48 @@ func NewParser(src []byte, v string, withTokens bool) *Parser {
|
||||
withTokens: withTokens,
|
||||
}
|
||||
|
||||
scannerConfig := scanner.Config{
|
||||
WithHiddenTokens: withTokens,
|
||||
ErrHandlerFunc: func(e *errors.Error) {
|
||||
parser.errors = append(parser.errors, e)
|
||||
},
|
||||
}
|
||||
|
||||
lexer := scanner.NewLexer(src, v, scannerConfig)
|
||||
lexer := scanner.NewLexer(src, v, withTokens, func(e *errors.Error) {
|
||||
parser.errors = append(parser.errors, e)
|
||||
})
|
||||
parser.Lexer = lexer
|
||||
|
||||
return parser
|
||||
}
|
||||
|
||||
func (l *Parser) Lex(lval *yySymType) int {
|
||||
t := l.Lexer.Lex()
|
||||
func (p *Parser) Lex(lval *yySymType) int {
|
||||
t := p.Lexer.Lex()
|
||||
|
||||
l.currentToken = t
|
||||
p.currentToken = t
|
||||
lval.token = t
|
||||
|
||||
return int(t.ID)
|
||||
}
|
||||
|
||||
func (l *Parser) Error(msg string) {
|
||||
var pos = l.currentToken.Position
|
||||
func (p *Parser) Error(msg string) {
|
||||
var pos = p.currentToken.Position
|
||||
|
||||
l.errors = append(l.errors, errors.NewError(msg, &pos))
|
||||
p.errors = append(p.errors, errors.NewError(msg, &pos))
|
||||
}
|
||||
|
||||
// GetErrors returns errors list
|
||||
func (l *Parser) GetErrors() []*errors.Error {
|
||||
return l.errors
|
||||
func (p *Parser) GetErrors() []*errors.Error {
|
||||
return p.errors
|
||||
}
|
||||
|
||||
// Parse the php7 Parser entrypoint
|
||||
func (l *Parser) Parse() int {
|
||||
func (p *Parser) Parse() int {
|
||||
// init
|
||||
l.errors = nil
|
||||
l.rootNode = nil
|
||||
p.errors = nil
|
||||
p.rootNode = nil
|
||||
|
||||
// parse
|
||||
|
||||
return yyParse(l)
|
||||
return yyParse(p)
|
||||
}
|
||||
|
||||
// GetRootNode returns root node
|
||||
func (l *Parser) GetRootNode() ast.Vertex {
|
||||
return l.rootNode
|
||||
func (p *Parser) GetRootNode() ast.Vertex {
|
||||
return p.rootNode
|
||||
}
|
||||
|
||||
// helpers
|
||||
@ -86,8 +81,8 @@ func isDollar(r rune) bool {
|
||||
return r == '$'
|
||||
}
|
||||
|
||||
func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
|
||||
if l.withTokens == false {
|
||||
func (p *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
|
||||
if p.withTokens == false {
|
||||
return
|
||||
}
|
||||
|
||||
@ -95,7 +90,7 @@ func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
|
||||
return
|
||||
}
|
||||
|
||||
l.setFreeFloating(dst, token.Start, src.GetNode().Tokens[token.Start])
|
||||
p.setFreeFloating(dst, token.Start, src.GetNode().Tokens[token.Start])
|
||||
delete(src.GetNode().Tokens, token.Start)
|
||||
}
|
||||
|
||||
@ -116,8 +111,8 @@ func (p *Parser) setFreeFloating(dst ast.Vertex, pos token.Position, strings []t
|
||||
(*dstCollection)[pos] = strings
|
||||
}
|
||||
|
||||
func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
|
||||
if l.withTokens == false {
|
||||
func (p *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
|
||||
if p.withTokens == false {
|
||||
return []token.Token{}
|
||||
}
|
||||
|
||||
@ -129,12 +124,12 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Parser) addDollarToken(v ast.Vertex) {
|
||||
if l.withTokens == false {
|
||||
func (p *Parser) addDollarToken(v ast.Vertex) {
|
||||
if p.withTokens == false {
|
||||
return
|
||||
}
|
||||
|
||||
l.setFreeFloating(v, token.Dollar, []token.Token{
|
||||
p.setFreeFloating(v, token.Dollar, []token.Token{
|
||||
{
|
||||
ID: token.ID('$'),
|
||||
Value: []byte("$"),
|
||||
@ -142,8 +137,8 @@ func (l *Parser) addDollarToken(v ast.Vertex) {
|
||||
})
|
||||
}
|
||||
|
||||
func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
|
||||
if l.withTokens == false {
|
||||
func (p *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
|
||||
if p.withTokens == false {
|
||||
return
|
||||
}
|
||||
|
||||
@ -154,7 +149,7 @@ func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.
|
||||
}
|
||||
|
||||
if semiColon[0].Value[0] == ';' {
|
||||
l.setFreeFloating(prevNode, token.SemiColon, []token.Token{
|
||||
p.setFreeFloating(prevNode, token.SemiColon, []token.Token{
|
||||
{
|
||||
ID: token.ID(';'),
|
||||
Value: semiColon[0].Value[0:1],
|
||||
@ -181,7 +176,7 @@ func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.
|
||||
Value: semiColon[0].Value[vlen-tlen:],
|
||||
})
|
||||
|
||||
l.setFreeFloating(htmlNode, token.Start, append(phpCloseTag, htmlNode.GetNode().Tokens[token.Start]...))
|
||||
p.setFreeFloating(htmlNode, token.Start, append(phpCloseTag, htmlNode.GetNode().Tokens[token.Start]...))
|
||||
}
|
||||
|
||||
func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
|
||||
|
||||
@ -10,13 +10,6 @@ import (
|
||||
"github.com/z7zmey/php-parser/pkg/token"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
WithHiddenTokens bool
|
||||
ErrHandlerFunc func(*errors.Error)
|
||||
}
|
||||
|
||||
var DefaultConfig = Config{}
|
||||
|
||||
type Lexer struct {
|
||||
data []byte
|
||||
phpVersion string
|
||||
@ -34,12 +27,12 @@ type Lexer struct {
|
||||
newLines NewLines
|
||||
}
|
||||
|
||||
func NewLexer(data []byte, phpVersion string, config Config) *Lexer {
|
||||
func NewLexer(data []byte, phpVersion string, withHiddenTokens bool, errHandlerFunc func(*errors.Error)) *Lexer {
|
||||
lex := &Lexer{
|
||||
data: data,
|
||||
phpVersion: phpVersion,
|
||||
errHandlerFunc: config.ErrHandlerFunc,
|
||||
withHiddenTokens: config.WithHiddenTokens,
|
||||
withHiddenTokens: withHiddenTokens,
|
||||
errHandlerFunc: errHandlerFunc,
|
||||
|
||||
pe: len(data),
|
||||
stack: make([]int, 0),
|
||||
|
||||
@ -353,7 +353,7 @@ func TestTokens(t *testing.T) {
|
||||
T_UNSET_CAST.String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
|
||||
@ -381,7 +381,7 @@ func TestShebang(t *testing.T) {
|
||||
"\n",
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
|
||||
@ -401,7 +401,7 @@ func TestShebangHtml(t *testing.T) {
|
||||
0.1
|
||||
`
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
tkn := lexer.Lex()
|
||||
@ -451,7 +451,7 @@ func TestNumberTokens(t *testing.T) {
|
||||
T_DNUMBER.String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
|
||||
@ -508,7 +508,7 @@ func TestConstantStrings(t *testing.T) {
|
||||
T_CONSTANT_ENCAPSED_STRING.String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
|
||||
@ -555,7 +555,7 @@ func TestSingleQuoteStringTokens(t *testing.T) {
|
||||
T_CONSTANT_ENCAPSED_STRING.String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
actual := []string{}
|
||||
|
||||
for {
|
||||
@ -642,7 +642,7 @@ func TestTeplateStringTokens(t *testing.T) {
|
||||
TokenID(int('"')).String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
|
||||
@ -727,7 +727,7 @@ func TestBackquoteStringTokens(t *testing.T) {
|
||||
TokenID(int('`')).String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
|
||||
@ -821,7 +821,7 @@ CAT;
|
||||
TokenID(int(';')).String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
|
||||
@ -894,7 +894,7 @@ CAT
|
||||
T_END_HEREDOC.String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
|
||||
@ -933,7 +933,7 @@ CAT;
|
||||
TokenID(int(';')).String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
|
||||
@ -964,7 +964,7 @@ func TestHereDocTokens73(t *testing.T) {
|
||||
T_VARIABLE.String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
|
||||
@ -994,7 +994,7 @@ CAT;`
|
||||
TokenID(int(';')).String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.phpVersion = "7.2"
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
@ -1027,7 +1027,7 @@ func TestInlineHtmlNopTokens(t *testing.T) {
|
||||
T_INLINE_HTML.String(),
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
actual := []string{}
|
||||
|
||||
@ -1062,7 +1062,7 @@ func TestStringTokensAfterVariable(t *testing.T) {
|
||||
"\"",
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
actual := []string{}
|
||||
actualTokens := []string{}
|
||||
|
||||
@ -1095,7 +1095,7 @@ func TestSlashAfterVariable(t *testing.T) {
|
||||
"3",
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
actual := []string{}
|
||||
actualTokens := []string{}
|
||||
|
||||
@ -1132,7 +1132,7 @@ func TestCommentEnd(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
lexer.Lex()
|
||||
@ -1161,7 +1161,7 @@ func TestCommentNewLine(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
tkn := lexer.Lex()
|
||||
@ -1190,7 +1190,7 @@ func TestCommentNewLine1(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
tkn := lexer.Lex()
|
||||
@ -1219,7 +1219,7 @@ func TestCommentNewLine2(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
tkn := lexer.Lex()
|
||||
@ -1249,7 +1249,7 @@ func TestCommentWithPhpEndTag(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
tkn := lexer.Lex()
|
||||
@ -1279,7 +1279,7 @@ func TestInlineComment(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
tkn := lexer.Lex()
|
||||
@ -1309,7 +1309,7 @@ func TestInlineComment2(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
lexer.Lex()
|
||||
@ -1343,7 +1343,7 @@ func TestEmptyInlineComment(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
lexer.Lex()
|
||||
@ -1373,7 +1373,7 @@ func TestEmptyInlineComment2(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
tkn := lexer.Lex()
|
||||
@ -1387,7 +1387,7 @@ func TestMethodCallTokens(t *testing.T) {
|
||||
src := `<?php
|
||||
$a -> bar ( '' ) ;`
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
expected := []token.Token{
|
||||
@ -1469,7 +1469,7 @@ func TestYieldFromTokens(t *testing.T) {
|
||||
src := `<?php
|
||||
yield from $a`
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
lexer.withHiddenTokens = true
|
||||
|
||||
expected := []token.Token{
|
||||
@ -1500,7 +1500,7 @@ func TestYieldFromTokens(t *testing.T) {
|
||||
func TestVarNameByteChars(t *testing.T) {
|
||||
src := "<?php $\x80 $\xff"
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
|
||||
tkn := lexer.Lex()
|
||||
assert.Equal(t, "$\x80", string(tkn.Value))
|
||||
@ -1512,7 +1512,7 @@ func TestVarNameByteChars(t *testing.T) {
|
||||
func TestStringVarNameByteChars(t *testing.T) {
|
||||
src := "<?php \"$\x80 $\xff\""
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
|
||||
tkn := lexer.Lex()
|
||||
assert.Equal(t, "\"", string(tkn.Value))
|
||||
@ -1534,13 +1534,9 @@ func TestIgnoreControllCharacters(t *testing.T) {
|
||||
src := "<?php \004 echo $b;"
|
||||
|
||||
var actualErr *errors.Error
|
||||
config := Config{
|
||||
ErrHandlerFunc: func(e *errors.Error) {
|
||||
actualErr = e
|
||||
},
|
||||
}
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", config)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, func(e *errors.Error) {
|
||||
actualErr = e
|
||||
})
|
||||
|
||||
expected := "echo"
|
||||
tkn := lexer.Lex()
|
||||
@ -1562,7 +1558,7 @@ func TestIgnoreControllCharacters(t *testing.T) {
|
||||
func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) {
|
||||
src := "<?php \"$a[test\004]\";"
|
||||
|
||||
lexer := NewLexer([]byte(src), "7.4", DefaultConfig)
|
||||
lexer := NewLexer([]byte(src), "7.4", false, nil)
|
||||
|
||||
expected := "\""
|
||||
tkn := lexer.Lex()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user