maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[refactoring] scanner.Lexer.withHiddenTokens

Browse Source
This commit is contained in:
Vadym Slizov 2020-05-18 21:15:07 +03:00
parent 291dc7e884
commit d9a7d20e73
16 changed files with 706 additions and 711 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/php-parser/main.go
View File

@ -108,15 +108,11 @@ func parserWorker(fileCh <-chan *file, r chan<- result) {
return
}
parserWorker, err := parser.NewParser(f.content, phpVersion)
parserWorker, err := parser.NewParser(f.content, phpVersion, *withFreeFloating)
if err != nil {
panic(err.Error())
}
if *withFreeFloating {
parserWorker.WithTokens()
}
parserWorker.Parse()
r <- result{path: f.path, parser: parserWorker}

32
internal/php5/parser.go
View File

@ -22,17 +22,23 @@ type Parser struct {
positionBuilder *positionbuilder.PositionBuilder
rootNode ast.Vertex
errors []*errors.Error
withTokens bool
}
// NewParser creates and returns new Parser
func NewParser(src []byte, v string) *Parser {
parser := &Parser{}
func NewParser(src []byte, v string, withTokens bool) *Parser {
parser := &Parser{
withTokens: withTokens,
}
lexer := scanner.NewLexer(src, func(e *errors.Error) {
parser.errors = append(parser.errors, e)
})
lexer.PHPVersion = v
scannerConfig := scanner.Config{
WithHiddenTokens: withTokens,
ErrHandlerFunc: func(e *errors.Error) {
parser.errors = append(parser.errors, e)
},
}
lexer := scanner.NewLexer(src, v, scannerConfig)
parser.Lexer = lexer
return parser
@ -59,10 +65,6 @@ func (l *Parser) GetErrors() []*errors.Error {
return l.errors
}
func (l *Parser) WithTokens() {
l.Lexer.SetWithHiddenTokens(true)
}
// Parse the php7 Parser entrypoint
func (l *Parser) Parse() int {
// init
@ -98,7 +100,7 @@ func isDollar(r rune) bool {
}
func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false {
if l.withTokens == false {
return
}
@ -111,7 +113,7 @@ func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
}
func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) {
if l.Lexer.GetWithHiddenTokens() == false {
if l.withTokens == false {
return
}
@ -128,7 +130,7 @@ func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []tok
}
func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
if l.Lexer.GetWithHiddenTokens() == false {
if l.withTokens == false {
return []token.Token{}
}
@ -141,7 +143,7 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
}
func (l *Parser) addDollarToken(v ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false {
if l.withTokens == false {
return
}
@ -154,7 +156,7 @@ func (l *Parser) addDollarToken(v ast.Vertex) {
}
func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false {
if l.withTokens == false {
return
}

516
internal/php5/parser_test.go
View File

File diff suppressed because it is too large Load Diff

2
internal/php5/php5_bench_test.go
View File

@ -413,7 +413,7 @@ CAD;
`
for n := 0; n < b.N; n++ {
php5parser := php5.NewParser([]byte(src), "5.6")
php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse()
}
}

8
internal/php5/php5_test.go
View File

@ -22455,7 +22455,7 @@ func TestPhp5(t *testing.T) {
},
}
php5parser := php5.NewParser([]byte(src), "5.6")
php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse()
actual := php5parser.GetRootNode()
assert.DeepEqual(t, expected, actual)
@ -22592,7 +22592,7 @@ func TestPhp5Strings(t *testing.T) {
},
}
php5parser := php5.NewParser([]byte(src), "5.6")
php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse()
actual := php5parser.GetRootNode()
assert.DeepEqual(t, expected, actual)
@ -22818,7 +22818,7 @@ CAD;
},
}
php5parser := php5.NewParser([]byte(src), "5.6")
php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse()
actual := php5parser.GetRootNode()
assert.DeepEqual(t, expected, actual)
@ -22838,7 +22838,7 @@ func TestPhp5ControlCharsErrors(t *testing.T) {
},
}
php5parser := php5.NewParser([]byte(src), "5.6")
php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse()
actual := php5parser.GetErrors()
assert.DeepEqual(t, expected, actual)

32
internal/php7/parser.go
View File

@ -21,17 +21,23 @@ type Parser struct {
positionBuilder *positionbuilder.PositionBuilder
rootNode ast.Vertex
errors []*errors.Error
withTokens bool
}
// NewParser creates and returns new Parser
func NewParser(src []byte, v string) *Parser {
parser := &Parser{}
func NewParser(src []byte, v string, withTokens bool) *Parser {
parser := &Parser{
withTokens: withTokens,
}
lexer := scanner.NewLexer(src, func(e *errors.Error) {
parser.errors = append(parser.errors, e)
})
lexer.PHPVersion = v
scannerConfig := scanner.Config{
WithHiddenTokens: withTokens,
ErrHandlerFunc: func(e *errors.Error) {
parser.errors = append(parser.errors, e)
},
}
lexer := scanner.NewLexer(src, v, scannerConfig)
parser.Lexer = lexer
return parser
@ -57,10 +63,6 @@ func (l *Parser) GetErrors() []*errors.Error {
return l.errors
}
func (l *Parser) WithTokens() {
l.Lexer.SetWithHiddenTokens(true)
}
// Parse the php7 Parser entrypoint
func (l *Parser) Parse() int {
// init
@ -96,7 +98,7 @@ func isDollar(r rune) bool {
}
func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false {
if l.withTokens == false {
return
}
@ -109,7 +111,7 @@ func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
}
func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) {
if l.Lexer.GetWithHiddenTokens() == false {
if l.withTokens == false {
return
}
@ -126,7 +128,7 @@ func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []tok
}
func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
if l.Lexer.GetWithHiddenTokens() == false {
if l.withTokens == false {
return []token.Token{}
}
@ -139,7 +141,7 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
}
func (l *Parser) addDollarToken(v ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false {
if l.withTokens == false {
return
}
@ -152,7 +154,7 @@ func (l *Parser) addDollarToken(v ast.Vertex) {
}
func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false {
if l.withTokens == false {
return
}

564
internal/php7/parser_test.go
View File

File diff suppressed because it is too large Load Diff

2
internal/php7/php7_bench_test.go
View File

@ -381,7 +381,7 @@ CAD;
`
for n := 0; n < b.N; n++ {
php7parser := php7.NewParser([]byte(src), "7.4")
php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse()
}
}

8
internal/php7/php7_test.go
View File

@ -19633,7 +19633,7 @@ func TestPhp7(t *testing.T) {
},
}
php7parser := php7.NewParser([]byte(src), "7.4")
php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse()
actual := php7parser.GetRootNode()
assert.DeepEqual(t, expected, actual)
@ -19770,7 +19770,7 @@ func TestPhp5Strings(t *testing.T) {
},
}
php7parser := php7.NewParser([]byte(src), "7.4")
php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse()
actual := php7parser.GetRootNode()
assert.DeepEqual(t, expected, actual)
@ -19996,7 +19996,7 @@ CAD;
},
}
php7parser := php7.NewParser([]byte(src), "7.4")
php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse()
actual := php7parser.GetRootNode()
assert.DeepEqual(t, expected, actual)
@ -20016,7 +20016,7 @@ func TestPhp7ControlCharsErrors(t *testing.T) {
},
}
php7parser := php7.NewParser([]byte(src), "7.4")
php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse()
actual := php7parser.GetErrors()
assert.DeepEqual(t, expected, actual)

71
internal/scanner/lexer.go
View File

@ -13,37 +13,42 @@ import (
type Scanner interface {
Lex() *Token
ReturnTokenToPool(t *Token)
GetWithHiddenTokens() bool
SetWithHiddenTokens(bool)
}
type Config struct {
WithHiddenTokens bool
ErrHandlerFunc func(*errors.Error)
}
type Lexer struct {
data []byte
errHandlerFunc func(*errors.Error)
data []byte
phpVersion string
withHiddenTokens bool
errHandlerFunc func(*errors.Error)
p, pe, cs int
ts, te, act int
stack []int
top int
p, pe, cs int
ts, te, act int
stack []int
top int
heredocLabel []byte
TokenPool *TokenPool
HiddenTokens []token.Token
WithHiddenTokens bool
NewLines NewLines
PHPVersion string
tokenPool *TokenPool
hiddenTokens []token.Token
newLines NewLines
}
func NewLexer(data []byte, errHandlerFunc func(*errors.Error)) *Lexer {
func NewLexer(data []byte, phpVersion string, config Config) *Lexer {
lex := &Lexer{
data: data,
errHandlerFunc: errHandlerFunc,
data: data,
phpVersion: phpVersion,
errHandlerFunc: config.ErrHandlerFunc,
withHiddenTokens: config.WithHiddenTokens,
pe: len(data),
stack: make([]int, 0),
TokenPool: &TokenPool{},
NewLines: NewLines{make([]int, 0, 128)},
tokenPool: &TokenPool{},
newLines: NewLines{make([]int, 0, 128)},
}
initLexer(lex)
@ -51,31 +56,23 @@ func NewLexer(data []byte, errHandlerFunc func(*errors.Error)) *Lexer {
return lex
}
func (l *Lexer) ReturnTokenToPool(t *Token) {
l.TokenPool.Put(t)
}
func (l *Lexer) GetWithHiddenTokens() bool {
return l.WithHiddenTokens
}
func (l *Lexer) SetWithHiddenTokens(b bool) {
l.WithHiddenTokens = b
func (lex *Lexer) ReturnTokenToPool(t *Token) {
lex.tokenPool.Put(t)
}
func (lex *Lexer) setTokenPosition(token *Token) {
token.Position.StartLine = lex.NewLines.GetLine(lex.ts)
token.Position.EndLine = lex.NewLines.GetLine(lex.te - 1)
token.Position.StartLine = lex.newLines.GetLine(lex.ts)
token.Position.EndLine = lex.newLines.GetLine(lex.te - 1)
token.Position.StartPos = lex.ts
token.Position.EndPos = lex.te
}
func (lex *Lexer) addToken(id TokenID, ps, pe int) {
if !lex.WithHiddenTokens {
func (lex *Lexer) addHiddenToken(id TokenID, ps, pe int) {
if !lex.withHiddenTokens {
return
}
lex.HiddenTokens = append(lex.HiddenTokens, token.Token{
lex.hiddenTokens = append(lex.hiddenTokens, token.Token{
ID: token.ID(id),
Value: lex.data[ps:pe],
})
@ -112,7 +109,7 @@ func (lex *Lexer) isNotStringEnd(s byte) bool {
}
func (lex *Lexer) isHeredocEnd(p int) bool {
r, err := version.Compare(lex.PHPVersion, "7.3")
r, err := version.Compare(lex.phpVersion, "7.3")
if err != nil {
return lex.isHeredocEndSince73(p)
}
@ -239,8 +236,8 @@ func (lex *Lexer) error(msg string) {
}
pos := position.NewPosition(
lex.NewLines.GetLine(lex.ts),
lex.NewLines.GetLine(lex.te-1),
lex.newLines.GetLine(lex.ts),
lex.newLines.GetLine(lex.te-1),
lex.ts,
lex.te,
)

BIN
internal/scanner/scanner.go
View File

Binary file not shown.

36
internal/scanner/scanner.rl
View File

@ -19,11 +19,11 @@ func initLexer(lex *Lexer) {
}
func (lex *Lexer) Lex() *Token {
lex.HiddenTokens = nil
lex.hiddenTokens = nil
eof := lex.pe
var tok TokenID
token := lex.TokenPool.Get()
token := lex.tokenPool.Get()
token.Hidden = nil
token.Value = lex.data[0:0]
@ -38,11 +38,11 @@ func (lex *Lexer) Lex() *Token {
action constant_string_new_line {
if lex.data[lex.p] == '\n' {
lex.NewLines.Append(lex.p)
lex.newLines.Append(lex.p)
}
if lex.data[lex.p] == '\r' && lex.data[lex.p+1] != '\n' {
lex.NewLines.Append(lex.p)
lex.newLines.Append(lex.p)
}
}
@ -52,7 +52,7 @@ func (lex *Lexer) Lex() *Token {
action is_not_string_end_or_var { lex.isNotStringEnd('"') && lex.isNotStringVar() }
action is_not_backqoute_end_or_var { lex.isNotStringEnd('`') && lex.isNotStringVar() }
newline = ('\r\n' >(nl, 1) | '\r' >(nl, 0) | '\n' >(nl, 0)) %{lex.NewLines.Append(lex.p);};
newline = ('\r\n' >(nl, 1) | '\r' >(nl, 0) | '\n' >(nl, 0)) %{lex.newLines.Append(lex.p);};
any_line = any | newline;
whitespace = [\t\v\f ];
whitespace_line = [\t\v\f ] | newline;
@ -125,7 +125,7 @@ func (lex *Lexer) Lex() *Token {
main := |*
"#!" any* :>> newline => {
lex.addToken(T_COMMENT, lex.ts, lex.te)
lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
};
any => {
fnext html;
@ -141,12 +141,12 @@ func (lex *Lexer) Lex() *Token {
fbreak;
};
'<?' => {
lex.addToken(T_OPEN_TAG, lex.ts, lex.te)
lex.addHiddenToken(T_OPEN_TAG, lex.ts, lex.te)
fnext php;
};
'<?php'i ( [ \t] | newline ) => {
lex.ungetCnt(lex.te - lex.ts - 5)
lex.addToken(T_OPEN_TAG, lex.ts, lex.ts+5)
lex.addHiddenToken(T_OPEN_TAG, lex.ts, lex.ts+5)
fnext php;
};
'<?='i => {
@ -158,7 +158,7 @@ func (lex *Lexer) Lex() *Token {
*|;
php := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)};
whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
'?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext html; fbreak;};
';' whitespace_line* '?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext html; fbreak;};
@ -318,7 +318,7 @@ func (lex *Lexer) Lex() *Token {
('#' | '//') any_line* when is_not_comment_end => {
lex.ungetStr("?>")
lex.addToken(T_COMMENT, lex.ts, lex.te)
lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
};
'/*' any_line* :>> '*/' {
isDocComment := false;
@ -327,9 +327,9 @@ func (lex *Lexer) Lex() *Token {
}
if isDocComment {
lex.addToken(T_DOC_COMMENT, lex.ts, lex.te)
lex.addHiddenToken(T_DOC_COMMENT, lex.ts, lex.te)
} else {
lex.addToken(T_COMMENT, lex.ts, lex.te)
lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
}
};
@ -378,7 +378,7 @@ func (lex *Lexer) Lex() *Token {
*|;
property := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)};
whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
"->" => {lex.setTokenPosition(token); tok = T_OBJECT_OPERATOR; fbreak;};
varname => {lex.setTokenPosition(token); tok = T_STRING; fnext php; fbreak;};
any => {lex.ungetCnt(1); fgoto php;};
@ -474,31 +474,31 @@ func (lex *Lexer) Lex() *Token {
*|;
halt_compiller_open_parenthesis := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)};
whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
"(" => {lex.setTokenPosition(token); tok = TokenID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;};
any => {lex.ungetCnt(1); fnext php;};
*|;
halt_compiller_close_parenthesis := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)};
whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
")" => {lex.setTokenPosition(token); tok = TokenID(int(')')); fnext halt_compiller_close_semicolon; fbreak;};
any => {lex.ungetCnt(1); fnext php;};
*|;
halt_compiller_close_semicolon := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)};
whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
";" => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext halt_compiller_end; fbreak;};
any => {lex.ungetCnt(1); fnext php;};
*|;
halt_compiller_end := |*
any_line* => { lex.addToken(T_HALT_COMPILER, lex.ts, lex.te); };
any_line* => { lex.addHiddenToken(T_HALT_COMPILER, lex.ts, lex.te); };
*|;
write exec;
}%%
token.Hidden = lex.HiddenTokens
token.Hidden = lex.hiddenTokens
token.Value = lex.data[lex.ts:lex.te]
token.ID = tok

124
internal/scanner/scanner_test.go
View File

@ -353,8 +353,8 @@ func TestTokens(t *testing.T) {
T_UNSET_CAST.String(),
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
actual := []string{}
for {
@ -381,8 +381,8 @@ func TestShebang(t *testing.T) {
"\n",
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
actual := []string{}
tkn := lexer.Lex()
@ -401,8 +401,8 @@ func TestShebangHtml(t *testing.T) {
0.1
`
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
tkn := lexer.Lex()
assert.Equal(t, tkn.ID, T_INLINE_HTML)
@ -451,8 +451,8 @@ func TestNumberTokens(t *testing.T) {
T_DNUMBER.String(),
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
actual := []string{}
for {
@ -508,8 +508,8 @@ func TestConstantStrings(t *testing.T) {
T_CONSTANT_ENCAPSED_STRING.String(),
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
actual := []string{}
for {
@ -555,7 +555,7 @@ func TestSingleQuoteStringTokens(t *testing.T) {
T_CONSTANT_ENCAPSED_STRING.String(),
}
lexer := NewLexer([]byte(src), nil)
lexer := NewLexer([]byte(src), "7.4", Config{})
actual := []string{}
for {
@ -642,8 +642,8 @@ func TestTeplateStringTokens(t *testing.T) {
TokenID(int('"')).String(),
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
actual := []string{}
for {
@ -727,8 +727,8 @@ func TestBackquoteStringTokens(t *testing.T) {
TokenID(int('`')).String(),
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
actual := []string{}
for {
@ -821,8 +821,8 @@ CAT;
TokenID(int(';')).String(),
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
actual := []string{}
for {
@ -894,8 +894,8 @@ CAT
T_END_HEREDOC.String(),
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
actual := []string{}
for {
@ -933,8 +933,8 @@ CAT;
TokenID(int(';')).String(),
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
actual := []string{}
for {
@ -964,8 +964,8 @@ func TestHereDocTokens73(t *testing.T) {
T_VARIABLE.String(),
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
actual := []string{}
for {
@ -994,9 +994,9 @@ CAT;`
TokenID(int(';')).String(),
}
lexer := NewLexer([]byte(src), nil)
lexer.PHPVersion = "7.2"
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.phpVersion = "7.2"
lexer.withHiddenTokens = true
actual := []string{}
for {
@ -1027,8 +1027,8 @@ func TestInlineHtmlNopTokens(t *testing.T) {
T_INLINE_HTML.String(),
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
actual := []string{}
for {
@ -1062,7 +1062,7 @@ func TestStringTokensAfterVariable(t *testing.T) {
"\"",
}
lexer := NewLexer([]byte(src), nil)
lexer := NewLexer([]byte(src), "7.4", Config{})
actual := []string{}
actualTokens := []string{}
@ -1095,7 +1095,7 @@ func TestSlashAfterVariable(t *testing.T) {
"3",
}
lexer := NewLexer([]byte(src), nil)
lexer := NewLexer([]byte(src), "7.4", Config{})
actual := []string{}
actualTokens := []string{}
@ -1132,12 +1132,12 @@ func TestCommentEnd(t *testing.T) {
},
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
lexer.Lex()
actual := lexer.HiddenTokens
actual := lexer.hiddenTokens
assert.DeepEqual(t, expected, actual)
}
@ -1161,8 +1161,8 @@ func TestCommentNewLine(t *testing.T) {
},
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
tkn := lexer.Lex()
@ -1190,8 +1190,8 @@ func TestCommentNewLine1(t *testing.T) {
},
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
tkn := lexer.Lex()
@ -1219,8 +1219,8 @@ func TestCommentNewLine2(t *testing.T) {
},
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
tkn := lexer.Lex()
@ -1249,8 +1249,8 @@ func TestCommentWithPhpEndTag(t *testing.T) {
},
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
tkn := lexer.Lex()
@ -1279,8 +1279,8 @@ func TestInlineComment(t *testing.T) {
},
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
tkn := lexer.Lex()
@ -1309,12 +1309,12 @@ func TestInlineComment2(t *testing.T) {
},
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
lexer.Lex()
actual := lexer.HiddenTokens
actual := lexer.hiddenTokens
assert.DeepEqual(t, expected, actual)
}
@ -1343,12 +1343,12 @@ func TestEmptyInlineComment(t *testing.T) {
},
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
lexer.Lex()
actual := lexer.HiddenTokens
actual := lexer.hiddenTokens
assert.DeepEqual(t, expected, actual)
}
@ -1373,8 +1373,8 @@ func TestEmptyInlineComment2(t *testing.T) {
},
}
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
tkn := lexer.Lex()
@ -1387,8 +1387,8 @@ func TestMethodCallTokens(t *testing.T) {
src := `<?php
$a -> bar ( '' ) ;`
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
expected := []token.Token{
{
@ -1469,8 +1469,8 @@ func TestYieldFromTokens(t *testing.T) {
src := `<?php
yield from $a`
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.withHiddenTokens = true
expected := []token.Token{
{
@ -1500,7 +1500,7 @@ func TestYieldFromTokens(t *testing.T) {
func TestVarNameByteChars(t *testing.T) {
src := "<?php $\x80 $\xff"
lexer := NewLexer([]byte(src), nil)
lexer := NewLexer([]byte(src), "7.4", Config{})
tkn := lexer.Lex()
assert.Equal(t, "$\x80", string(tkn.Value))
@ -1512,7 +1512,7 @@ func TestVarNameByteChars(t *testing.T) {
func TestStringVarNameByteChars(t *testing.T) {
src := "<?php \"$\x80 $\xff\""
lexer := NewLexer([]byte(src), nil)
lexer := NewLexer([]byte(src), "7.4", Config{})
tkn := lexer.Lex()
assert.Equal(t, "\"", string(tkn.Value))
@ -1534,11 +1534,13 @@ func TestIgnoreControllCharacters(t *testing.T) {
src := "<?php \004 echo $b;"
var actualErr *errors.Error
errHandler := func(e *errors.Error) {
actualErr = e
config := Config{
ErrHandlerFunc: func(e *errors.Error) {
actualErr = e
},
}
lexer := NewLexer([]byte(src), errHandler)
lexer := NewLexer([]byte(src), "7.4", config)
expected := "echo"
tkn := lexer.Lex()
@ -1560,7 +1562,7 @@ func TestIgnoreControllCharacters(t *testing.T) {
func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) {
src := "<?php \"$a[test\004]\";"
lexer := NewLexer([]byte(src), nil)
lexer := NewLexer([]byte(src), "7.4", Config{})
expected := "\""
tkn := lexer.Lex()

7
pkg/parser/parser.go
View File

@ -13,10 +13,9 @@ type Parser interface {
Parse() int
GetRootNode() ast.Vertex
GetErrors() []*errors.Error
WithTokens()
}
func NewParser(src []byte, v string) (Parser, error) {
func NewParser(src []byte, v string, withTokens bool) (Parser, error) {
var parser Parser
r, err := version.Compare(v, "7.0")
@ -25,9 +24,9 @@ func NewParser(src []byte, v string) (Parser, error) {
}
if r == -1 {
parser = php5.NewParser(src, v)
parser = php5.NewParser(src, v, withTokens)
} else {
parser = php7.NewParser(src, v)
parser = php7.NewParser(src, v, withTokens)
}
return parser, nil

3
pkg/printer/printer_parsed_php5_test.go
View File

@ -10,8 +10,7 @@ import (
)
func parsePhp5(src string) ast.Vertex {
php5parser := php5.NewParser([]byte(src), "5.6")
php5parser.WithTokens()
php5parser := php5.NewParser([]byte(src), "5.6", true)
php5parser.Parse()
return php5parser.GetRootNode()

6
pkg/printer/printer_parsed_php7_test.go
View File

@ -27,8 +27,7 @@ abstract class Bar extends Baz
// parse
php7parser := php7.NewParser([]byte(src), "7.4")
php7parser.WithTokens()
php7parser := php7.NewParser([]byte(src), "7.4", true)
php7parser.Parse()
rootNode := php7parser.GetRootNode()
@ -59,8 +58,7 @@ abstract class Bar extends Baz
}
func parse(src string) ast.Vertex {
php7parser := php7.NewParser([]byte(src), "7.4")
php7parser.WithTokens()
php7parser := php7.NewParser([]byte(src), "7.4", true)
php7parser.Parse()
return php7parser.GetRootNode()