maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[refactoring] scanner.Lexer.withHiddenTokens

Browse Source
This commit is contained in:
Vadym Slizov 2020-05-18 21:15:07 +03:00
parent 291dc7e884
commit d9a7d20e73
16 changed files with 1394 additions and 1399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/php-parser/main.go
View File

@ -108,15 +108,11 @@ func parserWorker(fileCh <-chan *file, r chan<- result) {
return return
} }
parserWorker, err := parser.NewParser(f.content, phpVersion) parserWorker, err := parser.NewParser(f.content, phpVersion, *withFreeFloating)
if err != nil { if err != nil {
panic(err.Error()) panic(err.Error())
} }
if *withFreeFloating {
parserWorker.WithTokens()
}
parserWorker.Parse() parserWorker.Parse()
r <- result{path: f.path, parser: parserWorker} r <- result{path: f.path, parser: parserWorker}

32
internal/php5/parser.go
View File

@ -22,17 +22,23 @@ type Parser struct {
positionBuilder *positionbuilder.PositionBuilder positionBuilder *positionbuilder.PositionBuilder
rootNode ast.Vertex rootNode ast.Vertex
errors []*errors.Error errors []*errors.Error
withTokens bool
} }
// NewParser creates and returns new Parser // NewParser creates and returns new Parser
func NewParser(src []byte, v string) *Parser { func NewParser(src []byte, v string, withTokens bool) *Parser {
parser := &Parser{} parser := &Parser{
withTokens: withTokens,
}
lexer := scanner.NewLexer(src, func(e *errors.Error) { scannerConfig := scanner.Config{
parser.errors = append(parser.errors, e) WithHiddenTokens: withTokens,
}) ErrHandlerFunc: func(e *errors.Error) {
lexer.PHPVersion = v parser.errors = append(parser.errors, e)
},
}
lexer := scanner.NewLexer(src, v, scannerConfig)
parser.Lexer = lexer parser.Lexer = lexer
return parser return parser
@ -59,10 +65,6 @@ func (l *Parser) GetErrors() []*errors.Error {
return l.errors return l.errors
} }
func (l *Parser) WithTokens() {
l.Lexer.SetWithHiddenTokens(true)
}
// Parse the php7 Parser entrypoint // Parse the php7 Parser entrypoint
func (l *Parser) Parse() int { func (l *Parser) Parse() int {
// init // init
@ -98,7 +100,7 @@ func isDollar(r rune) bool {
} }
func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) { func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -111,7 +113,7 @@ func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
} }
func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) { func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -128,7 +130,7 @@ func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []tok
} }
func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token { func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return []token.Token{} return []token.Token{}
} }
@ -141,7 +143,7 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
} }
func (l *Parser) addDollarToken(v ast.Vertex) { func (l *Parser) addDollarToken(v ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -154,7 +156,7 @@ func (l *Parser) addDollarToken(v ast.Vertex) {
} }
func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) { func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }

516
internal/php5/parser_test.go
View File

File diff suppressed because it is too large Load Diff

2
internal/php5/php5_bench_test.go
View File

@ -413,7 +413,7 @@ CAD;
` `
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse() php5parser.Parse()
} }
} }

8
internal/php5/php5_test.go
View File

@ -22455,7 +22455,7 @@ func TestPhp5(t *testing.T) {
}, },
} }
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse() php5parser.Parse()
actual := php5parser.GetRootNode() actual := php5parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -22592,7 +22592,7 @@ func TestPhp5Strings(t *testing.T) {
}, },
} }
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse() php5parser.Parse()
actual := php5parser.GetRootNode() actual := php5parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -22818,7 +22818,7 @@ CAD;
}, },
} }
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse() php5parser.Parse()
actual := php5parser.GetRootNode() actual := php5parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -22838,7 +22838,7 @@ func TestPhp5ControlCharsErrors(t *testing.T) {
}, },
} }
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse() php5parser.Parse()
actual := php5parser.GetErrors() actual := php5parser.GetErrors()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)

32
internal/php7/parser.go
View File

@ -21,17 +21,23 @@ type Parser struct {
positionBuilder *positionbuilder.PositionBuilder positionBuilder *positionbuilder.PositionBuilder
rootNode ast.Vertex rootNode ast.Vertex
errors []*errors.Error errors []*errors.Error
withTokens bool
} }
// NewParser creates and returns new Parser // NewParser creates and returns new Parser
func NewParser(src []byte, v string) *Parser { func NewParser(src []byte, v string, withTokens bool) *Parser {
parser := &Parser{} parser := &Parser{
withTokens: withTokens,
}
lexer := scanner.NewLexer(src, func(e *errors.Error) { scannerConfig := scanner.Config{
parser.errors = append(parser.errors, e) WithHiddenTokens: withTokens,
}) ErrHandlerFunc: func(e *errors.Error) {
lexer.PHPVersion = v parser.errors = append(parser.errors, e)
},
}
lexer := scanner.NewLexer(src, v, scannerConfig)
parser.Lexer = lexer parser.Lexer = lexer
return parser return parser
@ -57,10 +63,6 @@ func (l *Parser) GetErrors() []*errors.Error {
return l.errors return l.errors
} }
func (l *Parser) WithTokens() {
l.Lexer.SetWithHiddenTokens(true)
}
// Parse the php7 Parser entrypoint // Parse the php7 Parser entrypoint
func (l *Parser) Parse() int { func (l *Parser) Parse() int {
// init // init
@ -96,7 +98,7 @@ func isDollar(r rune) bool {
} }
func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) { func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -109,7 +111,7 @@ func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
} }
func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) { func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -126,7 +128,7 @@ func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []tok
} }
func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token { func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return []token.Token{} return []token.Token{}
} }
@ -139,7 +141,7 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
} }
func (l *Parser) addDollarToken(v ast.Vertex) { func (l *Parser) addDollarToken(v ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -152,7 +154,7 @@ func (l *Parser) addDollarToken(v ast.Vertex) {
} }
func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) { func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }

564
internal/php7/parser_test.go
View File

File diff suppressed because it is too large Load Diff

2
internal/php7/php7_bench_test.go
View File

@ -381,7 +381,7 @@ CAD;
` `
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse() php7parser.Parse()
} }
} }

8
internal/php7/php7_test.go
View File

@ -19633,7 +19633,7 @@ func TestPhp7(t *testing.T) {
}, },
} }
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse() php7parser.Parse()
actual := php7parser.GetRootNode() actual := php7parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -19770,7 +19770,7 @@ func TestPhp5Strings(t *testing.T) {
}, },
} }
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse() php7parser.Parse()
actual := php7parser.GetRootNode() actual := php7parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -19996,7 +19996,7 @@ CAD;
}, },
} }
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse() php7parser.Parse()
actual := php7parser.GetRootNode() actual := php7parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -20016,7 +20016,7 @@ func TestPhp7ControlCharsErrors(t *testing.T) {
}, },
} }
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse() php7parser.Parse()
actual := php7parser.GetErrors() actual := php7parser.GetErrors()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)

71
internal/scanner/lexer.go
View File

@ -13,37 +13,42 @@ import (
type Scanner interface { type Scanner interface {
Lex() *Token Lex() *Token
ReturnTokenToPool(t *Token) ReturnTokenToPool(t *Token)
GetWithHiddenTokens() bool }
SetWithHiddenTokens(bool)
type Config struct {
WithHiddenTokens bool
ErrHandlerFunc func(*errors.Error)
} }
type Lexer struct { type Lexer struct {
data []byte data []byte
errHandlerFunc func(*errors.Error) phpVersion string
withHiddenTokens bool
errHandlerFunc func(*errors.Error)
p, pe, cs int
ts, te, act int
stack []int
top int
p, pe, cs int
ts, te, act int
stack []int
top int
heredocLabel []byte heredocLabel []byte
tokenPool *TokenPool
TokenPool *TokenPool hiddenTokens []token.Token
HiddenTokens []token.Token newLines NewLines
WithHiddenTokens bool
NewLines NewLines
PHPVersion string
} }
func NewLexer(data []byte, errHandlerFunc func(*errors.Error)) *Lexer { func NewLexer(data []byte, phpVersion string, config Config) *Lexer {
lex := &Lexer{ lex := &Lexer{
data: data, data: data,
errHandlerFunc: errHandlerFunc, phpVersion: phpVersion,
errHandlerFunc: config.ErrHandlerFunc,
withHiddenTokens: config.WithHiddenTokens,
pe: len(data), pe: len(data),
stack: make([]int, 0), stack: make([]int, 0),
TokenPool: &TokenPool{}, tokenPool: &TokenPool{},
NewLines: NewLines{make([]int, 0, 128)}, newLines: NewLines{make([]int, 0, 128)},
} }
initLexer(lex) initLexer(lex)
@ -51,31 +56,23 @@ func NewLexer(data []byte, errHandlerFunc func(*errors.Error)) *Lexer {
return lex return lex
} }
func (l *Lexer) ReturnTokenToPool(t *Token) { func (lex *Lexer) ReturnTokenToPool(t *Token) {
l.TokenPool.Put(t) lex.tokenPool.Put(t)
}
func (l *Lexer) GetWithHiddenTokens() bool {
return l.WithHiddenTokens
}
func (l *Lexer) SetWithHiddenTokens(b bool) {
l.WithHiddenTokens = b
} }
func (lex *Lexer) setTokenPosition(token *Token) { func (lex *Lexer) setTokenPosition(token *Token) {
token.Position.StartLine = lex.NewLines.GetLine(lex.ts) token.Position.StartLine = lex.newLines.GetLine(lex.ts)
token.Position.EndLine = lex.NewLines.GetLine(lex.te - 1) token.Position.EndLine = lex.newLines.GetLine(lex.te - 1)
token.Position.StartPos = lex.ts token.Position.StartPos = lex.ts
token.Position.EndPos = lex.te token.Position.EndPos = lex.te
} }
func (lex *Lexer) addToken(id TokenID, ps, pe int) { func (lex *Lexer) addHiddenToken(id TokenID, ps, pe int) {
if !lex.WithHiddenTokens { if !lex.withHiddenTokens {
return return
} }
lex.HiddenTokens = append(lex.HiddenTokens, token.Token{ lex.hiddenTokens = append(lex.hiddenTokens, token.Token{
ID: token.ID(id), ID: token.ID(id),
Value: lex.data[ps:pe], Value: lex.data[ps:pe],
}) })
@ -112,7 +109,7 @@ func (lex *Lexer) isNotStringEnd(s byte) bool {
} }
func (lex *Lexer) isHeredocEnd(p int) bool { func (lex *Lexer) isHeredocEnd(p int) bool {
r, err := version.Compare(lex.PHPVersion, "7.3") r, err := version.Compare(lex.phpVersion, "7.3")
if err != nil { if err != nil {
return lex.isHeredocEndSince73(p) return lex.isHeredocEndSince73(p)
} }
@ -239,8 +236,8 @@ func (lex *Lexer) error(msg string) {
} }
pos := position.NewPosition( pos := position.NewPosition(
lex.NewLines.GetLine(lex.ts), lex.newLines.GetLine(lex.ts),
lex.NewLines.GetLine(lex.te-1), lex.newLines.GetLine(lex.te-1),
lex.ts, lex.ts,
lex.te, lex.te,
) )

1376
internal/scanner/scanner.go
View File

File diff suppressed because it is too large Load Diff

36
internal/scanner/scanner.rl
View File

@ -19,11 +19,11 @@ func initLexer(lex *Lexer) {
} }
func (lex *Lexer) Lex() *Token { func (lex *Lexer) Lex() *Token {
lex.HiddenTokens = nil lex.hiddenTokens = nil
eof := lex.pe eof := lex.pe
var tok TokenID var tok TokenID
token := lex.TokenPool.Get() token := lex.tokenPool.Get()
token.Hidden = nil token.Hidden = nil
token.Value = lex.data[0:0] token.Value = lex.data[0:0]
@ -38,11 +38,11 @@ func (lex *Lexer) Lex() *Token {
action constant_string_new_line { action constant_string_new_line {
if lex.data[lex.p] == '\n' { if lex.data[lex.p] == '\n' {
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
} }
if lex.data[lex.p] == '\r' && lex.data[lex.p+1] != '\n' { if lex.data[lex.p] == '\r' && lex.data[lex.p+1] != '\n' {
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
} }
} }
@ -52,7 +52,7 @@ func (lex *Lexer) Lex() *Token {
action is_not_string_end_or_var { lex.isNotStringEnd('"') && lex.isNotStringVar() } action is_not_string_end_or_var { lex.isNotStringEnd('"') && lex.isNotStringVar() }
action is_not_backqoute_end_or_var { lex.isNotStringEnd('`') && lex.isNotStringVar() } action is_not_backqoute_end_or_var { lex.isNotStringEnd('`') && lex.isNotStringVar() }
newline = ('\r\n' >(nl, 1) | '\r' >(nl, 0) | '\n' >(nl, 0)) %{lex.NewLines.Append(lex.p);}; newline = ('\r\n' >(nl, 1) | '\r' >(nl, 0) | '\n' >(nl, 0)) %{lex.newLines.Append(lex.p);};
any_line = any | newline; any_line = any | newline;
whitespace = [\t\v\f ]; whitespace = [\t\v\f ];
whitespace_line = [\t\v\f ] | newline; whitespace_line = [\t\v\f ] | newline;
@ -125,7 +125,7 @@ func (lex *Lexer) Lex() *Token {
main := |* main := |*
"#!" any* :>> newline => { "#!" any* :>> newline => {
lex.addToken(T_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
}; };
any => { any => {
fnext html; fnext html;
@ -141,12 +141,12 @@ func (lex *Lexer) Lex() *Token {
fbreak; fbreak;
}; };
'<?' => { '<?' => {
lex.addToken(T_OPEN_TAG, lex.ts, lex.te) lex.addHiddenToken(T_OPEN_TAG, lex.ts, lex.te)
fnext php; fnext php;
}; };
'<?php'i ( [ \t] | newline ) => { '<?php'i ( [ \t] | newline ) => {
lex.ungetCnt(lex.te - lex.ts - 5) lex.ungetCnt(lex.te - lex.ts - 5)
lex.addToken(T_OPEN_TAG, lex.ts, lex.ts+5) lex.addHiddenToken(T_OPEN_TAG, lex.ts, lex.ts+5)
fnext php; fnext php;
}; };
'<?='i => { '<?='i => {
@ -158,7 +158,7 @@ func (lex *Lexer) Lex() *Token {
*|; *|;
php := |* php := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
'?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext html; fbreak;}; '?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext html; fbreak;};
';' whitespace_line* '?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext html; fbreak;}; ';' whitespace_line* '?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext html; fbreak;};
@ -318,7 +318,7 @@ func (lex *Lexer) Lex() *Token {
('#' | '//') any_line* when is_not_comment_end => { ('#' | '//') any_line* when is_not_comment_end => {
lex.ungetStr("?>") lex.ungetStr("?>")
lex.addToken(T_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
}; };
'/*' any_line* :>> '*/' { '/*' any_line* :>> '*/' {
isDocComment := false; isDocComment := false;
@ -327,9 +327,9 @@ func (lex *Lexer) Lex() *Token {
} }
if isDocComment { if isDocComment {
lex.addToken(T_DOC_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_DOC_COMMENT, lex.ts, lex.te)
} else { } else {
lex.addToken(T_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
} }
}; };
@ -378,7 +378,7 @@ func (lex *Lexer) Lex() *Token {
*|; *|;
property := |* property := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
"->" => {lex.setTokenPosition(token); tok = T_OBJECT_OPERATOR; fbreak;}; "->" => {lex.setTokenPosition(token); tok = T_OBJECT_OPERATOR; fbreak;};
varname => {lex.setTokenPosition(token); tok = T_STRING; fnext php; fbreak;}; varname => {lex.setTokenPosition(token); tok = T_STRING; fnext php; fbreak;};
any => {lex.ungetCnt(1); fgoto php;}; any => {lex.ungetCnt(1); fgoto php;};
@ -474,31 +474,31 @@ func (lex *Lexer) Lex() *Token {
*|; *|;
halt_compiller_open_parenthesis := |* halt_compiller_open_parenthesis := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
"(" => {lex.setTokenPosition(token); tok = TokenID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;}; "(" => {lex.setTokenPosition(token); tok = TokenID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_close_parenthesis := |* halt_compiller_close_parenthesis := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
")" => {lex.setTokenPosition(token); tok = TokenID(int(')')); fnext halt_compiller_close_semicolon; fbreak;}; ")" => {lex.setTokenPosition(token); tok = TokenID(int(')')); fnext halt_compiller_close_semicolon; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_close_semicolon := |* halt_compiller_close_semicolon := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
";" => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext halt_compiller_end; fbreak;}; ";" => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext halt_compiller_end; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_end := |* halt_compiller_end := |*
any_line* => { lex.addToken(T_HALT_COMPILER, lex.ts, lex.te); }; any_line* => { lex.addHiddenToken(T_HALT_COMPILER, lex.ts, lex.te); };
*|; *|;
write exec; write exec;
}%% }%%
token.Hidden = lex.HiddenTokens token.Hidden = lex.hiddenTokens
token.Value = lex.data[lex.ts:lex.te] token.Value = lex.data[lex.ts:lex.te]
token.ID = tok token.ID = tok

124
internal/scanner/scanner_test.go
View File

@ -353,8 +353,8 @@ func TestTokens(t *testing.T) {
T_UNSET_CAST.String(), T_UNSET_CAST.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -381,8 +381,8 @@ func TestShebang(t *testing.T) {
"\n", "\n",
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
tkn := lexer.Lex() tkn := lexer.Lex()
@ -401,8 +401,8 @@ func TestShebangHtml(t *testing.T) {
0.1 0.1
` `
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
assert.Equal(t, tkn.ID, T_INLINE_HTML) assert.Equal(t, tkn.ID, T_INLINE_HTML)
@ -451,8 +451,8 @@ func TestNumberTokens(t *testing.T) {
T_DNUMBER.String(), T_DNUMBER.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -508,8 +508,8 @@ func TestConstantStrings(t *testing.T) {
T_CONSTANT_ENCAPSED_STRING.String(), T_CONSTANT_ENCAPSED_STRING.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -555,7 +555,7 @@ func TestSingleQuoteStringTokens(t *testing.T) {
T_CONSTANT_ENCAPSED_STRING.String(), T_CONSTANT_ENCAPSED_STRING.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
actual := []string{} actual := []string{}
for { for {
@ -642,8 +642,8 @@ func TestTeplateStringTokens(t *testing.T) {
TokenID(int('"')).String(), TokenID(int('"')).String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -727,8 +727,8 @@ func TestBackquoteStringTokens(t *testing.T) {
TokenID(int('`')).String(), TokenID(int('`')).String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -821,8 +821,8 @@ CAT;
TokenID(int(';')).String(), TokenID(int(';')).String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -894,8 +894,8 @@ CAT
T_END_HEREDOC.String(), T_END_HEREDOC.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -933,8 +933,8 @@ CAT;
TokenID(int(';')).String(), TokenID(int(';')).String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -964,8 +964,8 @@ func TestHereDocTokens73(t *testing.T) {
T_VARIABLE.String(), T_VARIABLE.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -994,9 +994,9 @@ CAT;`
TokenID(int(';')).String(), TokenID(int(';')).String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.PHPVersion = "7.2" lexer.phpVersion = "7.2"
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -1027,8 +1027,8 @@ func TestInlineHtmlNopTokens(t *testing.T) {
T_INLINE_HTML.String(), T_INLINE_HTML.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -1062,7 +1062,7 @@ func TestStringTokensAfterVariable(t *testing.T) {
"\"", "\"",
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
actual := []string{} actual := []string{}
actualTokens := []string{} actualTokens := []string{}
@ -1095,7 +1095,7 @@ func TestSlashAfterVariable(t *testing.T) {
"3", "3",
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
actual := []string{} actual := []string{}
actualTokens := []string{} actualTokens := []string{}
@ -1132,12 +1132,12 @@ func TestCommentEnd(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
lexer.Lex() lexer.Lex()
actual := lexer.HiddenTokens actual := lexer.hiddenTokens
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
} }
@ -1161,8 +1161,8 @@ func TestCommentNewLine(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1190,8 +1190,8 @@ func TestCommentNewLine1(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1219,8 +1219,8 @@ func TestCommentNewLine2(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1249,8 +1249,8 @@ func TestCommentWithPhpEndTag(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1279,8 +1279,8 @@ func TestInlineComment(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1309,12 +1309,12 @@ func TestInlineComment2(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
lexer.Lex() lexer.Lex()
actual := lexer.HiddenTokens actual := lexer.hiddenTokens
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
} }
@ -1343,12 +1343,12 @@ func TestEmptyInlineComment(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
lexer.Lex() lexer.Lex()
actual := lexer.HiddenTokens actual := lexer.hiddenTokens
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
} }
@ -1373,8 +1373,8 @@ func TestEmptyInlineComment2(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1387,8 +1387,8 @@ func TestMethodCallTokens(t *testing.T) {
src := `<?php src := `<?php
$a -> bar ( '' ) ;` $a -> bar ( '' ) ;`
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
expected := []token.Token{ expected := []token.Token{
{ {
@ -1469,8 +1469,8 @@ func TestYieldFromTokens(t *testing.T) {
src := `<?php src := `<?php
yield from $a` yield from $a`
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
expected := []token.Token{ expected := []token.Token{
{ {
@ -1500,7 +1500,7 @@ func TestYieldFromTokens(t *testing.T) {
func TestVarNameByteChars(t *testing.T) { func TestVarNameByteChars(t *testing.T) {
src := "<?php $\x80 $\xff" src := "<?php $\x80 $\xff"
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
tkn := lexer.Lex() tkn := lexer.Lex()
assert.Equal(t, "$\x80", string(tkn.Value)) assert.Equal(t, "$\x80", string(tkn.Value))
@ -1512,7 +1512,7 @@ func TestVarNameByteChars(t *testing.T) {
func TestStringVarNameByteChars(t *testing.T) { func TestStringVarNameByteChars(t *testing.T) {
src := "<?php \"$\x80 $\xff\"" src := "<?php \"$\x80 $\xff\""
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
tkn := lexer.Lex() tkn := lexer.Lex()
assert.Equal(t, "\"", string(tkn.Value)) assert.Equal(t, "\"", string(tkn.Value))
@ -1534,11 +1534,13 @@ func TestIgnoreControllCharacters(t *testing.T) {
src := "<?php \004 echo $b;" src := "<?php \004 echo $b;"
var actualErr *errors.Error var actualErr *errors.Error
errHandler := func(e *errors.Error) { config := Config{
actualErr = e ErrHandlerFunc: func(e *errors.Error) {
actualErr = e
},
} }
lexer := NewLexer([]byte(src), errHandler) lexer := NewLexer([]byte(src), "7.4", config)
expected := "echo" expected := "echo"
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1560,7 +1562,7 @@ func TestIgnoreControllCharacters(t *testing.T) {
func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) { func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) {
src := "<?php \"$a[test\004]\";" src := "<?php \"$a[test\004]\";"
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
expected := "\"" expected := "\""
tkn := lexer.Lex() tkn := lexer.Lex()

7
pkg/parser/parser.go
View File

@ -13,10 +13,9 @@ type Parser interface {
Parse() int Parse() int
GetRootNode() ast.Vertex GetRootNode() ast.Vertex
GetErrors() []*errors.Error GetErrors() []*errors.Error
WithTokens()
} }
func NewParser(src []byte, v string) (Parser, error) { func NewParser(src []byte, v string, withTokens bool) (Parser, error) {
var parser Parser var parser Parser
r, err := version.Compare(v, "7.0") r, err := version.Compare(v, "7.0")
@ -25,9 +24,9 @@ func NewParser(src []byte, v string) (Parser, error) {
} }
if r == -1 { if r == -1 {
parser = php5.NewParser(src, v) parser = php5.NewParser(src, v, withTokens)
} else { } else {
parser = php7.NewParser(src, v) parser = php7.NewParser(src, v, withTokens)
} }
return parser, nil return parser, nil

3
pkg/printer/printer_parsed_php5_test.go
View File

@ -10,8 +10,7 @@ import (
) )
func parsePhp5(src string) ast.Vertex { func parsePhp5(src string) ast.Vertex {
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", true)
php5parser.WithTokens()
php5parser.Parse() php5parser.Parse()
return php5parser.GetRootNode() return php5parser.GetRootNode()

6
pkg/printer/printer_parsed_php7_test.go
View File

@ -27,8 +27,7 @@ abstract class Bar extends Baz
// parse // parse
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", true)
php7parser.WithTokens()
php7parser.Parse() php7parser.Parse()
rootNode := php7parser.GetRootNode() rootNode := php7parser.GetRootNode()
@ -59,8 +58,7 @@ abstract class Bar extends Baz
} }
func parse(src string) ast.Vertex { func parse(src string) ast.Vertex {
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", true)
php7parser.WithTokens()
php7parser.Parse() php7parser.Parse()
return php7parser.GetRootNode() return php7parser.GetRootNode()