maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[refactoring] scanner.Lexer.withHiddenTokens

Browse Source
This commit is contained in:
Vadym Slizov 2020-05-18 21:15:07 +03:00
parent 291dc7e884
commit d9a7d20e73
16 changed files with 1394 additions and 1399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/php-parser/main.go
View File

@ -108,15 +108,11 @@ func parserWorker(fileCh <-chan *file, r chan<- result) {
return return
} }
parserWorker, err := parser.NewParser(f.content, phpVersion) parserWorker, err := parser.NewParser(f.content, phpVersion, *withFreeFloating)
if err != nil { if err != nil {
panic(err.Error()) panic(err.Error())
} }
if *withFreeFloating {
parserWorker.WithTokens()
}
parserWorker.Parse() parserWorker.Parse()
r <- result{path: f.path, parser: parserWorker} r <- result{path: f.path, parser: parserWorker}

30
internal/php5/parser.go
View File

@ -22,17 +22,23 @@ type Parser struct {
positionBuilder *positionbuilder.PositionBuilder positionBuilder *positionbuilder.PositionBuilder
rootNode ast.Vertex rootNode ast.Vertex
errors []*errors.Error errors []*errors.Error
withTokens bool
} }
// NewParser creates and returns new Parser // NewParser creates and returns new Parser
func NewParser(src []byte, v string) *Parser { func NewParser(src []byte, v string, withTokens bool) *Parser {
parser := &Parser{} parser := &Parser{
withTokens: withTokens,
}
lexer := scanner.NewLexer(src, func(e *errors.Error) { scannerConfig := scanner.Config{
WithHiddenTokens: withTokens,
ErrHandlerFunc: func(e *errors.Error) {
parser.errors = append(parser.errors, e) parser.errors = append(parser.errors, e)
}) },
lexer.PHPVersion = v }
lexer := scanner.NewLexer(src, v, scannerConfig)
parser.Lexer = lexer parser.Lexer = lexer
return parser return parser
@ -59,10 +65,6 @@ func (l *Parser) GetErrors() []*errors.Error {
return l.errors return l.errors
} }
func (l *Parser) WithTokens() {
l.Lexer.SetWithHiddenTokens(true)
}
// Parse the php7 Parser entrypoint // Parse the php7 Parser entrypoint
func (l *Parser) Parse() int { func (l *Parser) Parse() int {
// init // init
@ -98,7 +100,7 @@ func isDollar(r rune) bool {
} }
func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) { func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -111,7 +113,7 @@ func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
} }
func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) { func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -128,7 +130,7 @@ func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []tok
} }
func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token { func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return []token.Token{} return []token.Token{}
} }
@ -141,7 +143,7 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
} }
func (l *Parser) addDollarToken(v ast.Vertex) { func (l *Parser) addDollarToken(v ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -154,7 +156,7 @@ func (l *Parser) addDollarToken(v ast.Vertex) {
} }
func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) { func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }

516
internal/php5/parser_test.go
View File

File diff suppressed because it is too large Load Diff

2
internal/php5/php5_bench_test.go
View File

@ -413,7 +413,7 @@ CAD;
` `
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse() php5parser.Parse()
} }
} }

8
internal/php5/php5_test.go
View File

@ -22455,7 +22455,7 @@ func TestPhp5(t *testing.T) {
}, },
} }
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse() php5parser.Parse()
actual := php5parser.GetRootNode() actual := php5parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -22592,7 +22592,7 @@ func TestPhp5Strings(t *testing.T) {
}, },
} }
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse() php5parser.Parse()
actual := php5parser.GetRootNode() actual := php5parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -22818,7 +22818,7 @@ CAD;
}, },
} }
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse() php5parser.Parse()
actual := php5parser.GetRootNode() actual := php5parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -22838,7 +22838,7 @@ func TestPhp5ControlCharsErrors(t *testing.T) {
}, },
} }
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", false)
php5parser.Parse() php5parser.Parse()
actual := php5parser.GetErrors() actual := php5parser.GetErrors()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)

30
internal/php7/parser.go
View File

@ -21,17 +21,23 @@ type Parser struct {
positionBuilder *positionbuilder.PositionBuilder positionBuilder *positionbuilder.PositionBuilder
rootNode ast.Vertex rootNode ast.Vertex
errors []*errors.Error errors []*errors.Error
withTokens bool
} }
// NewParser creates and returns new Parser // NewParser creates and returns new Parser
func NewParser(src []byte, v string) *Parser { func NewParser(src []byte, v string, withTokens bool) *Parser {
parser := &Parser{} parser := &Parser{
withTokens: withTokens,
}
lexer := scanner.NewLexer(src, func(e *errors.Error) { scannerConfig := scanner.Config{
WithHiddenTokens: withTokens,
ErrHandlerFunc: func(e *errors.Error) {
parser.errors = append(parser.errors, e) parser.errors = append(parser.errors, e)
}) },
lexer.PHPVersion = v }
lexer := scanner.NewLexer(src, v, scannerConfig)
parser.Lexer = lexer parser.Lexer = lexer
return parser return parser
@ -57,10 +63,6 @@ func (l *Parser) GetErrors() []*errors.Error {
return l.errors return l.errors
} }
func (l *Parser) WithTokens() {
l.Lexer.SetWithHiddenTokens(true)
}
// Parse the php7 Parser entrypoint // Parse the php7 Parser entrypoint
func (l *Parser) Parse() int { func (l *Parser) Parse() int {
// init // init
@ -96,7 +98,7 @@ func isDollar(r rune) bool {
} }
func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) { func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -109,7 +111,7 @@ func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
} }
func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) { func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -126,7 +128,7 @@ func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []tok
} }
func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token { func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return []token.Token{} return []token.Token{}
} }
@ -139,7 +141,7 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
} }
func (l *Parser) addDollarToken(v ast.Vertex) { func (l *Parser) addDollarToken(v ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }
@ -152,7 +154,7 @@ func (l *Parser) addDollarToken(v ast.Vertex) {
} }
func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) { func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
if l.Lexer.GetWithHiddenTokens() == false { if l.withTokens == false {
return return
} }

564
internal/php7/parser_test.go
View File

File diff suppressed because it is too large Load Diff

2
internal/php7/php7_bench_test.go
View File

@ -381,7 +381,7 @@ CAD;
` `
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse() php7parser.Parse()
} }
} }

8
internal/php7/php7_test.go
View File

@ -19633,7 +19633,7 @@ func TestPhp7(t *testing.T) {
}, },
} }
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse() php7parser.Parse()
actual := php7parser.GetRootNode() actual := php7parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -19770,7 +19770,7 @@ func TestPhp5Strings(t *testing.T) {
}, },
} }
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse() php7parser.Parse()
actual := php7parser.GetRootNode() actual := php7parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -19996,7 +19996,7 @@ CAD;
}, },
} }
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse() php7parser.Parse()
actual := php7parser.GetRootNode() actual := php7parser.GetRootNode()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
@ -20016,7 +20016,7 @@ func TestPhp7ControlCharsErrors(t *testing.T) {
}, },
} }
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", false)
php7parser.Parse() php7parser.Parse()
actual := php7parser.GetErrors() actual := php7parser.GetErrors()
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)

57
internal/scanner/lexer.go
View File

@ -13,37 +13,42 @@ import (
type Scanner interface { type Scanner interface {
Lex() *Token Lex() *Token
ReturnTokenToPool(t *Token) ReturnTokenToPool(t *Token)
GetWithHiddenTokens() bool }
SetWithHiddenTokens(bool)
type Config struct {
WithHiddenTokens bool
ErrHandlerFunc func(*errors.Error)
} }
type Lexer struct { type Lexer struct {
data []byte data []byte
phpVersion string
withHiddenTokens bool
errHandlerFunc func(*errors.Error) errHandlerFunc func(*errors.Error)
p, pe, cs int p, pe, cs int
ts, te, act int ts, te, act int
stack []int stack []int
top int top int
heredocLabel []byte
TokenPool *TokenPool heredocLabel []byte
HiddenTokens []token.Token tokenPool *TokenPool
WithHiddenTokens bool hiddenTokens []token.Token
NewLines NewLines newLines NewLines
PHPVersion string
} }
func NewLexer(data []byte, errHandlerFunc func(*errors.Error)) *Lexer { func NewLexer(data []byte, phpVersion string, config Config) *Lexer {
lex := &Lexer{ lex := &Lexer{
data: data, data: data,
errHandlerFunc: errHandlerFunc, phpVersion: phpVersion,
errHandlerFunc: config.ErrHandlerFunc,
withHiddenTokens: config.WithHiddenTokens,
pe: len(data), pe: len(data),
stack: make([]int, 0), stack: make([]int, 0),
TokenPool: &TokenPool{}, tokenPool: &TokenPool{},
NewLines: NewLines{make([]int, 0, 128)}, newLines: NewLines{make([]int, 0, 128)},
} }
initLexer(lex) initLexer(lex)
@ -51,31 +56,23 @@ func NewLexer(data []byte, errHandlerFunc func(*errors.Error)) *Lexer {
return lex return lex
} }
func (l *Lexer) ReturnTokenToPool(t *Token) { func (lex *Lexer) ReturnTokenToPool(t *Token) {
l.TokenPool.Put(t) lex.tokenPool.Put(t)
}
func (l *Lexer) GetWithHiddenTokens() bool {
return l.WithHiddenTokens
}
func (l *Lexer) SetWithHiddenTokens(b bool) {
l.WithHiddenTokens = b
} }
func (lex *Lexer) setTokenPosition(token *Token) { func (lex *Lexer) setTokenPosition(token *Token) {
token.Position.StartLine = lex.NewLines.GetLine(lex.ts) token.Position.StartLine = lex.newLines.GetLine(lex.ts)
token.Position.EndLine = lex.NewLines.GetLine(lex.te - 1) token.Position.EndLine = lex.newLines.GetLine(lex.te - 1)
token.Position.StartPos = lex.ts token.Position.StartPos = lex.ts
token.Position.EndPos = lex.te token.Position.EndPos = lex.te
} }
func (lex *Lexer) addToken(id TokenID, ps, pe int) { func (lex *Lexer) addHiddenToken(id TokenID, ps, pe int) {
if !lex.WithHiddenTokens { if !lex.withHiddenTokens {
return return
} }
lex.HiddenTokens = append(lex.HiddenTokens, token.Token{ lex.hiddenTokens = append(lex.hiddenTokens, token.Token{
ID: token.ID(id), ID: token.ID(id),
Value: lex.data[ps:pe], Value: lex.data[ps:pe],
}) })
@ -112,7 +109,7 @@ func (lex *Lexer) isNotStringEnd(s byte) bool {
} }
func (lex *Lexer) isHeredocEnd(p int) bool { func (lex *Lexer) isHeredocEnd(p int) bool {
r, err := version.Compare(lex.PHPVersion, "7.3") r, err := version.Compare(lex.phpVersion, "7.3")
if err != nil { if err != nil {
return lex.isHeredocEndSince73(p) return lex.isHeredocEndSince73(p)
} }
@ -239,8 +236,8 @@ func (lex *Lexer) error(msg string) {
} }
pos := position.NewPosition( pos := position.NewPosition(
lex.NewLines.GetLine(lex.ts), lex.newLines.GetLine(lex.ts),
lex.NewLines.GetLine(lex.te-1), lex.newLines.GetLine(lex.te-1),
lex.ts, lex.ts,
lex.te, lex.te,
) )

184
internal/scanner/scanner.go
View File

@ -46,11 +46,11 @@ func initLexer(lex *Lexer) {
} }
func (lex *Lexer) Lex() *Token { func (lex *Lexer) Lex() *Token {
lex.HiddenTokens = nil lex.hiddenTokens = nil
eof := lex.pe eof := lex.pe
var tok TokenID var tok TokenID
token := lex.TokenPool.Get() token := lex.tokenPool.Get()
token.Hidden = nil token.Hidden = nil
token.Value = lex.data[0:0] token.Value = lex.data[0:0]
@ -2224,12 +2224,12 @@ func (lex *Lexer) Lex() *Token {
goto _again goto _again
tr165: tr165:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:127 // line internal/scanner/scanner.rl:127
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
} }
goto st111 goto st111
st111: st111:
@ -2283,7 +2283,7 @@ func (lex *Lexer) Lex() *Token {
// line internal/scanner/scanner.rl:143 // line internal/scanner/scanner.rl:143
(lex.p) = (lex.te) - 1 (lex.p) = (lex.te) - 1
{ {
lex.addToken(T_OPEN_TAG, lex.ts, lex.te) lex.addHiddenToken(T_OPEN_TAG, lex.ts, lex.te)
lex.cs = 121 lex.cs = 121
} }
goto _again goto _again
@ -2293,7 +2293,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
{ {
lex.ungetCnt(lex.te - lex.ts - 5) lex.ungetCnt(lex.te - lex.ts - 5)
lex.addToken(T_OPEN_TAG, lex.ts, lex.ts+5) lex.addHiddenToken(T_OPEN_TAG, lex.ts, lex.ts+5)
lex.cs = 121 lex.cs = 121
} }
goto _again goto _again
@ -2314,7 +2314,7 @@ func (lex *Lexer) Lex() *Token {
goto st114 goto st114
tr171: tr171:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:137 // line internal/scanner/scanner.rl:137
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
@ -2335,7 +2335,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_OPEN_TAG, lex.ts, lex.te) lex.addHiddenToken(T_OPEN_TAG, lex.ts, lex.te)
lex.cs = 121 lex.cs = 121
} }
goto _again goto _again
@ -2356,13 +2356,13 @@ func (lex *Lexer) Lex() *Token {
tr179: tr179:
lex.cs = 114 lex.cs = 114
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:147 // line internal/scanner/scanner.rl:147
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.ungetCnt(lex.te - lex.ts - 5) lex.ungetCnt(lex.te - lex.ts - 5)
lex.addToken(T_OPEN_TAG, lex.ts, lex.ts+5) lex.addHiddenToken(T_OPEN_TAG, lex.ts, lex.ts+5)
lex.cs = 121 lex.cs = 121
} }
goto _again goto _again
@ -2387,7 +2387,7 @@ func (lex *Lexer) Lex() *Token {
goto st115 goto st115
tr172: tr172:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st115 goto st115
st115: st115:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -2404,7 +2404,7 @@ func (lex *Lexer) Lex() *Token {
goto st115 goto st115
tr173: tr173:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st116 goto st116
st116: st116:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -2421,7 +2421,7 @@ func (lex *Lexer) Lex() *Token {
goto tr172 goto tr172
tr174: tr174:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st117 goto st117
st117: st117:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -2531,7 +2531,7 @@ func (lex *Lexer) Lex() *Token {
// line internal/scanner/scanner.rl:161 // line internal/scanner/scanner.rl:161
(lex.p) = (lex.te) - 1 (lex.p) = (lex.te) - 1
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st121 goto st121
tr11: tr11:
@ -3510,9 +3510,9 @@ func (lex *Lexer) Lex() *Token {
} }
if isDocComment { if isDocComment {
lex.addToken(T_DOC_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_DOC_COMMENT, lex.ts, lex.te)
} else { } else {
lex.addToken(T_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
} }
} }
goto st121 goto st121
@ -3689,17 +3689,17 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st121 goto st121
tr234: tr234:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:161 // line internal/scanner/scanner.rl:161
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st121 goto st121
tr238: tr238:
@ -3775,18 +3775,18 @@ func (lex *Lexer) Lex() *Token {
(lex.p)-- (lex.p)--
{ {
lex.ungetStr("?>") lex.ungetStr("?>")
lex.addToken(T_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
} }
goto st121 goto st121
tr246: tr246:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:319 // line internal/scanner/scanner.rl:319
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.ungetStr("?>") lex.ungetStr("?>")
lex.addToken(T_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
} }
goto st121 goto st121
tr250: tr250:
@ -4105,7 +4105,7 @@ func (lex *Lexer) Lex() *Token {
tr279: tr279:
lex.cs = 121 lex.cs = 121
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:163 // line internal/scanner/scanner.rl:163
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
@ -4162,7 +4162,7 @@ func (lex *Lexer) Lex() *Token {
tr285: tr285:
lex.cs = 121 lex.cs = 121
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:357 // line internal/scanner/scanner.rl:357
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
@ -4309,7 +4309,7 @@ func (lex *Lexer) Lex() *Token {
tr300: tr300:
lex.cs = 121 lex.cs = 121
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:162 // line internal/scanner/scanner.rl:162
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
@ -4698,7 +4698,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st122 goto st122
st122: st122:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -4728,7 +4728,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st123 goto st123
st123: st123:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -4750,7 +4750,7 @@ func (lex *Lexer) Lex() *Token {
goto tr234 goto tr234
tr237: tr237:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st6 goto st6
st6: st6:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -4821,11 +4821,11 @@ func (lex *Lexer) Lex() *Token {
// line internal/scanner/scanner.rl:39 // line internal/scanner/scanner.rl:39
if lex.data[lex.p] == '\n' { if lex.data[lex.p] == '\n' {
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
} }
if lex.data[lex.p] == '\r' && lex.data[lex.p+1] != '\n' { if lex.data[lex.p] == '\r' && lex.data[lex.p+1] != '\n' {
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
} }
goto st7 goto st7
@ -4912,7 +4912,7 @@ func (lex *Lexer) Lex() *Token {
goto st7 goto st7
tr247: tr247:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st128 goto st128
st128: st128:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -4967,7 +4967,7 @@ func (lex *Lexer) Lex() *Token {
goto tr244 goto tr244
tr248: tr248:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st129 goto st129
st129: st129:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -5114,11 +5114,11 @@ func (lex *Lexer) Lex() *Token {
// line internal/scanner/scanner.rl:39 // line internal/scanner/scanner.rl:39
if lex.data[lex.p] == '\n' { if lex.data[lex.p] == '\n' {
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
} }
if lex.data[lex.p] == '\r' && lex.data[lex.p+1] != '\n' { if lex.data[lex.p] == '\r' && lex.data[lex.p+1] != '\n' {
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
} }
goto st11 goto st11
@ -6129,7 +6129,7 @@ func (lex *Lexer) Lex() *Token {
goto tr239 goto tr239
tr92: tr92:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st71 goto st71
st71: st71:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -6146,7 +6146,7 @@ func (lex *Lexer) Lex() *Token {
goto st71 goto st71
tr93: tr93:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st72 goto st72
st72: st72:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -6163,7 +6163,7 @@ func (lex *Lexer) Lex() *Token {
goto tr92 goto tr92
tr94: tr94:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st73 goto st73
st73: st73:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -6382,7 +6382,7 @@ func (lex *Lexer) Lex() *Token {
goto tr239 goto tr239
tr104: tr104:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st77 goto st77
st77: st77:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -6406,7 +6406,7 @@ func (lex *Lexer) Lex() *Token {
goto tr22 goto tr22
tr105: tr105:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st78 goto st78
st78: st78:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -6430,7 +6430,7 @@ func (lex *Lexer) Lex() *Token {
goto tr22 goto tr22
tr106: tr106:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st79 goto st79
st79: st79:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -6444,7 +6444,7 @@ func (lex *Lexer) Lex() *Token {
goto tr22 goto tr22
tr107: tr107:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st80 goto st80
st80: st80:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -14766,7 +14766,7 @@ func (lex *Lexer) Lex() *Token {
goto tr211 goto tr211
tr134: tr134:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st93 goto st93
st93: st93:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -14792,7 +14792,7 @@ func (lex *Lexer) Lex() *Token {
goto tr129 goto tr129
tr135: tr135:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st94 goto st94
st94: st94:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -14818,7 +14818,7 @@ func (lex *Lexer) Lex() *Token {
goto tr129 goto tr129
tr136: tr136:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st95 goto st95
st95: st95:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -14832,7 +14832,7 @@ func (lex *Lexer) Lex() *Token {
goto tr129 goto tr129
tr137: tr137:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st96 goto st96
st96: st96:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -16946,7 +16946,7 @@ func (lex *Lexer) Lex() *Token {
// line internal/scanner/scanner.rl:381 // line internal/scanner/scanner.rl:381
(lex.p) = (lex.te) - 1 (lex.p) = (lex.te) - 1
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st468 goto st468
tr663: tr663:
@ -16964,17 +16964,17 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st468 goto st468
tr670: tr670:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:381 // line internal/scanner/scanner.rl:381
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st468 goto st468
tr674: tr674:
@ -17073,7 +17073,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st469 goto st469
st469: st469:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -17103,7 +17103,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st470 goto st470
st470: st470:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -17125,7 +17125,7 @@ func (lex *Lexer) Lex() *Token {
goto tr670 goto tr670
tr673: tr673:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st99 goto st99
st99: st99:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -17209,7 +17209,7 @@ func (lex *Lexer) Lex() *Token {
tr680: tr680:
lex.cs = 474 lex.cs = 474
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:388 // line internal/scanner/scanner.rl:388
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
@ -17299,7 +17299,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:388 // line internal/scanner/scanner.rl:388
lex.act = 146 lex.act = 146
goto st475 goto st475
@ -17356,7 +17356,7 @@ func (lex *Lexer) Lex() *Token {
goto tr679 goto tr679
tr682: tr682:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st476 goto st476
st476: st476:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -17488,7 +17488,7 @@ func (lex *Lexer) Lex() *Token {
tr692: tr692:
lex.cs = 477 lex.cs = 477
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:400 // line internal/scanner/scanner.rl:400
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
@ -17619,7 +17619,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:400 // line internal/scanner/scanner.rl:400
lex.act = 150 lex.act = 150
goto st479 goto st479
@ -17690,7 +17690,7 @@ func (lex *Lexer) Lex() *Token {
goto tr691 goto tr691
tr694: tr694:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st480 goto st480
st480: st480:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -17952,7 +17952,7 @@ func (lex *Lexer) Lex() *Token {
goto _again goto _again
tr709: tr709:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:416 // line internal/scanner/scanner.rl:416
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
@ -18087,7 +18087,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:416 // line internal/scanner/scanner.rl:416
lex.act = 155 lex.act = 155
goto st485 goto st485
@ -18158,7 +18158,7 @@ func (lex *Lexer) Lex() *Token {
goto tr708 goto tr708
tr711: tr711:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st486 goto st486
st486: st486:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -18420,7 +18420,7 @@ func (lex *Lexer) Lex() *Token {
goto _again goto _again
tr726: tr726:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:428 // line internal/scanner/scanner.rl:428
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
@ -18555,7 +18555,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:428 // line internal/scanner/scanner.rl:428
lex.act = 160 lex.act = 160
goto st491 goto st491
@ -18626,7 +18626,7 @@ func (lex *Lexer) Lex() *Token {
goto tr725 goto tr725
tr728: tr728:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st492 goto st492
st492: st492:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -19147,7 +19147,7 @@ func (lex *Lexer) Lex() *Token {
goto st502 goto st502
tr753: tr753:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:462 // line internal/scanner/scanner.rl:462
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
@ -19652,7 +19652,7 @@ func (lex *Lexer) Lex() *Token {
// line internal/scanner/scanner.rl:477 // line internal/scanner/scanner.rl:477
(lex.p) = (lex.te) - 1 (lex.p) = (lex.te) - 1
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st514 goto st514
tr766: tr766:
@ -19683,17 +19683,17 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st514 goto st514
tr772: tr772:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:477 // line internal/scanner/scanner.rl:477
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st514 goto st514
tr776: tr776:
@ -19742,7 +19742,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st515 goto st515
st515: st515:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -19772,7 +19772,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st516 goto st516
st516: st516:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -19794,7 +19794,7 @@ func (lex *Lexer) Lex() *Token {
goto tr772 goto tr772
tr775: tr775:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st108 goto st108
st108: st108:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -19819,7 +19819,7 @@ func (lex *Lexer) Lex() *Token {
// line internal/scanner/scanner.rl:483 // line internal/scanner/scanner.rl:483
(lex.p) = (lex.te) - 1 (lex.p) = (lex.te) - 1
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st518 goto st518
tr777: tr777:
@ -19850,17 +19850,17 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st518 goto st518
tr783: tr783:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:483 // line internal/scanner/scanner.rl:483
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st518 goto st518
tr787: tr787:
@ -19909,7 +19909,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st519 goto st519
st519: st519:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -19939,7 +19939,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st520 goto st520
st520: st520:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -19961,7 +19961,7 @@ func (lex *Lexer) Lex() *Token {
goto tr783 goto tr783
tr786: tr786:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st109 goto st109
st109: st109:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -19986,7 +19986,7 @@ func (lex *Lexer) Lex() *Token {
// line internal/scanner/scanner.rl:489 // line internal/scanner/scanner.rl:489
(lex.p) = (lex.te) - 1 (lex.p) = (lex.te) - 1
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st522 goto st522
tr788: tr788:
@ -20017,17 +20017,17 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st522 goto st522
tr794: tr794:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:489 // line internal/scanner/scanner.rl:489
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_WHITESPACE, lex.ts, lex.te) lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)
} }
goto st522 goto st522
tr798: tr798:
@ -20076,7 +20076,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st523 goto st523
st523: st523:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -20106,7 +20106,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st524 goto st524
st524: st524:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -20128,7 +20128,7 @@ func (lex *Lexer) Lex() *Token {
goto tr794 goto tr794
tr797: tr797:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st110 goto st110
st110: st110:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -20161,19 +20161,19 @@ func (lex *Lexer) Lex() *Token {
case 186: case 186:
{ {
(lex.p) = (lex.te) - 1 (lex.p) = (lex.te) - 1
lex.addToken(T_HALT_COMPILER, lex.ts, lex.te) lex.addHiddenToken(T_HALT_COMPILER, lex.ts, lex.te)
} }
} }
goto st526 goto st526
tr802: tr802:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:495 // line internal/scanner/scanner.rl:495
lex.te = (lex.p) lex.te = (lex.p)
(lex.p)-- (lex.p)--
{ {
lex.addToken(T_HALT_COMPILER, lex.ts, lex.te) lex.addHiddenToken(T_HALT_COMPILER, lex.ts, lex.te)
} }
goto st526 goto st526
st526: st526:
@ -20207,7 +20207,7 @@ func (lex *Lexer) Lex() *Token {
lex.te = (lex.p) + 1 lex.te = (lex.p) + 1
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
// line internal/scanner/scanner.rl:495 // line internal/scanner/scanner.rl:495
lex.act = 186 lex.act = 186
goto st527 goto st527
@ -20223,7 +20223,7 @@ func (lex *Lexer) Lex() *Token {
goto tr799 goto tr799
tr804: tr804:
// line internal/scanner/scanner.rl:55 // line internal/scanner/scanner.rl:55
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
goto st528 goto st528
st528: st528:
if (lex.p)++; (lex.p) == (lex.pe) { if (lex.p)++; (lex.p) == (lex.pe) {
@ -22854,7 +22854,7 @@ func (lex *Lexer) Lex() *Token {
// line internal/scanner/scanner.rl:499 // line internal/scanner/scanner.rl:499
token.Hidden = lex.HiddenTokens token.Hidden = lex.hiddenTokens
token.Value = lex.data[lex.ts:lex.te] token.Value = lex.data[lex.ts:lex.te]
token.ID = tok token.ID = tok

36
internal/scanner/scanner.rl
View File

@ -19,11 +19,11 @@ func initLexer(lex *Lexer) {
} }
func (lex *Lexer) Lex() *Token { func (lex *Lexer) Lex() *Token {
lex.HiddenTokens = nil lex.hiddenTokens = nil
eof := lex.pe eof := lex.pe
var tok TokenID var tok TokenID
token := lex.TokenPool.Get() token := lex.tokenPool.Get()
token.Hidden = nil token.Hidden = nil
token.Value = lex.data[0:0] token.Value = lex.data[0:0]
@ -38,11 +38,11 @@ func (lex *Lexer) Lex() *Token {
action constant_string_new_line { action constant_string_new_line {
if lex.data[lex.p] == '\n' { if lex.data[lex.p] == '\n' {
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
} }
if lex.data[lex.p] == '\r' && lex.data[lex.p+1] != '\n' { if lex.data[lex.p] == '\r' && lex.data[lex.p+1] != '\n' {
lex.NewLines.Append(lex.p) lex.newLines.Append(lex.p)
} }
} }
@ -52,7 +52,7 @@ func (lex *Lexer) Lex() *Token {
action is_not_string_end_or_var { lex.isNotStringEnd('"') && lex.isNotStringVar() } action is_not_string_end_or_var { lex.isNotStringEnd('"') && lex.isNotStringVar() }
action is_not_backqoute_end_or_var { lex.isNotStringEnd('`') && lex.isNotStringVar() } action is_not_backqoute_end_or_var { lex.isNotStringEnd('`') && lex.isNotStringVar() }
newline = ('\r\n' >(nl, 1) | '\r' >(nl, 0) | '\n' >(nl, 0)) %{lex.NewLines.Append(lex.p);}; newline = ('\r\n' >(nl, 1) | '\r' >(nl, 0) | '\n' >(nl, 0)) %{lex.newLines.Append(lex.p);};
any_line = any | newline; any_line = any | newline;
whitespace = [\t\v\f ]; whitespace = [\t\v\f ];
whitespace_line = [\t\v\f ] | newline; whitespace_line = [\t\v\f ] | newline;
@ -125,7 +125,7 @@ func (lex *Lexer) Lex() *Token {
main := |* main := |*
"#!" any* :>> newline => { "#!" any* :>> newline => {
lex.addToken(T_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
}; };
any => { any => {
fnext html; fnext html;
@ -141,12 +141,12 @@ func (lex *Lexer) Lex() *Token {
fbreak; fbreak;
}; };
'<?' => { '<?' => {
lex.addToken(T_OPEN_TAG, lex.ts, lex.te) lex.addHiddenToken(T_OPEN_TAG, lex.ts, lex.te)
fnext php; fnext php;
}; };
'<?php'i ( [ \t] | newline ) => { '<?php'i ( [ \t] | newline ) => {
lex.ungetCnt(lex.te - lex.ts - 5) lex.ungetCnt(lex.te - lex.ts - 5)
lex.addToken(T_OPEN_TAG, lex.ts, lex.ts+5) lex.addHiddenToken(T_OPEN_TAG, lex.ts, lex.ts+5)
fnext php; fnext php;
}; };
'<?='i => { '<?='i => {
@ -158,7 +158,7 @@ func (lex *Lexer) Lex() *Token {
*|; *|;
php := |* php := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
'?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext html; fbreak;}; '?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext html; fbreak;};
';' whitespace_line* '?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext html; fbreak;}; ';' whitespace_line* '?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext html; fbreak;};
@ -318,7 +318,7 @@ func (lex *Lexer) Lex() *Token {
('#' | '//') any_line* when is_not_comment_end => { ('#' | '//') any_line* when is_not_comment_end => {
lex.ungetStr("?>") lex.ungetStr("?>")
lex.addToken(T_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
}; };
'/*' any_line* :>> '*/' { '/*' any_line* :>> '*/' {
isDocComment := false; isDocComment := false;
@ -327,9 +327,9 @@ func (lex *Lexer) Lex() *Token {
} }
if isDocComment { if isDocComment {
lex.addToken(T_DOC_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_DOC_COMMENT, lex.ts, lex.te)
} else { } else {
lex.addToken(T_COMMENT, lex.ts, lex.te) lex.addHiddenToken(T_COMMENT, lex.ts, lex.te)
} }
}; };
@ -378,7 +378,7 @@ func (lex *Lexer) Lex() *Token {
*|; *|;
property := |* property := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
"->" => {lex.setTokenPosition(token); tok = T_OBJECT_OPERATOR; fbreak;}; "->" => {lex.setTokenPosition(token); tok = T_OBJECT_OPERATOR; fbreak;};
varname => {lex.setTokenPosition(token); tok = T_STRING; fnext php; fbreak;}; varname => {lex.setTokenPosition(token); tok = T_STRING; fnext php; fbreak;};
any => {lex.ungetCnt(1); fgoto php;}; any => {lex.ungetCnt(1); fgoto php;};
@ -474,31 +474,31 @@ func (lex *Lexer) Lex() *Token {
*|; *|;
halt_compiller_open_parenthesis := |* halt_compiller_open_parenthesis := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
"(" => {lex.setTokenPosition(token); tok = TokenID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;}; "(" => {lex.setTokenPosition(token); tok = TokenID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_close_parenthesis := |* halt_compiller_close_parenthesis := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
")" => {lex.setTokenPosition(token); tok = TokenID(int(')')); fnext halt_compiller_close_semicolon; fbreak;}; ")" => {lex.setTokenPosition(token); tok = TokenID(int(')')); fnext halt_compiller_close_semicolon; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_close_semicolon := |* halt_compiller_close_semicolon := |*
whitespace_line* => {lex.addToken(T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addHiddenToken(T_WHITESPACE, lex.ts, lex.te)};
";" => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext halt_compiller_end; fbreak;}; ";" => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext halt_compiller_end; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_end := |* halt_compiller_end := |*
any_line* => { lex.addToken(T_HALT_COMPILER, lex.ts, lex.te); }; any_line* => { lex.addHiddenToken(T_HALT_COMPILER, lex.ts, lex.te); };
*|; *|;
write exec; write exec;
}%% }%%
token.Hidden = lex.HiddenTokens token.Hidden = lex.hiddenTokens
token.Value = lex.data[lex.ts:lex.te] token.Value = lex.data[lex.ts:lex.te]
token.ID = tok token.ID = tok

122
internal/scanner/scanner_test.go
View File

@ -353,8 +353,8 @@ func TestTokens(t *testing.T) {
T_UNSET_CAST.String(), T_UNSET_CAST.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -381,8 +381,8 @@ func TestShebang(t *testing.T) {
"\n", "\n",
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
tkn := lexer.Lex() tkn := lexer.Lex()
@ -401,8 +401,8 @@ func TestShebangHtml(t *testing.T) {
0.1 0.1
` `
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
assert.Equal(t, tkn.ID, T_INLINE_HTML) assert.Equal(t, tkn.ID, T_INLINE_HTML)
@ -451,8 +451,8 @@ func TestNumberTokens(t *testing.T) {
T_DNUMBER.String(), T_DNUMBER.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -508,8 +508,8 @@ func TestConstantStrings(t *testing.T) {
T_CONSTANT_ENCAPSED_STRING.String(), T_CONSTANT_ENCAPSED_STRING.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -555,7 +555,7 @@ func TestSingleQuoteStringTokens(t *testing.T) {
T_CONSTANT_ENCAPSED_STRING.String(), T_CONSTANT_ENCAPSED_STRING.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
actual := []string{} actual := []string{}
for { for {
@ -642,8 +642,8 @@ func TestTeplateStringTokens(t *testing.T) {
TokenID(int('"')).String(), TokenID(int('"')).String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -727,8 +727,8 @@ func TestBackquoteStringTokens(t *testing.T) {
TokenID(int('`')).String(), TokenID(int('`')).String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -821,8 +821,8 @@ CAT;
TokenID(int(';')).String(), TokenID(int(';')).String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -894,8 +894,8 @@ CAT
T_END_HEREDOC.String(), T_END_HEREDOC.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -933,8 +933,8 @@ CAT;
TokenID(int(';')).String(), TokenID(int(';')).String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -964,8 +964,8 @@ func TestHereDocTokens73(t *testing.T) {
T_VARIABLE.String(), T_VARIABLE.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -994,9 +994,9 @@ CAT;`
TokenID(int(';')).String(), TokenID(int(';')).String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.PHPVersion = "7.2" lexer.phpVersion = "7.2"
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -1027,8 +1027,8 @@ func TestInlineHtmlNopTokens(t *testing.T) {
T_INLINE_HTML.String(), T_INLINE_HTML.String(),
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
actual := []string{} actual := []string{}
for { for {
@ -1062,7 +1062,7 @@ func TestStringTokensAfterVariable(t *testing.T) {
"\"", "\"",
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
actual := []string{} actual := []string{}
actualTokens := []string{} actualTokens := []string{}
@ -1095,7 +1095,7 @@ func TestSlashAfterVariable(t *testing.T) {
"3", "3",
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
actual := []string{} actual := []string{}
actualTokens := []string{} actualTokens := []string{}
@ -1132,12 +1132,12 @@ func TestCommentEnd(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
lexer.Lex() lexer.Lex()
actual := lexer.HiddenTokens actual := lexer.hiddenTokens
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
} }
@ -1161,8 +1161,8 @@ func TestCommentNewLine(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1190,8 +1190,8 @@ func TestCommentNewLine1(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1219,8 +1219,8 @@ func TestCommentNewLine2(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1249,8 +1249,8 @@ func TestCommentWithPhpEndTag(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1279,8 +1279,8 @@ func TestInlineComment(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1309,12 +1309,12 @@ func TestInlineComment2(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
lexer.Lex() lexer.Lex()
actual := lexer.HiddenTokens actual := lexer.hiddenTokens
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
} }
@ -1343,12 +1343,12 @@ func TestEmptyInlineComment(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
lexer.Lex() lexer.Lex()
actual := lexer.HiddenTokens actual := lexer.hiddenTokens
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
} }
@ -1373,8 +1373,8 @@ func TestEmptyInlineComment2(t *testing.T) {
}, },
} }
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1387,8 +1387,8 @@ func TestMethodCallTokens(t *testing.T) {
src := `<?php src := `<?php
$a -> bar ( '' ) ;` $a -> bar ( '' ) ;`
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
expected := []token.Token{ expected := []token.Token{
{ {
@ -1469,8 +1469,8 @@ func TestYieldFromTokens(t *testing.T) {
src := `<?php src := `<?php
yield from $a` yield from $a`
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
lexer.WithHiddenTokens = true lexer.withHiddenTokens = true
expected := []token.Token{ expected := []token.Token{
{ {
@ -1500,7 +1500,7 @@ func TestYieldFromTokens(t *testing.T) {
func TestVarNameByteChars(t *testing.T) { func TestVarNameByteChars(t *testing.T) {
src := "<?php $\x80 $\xff" src := "<?php $\x80 $\xff"
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
tkn := lexer.Lex() tkn := lexer.Lex()
assert.Equal(t, "$\x80", string(tkn.Value)) assert.Equal(t, "$\x80", string(tkn.Value))
@ -1512,7 +1512,7 @@ func TestVarNameByteChars(t *testing.T) {
func TestStringVarNameByteChars(t *testing.T) { func TestStringVarNameByteChars(t *testing.T) {
src := "<?php \"$\x80 $\xff\"" src := "<?php \"$\x80 $\xff\""
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
tkn := lexer.Lex() tkn := lexer.Lex()
assert.Equal(t, "\"", string(tkn.Value)) assert.Equal(t, "\"", string(tkn.Value))
@ -1534,11 +1534,13 @@ func TestIgnoreControllCharacters(t *testing.T) {
src := "<?php \004 echo $b;" src := "<?php \004 echo $b;"
var actualErr *errors.Error var actualErr *errors.Error
errHandler := func(e *errors.Error) { config := Config{
ErrHandlerFunc: func(e *errors.Error) {
actualErr = e actualErr = e
},
} }
lexer := NewLexer([]byte(src), errHandler) lexer := NewLexer([]byte(src), "7.4", config)
expected := "echo" expected := "echo"
tkn := lexer.Lex() tkn := lexer.Lex()
@ -1560,7 +1562,7 @@ func TestIgnoreControllCharacters(t *testing.T) {
func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) { func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) {
src := "<?php \"$a[test\004]\";" src := "<?php \"$a[test\004]\";"
lexer := NewLexer([]byte(src), nil) lexer := NewLexer([]byte(src), "7.4", Config{})
expected := "\"" expected := "\""
tkn := lexer.Lex() tkn := lexer.Lex()

7
pkg/parser/parser.go
View File

@ -13,10 +13,9 @@ type Parser interface {
Parse() int Parse() int
GetRootNode() ast.Vertex GetRootNode() ast.Vertex
GetErrors() []*errors.Error GetErrors() []*errors.Error
WithTokens()
} }
func NewParser(src []byte, v string) (Parser, error) { func NewParser(src []byte, v string, withTokens bool) (Parser, error) {
var parser Parser var parser Parser
r, err := version.Compare(v, "7.0") r, err := version.Compare(v, "7.0")
@ -25,9 +24,9 @@ func NewParser(src []byte, v string) (Parser, error) {
} }
if r == -1 { if r == -1 {
parser = php5.NewParser(src, v) parser = php5.NewParser(src, v, withTokens)
} else { } else {
parser = php7.NewParser(src, v) parser = php7.NewParser(src, v, withTokens)
} }
return parser, nil return parser, nil

3
pkg/printer/printer_parsed_php5_test.go
View File

@ -10,8 +10,7 @@ import (
) )
func parsePhp5(src string) ast.Vertex { func parsePhp5(src string) ast.Vertex {
php5parser := php5.NewParser([]byte(src), "5.6") php5parser := php5.NewParser([]byte(src), "5.6", true)
php5parser.WithTokens()
php5parser.Parse() php5parser.Parse()
return php5parser.GetRootNode() return php5parser.GetRootNode()

6
pkg/printer/printer_parsed_php7_test.go
View File

@ -27,8 +27,7 @@ abstract class Bar extends Baz
// parse // parse
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", true)
php7parser.WithTokens()
php7parser.Parse() php7parser.Parse()
rootNode := php7parser.GetRootNode() rootNode := php7parser.GetRootNode()
@ -59,8 +58,7 @@ abstract class Bar extends Baz
} }
func parse(src string) ast.Vertex { func parse(src string) ast.Vertex {
php7parser := php7.NewParser([]byte(src), "7.4") php7parser := php7.NewParser([]byte(src), "7.4", true)
php7parser.WithTokens()
php7parser.Parse() php7parser.Parse()
return php7parser.GetRootNode() return php7parser.GetRootNode()