maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[#82] Lexer: handle PHP 7.3 heredoc

Browse Source
This commit is contained in:
z7zmey 2019-12-26 15:41:06 +02:00
parent 777873afae
commit 6afa2a089b
4 changed files with 707 additions and 572 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
scanner/lexer.go
View File

@ -41,6 +41,7 @@ type Lexer struct {
lastToken *Token
Errors []*errors.Error
NewLines NewLines
PHPVersion string
}
func (l *Lexer) ReturnTokenToPool(t *Token) {
@ -132,6 +133,18 @@ func (lex *Lexer) isNotStringEnd(s byte) bool {
}
func (lex *Lexer) isHeredocEnd(p int) bool {
if lex.PHPVersion == "" {
return lex.isHeredocEndSince73(p)
}
if comparePHPVersion(lex.PHPVersion, "7.3") == -1 {
return lex.isHeredocEndBefore73(p)
}
return lex.isHeredocEndSince73(p)
}
func (lex *Lexer) isHeredocEndBefore73(p int) bool {
if lex.data[p-1] != '\r' && lex.data[p-1] != '\n' {
return false
}
@ -152,6 +165,37 @@ func (lex *Lexer) isHeredocEnd(p int) bool {
return bytes.Equal(lex.heredocLabel, lex.data[p:p+l])
}
func (lex *Lexer) isHeredocEndSince73(p int) bool {
if lex.data[p-1] != '\r' && lex.data[p-1] != '\n' {
return false
}
for lex.data[p] == ' ' || lex.data[p] == '\t' {
p++
}
l := len(lex.heredocLabel)
if len(lex.data) < p+l {
return false
}
if len(lex.data) > p+l && isValidVarName(lex.data[p+l]) {
return false
}
a := string(lex.heredocLabel)
b := string(lex.data[p : p+l])
_, _ = a, b
if bytes.Equal(lex.heredocLabel, lex.data[p:p+l]) {
lex.p = p
return true
}
return false
}
func (lex *Lexer) isNotHeredocEnd(p int) bool {
return !lex.isHeredocEnd(p)
}
@ -221,5 +265,32 @@ func (lex *Lexer) Error(msg string) {
}
func isValidVarNameStart(r byte) bool {
return r >= 'A' && r <= 'Z' || r == '_' || r >= 'a' && r <= 'z' || r >= '\u007f' && r <= 'ÿ'
return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || r == '_' || (r >= 0x80 && r <= 0xff)
}
func isValidVarName(r byte) bool {
return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || (r >= 0x80 && r <= 0xff)
}
func comparePHPVersion(a string, b string) int {
first := strings.Split(a, ".")
second := strings.Split(b, ".")
if first[0] < second[0] {
return -1
}
if first[0] > second[0] {
return 1
}
if first[1] < second[1] {
return -1
}
if first[1] > second[1] {
return 1
}
return 0
}

6
scanner/scanner.go
View File

@ -17157,7 +17157,7 @@ func (lex *Lexer) Lex(lval Lval) int {
lex.setTokenPosition(token)
tok = T_ENCAPSED_AND_WHITESPACE
if lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
if len(lex.data) > lex.p+1 && lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
lex.cs = 486
}
{
@ -17179,7 +17179,7 @@ func (lex *Lexer) Lex(lval Lval) int {
lex.setTokenPosition(token)
tok = T_ENCAPSED_AND_WHITESPACE
if lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
if len(lex.data) > lex.p+1 && lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
lex.cs = 486
}
{
@ -17197,7 +17197,7 @@ func (lex *Lexer) Lex(lval Lval) int {
lex.setTokenPosition(token)
tok = T_ENCAPSED_AND_WHITESPACE
if lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
if len(lex.data) > lex.p+1 && lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
lex.cs = 486
}
{

2
scanner/scanner.rl
View File

@ -394,7 +394,7 @@ func (lex *Lexer) Lex(lval Lval) int {
lex.setTokenPosition(token);
tok = T_ENCAPSED_AND_WHITESPACE;
if lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
if len(lex.data) > lex.p+1 && lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
fnext heredoc_end;
}
fbreak;

64
scanner/scanner_test.go
View File

@ -892,6 +892,70 @@ CAT;
assert.DeepEqual(t, expected, actual)
}
func TestHereDocTokens73(t *testing.T) {
src := `<?php
<<<"CAT"
text
CAT, $b`
expected := []string{
T_START_HEREDOC.String(),
T_ENCAPSED_AND_WHITESPACE.String(),
T_END_HEREDOC.String(),
TokenID(int(',')).String(),
T_VARIABLE.String(),
}
lexer := NewLexer([]byte(src))
lexer.WithFreeFloating = true
lv := &lval{}
actual := []string{}
for {
token := lexer.Lex(lv)
if token == 0 {
break
}
actual = append(actual, TokenID(token).String())
}
assert.DeepEqual(t, expected, actual)
}
func TestHereDocTokensBefore73(t *testing.T) {
src := `<?php
<<<"CAT"
CAT
CAT;`
expected := []string{
T_START_HEREDOC.String(),
T_ENCAPSED_AND_WHITESPACE.String(),
T_END_HEREDOC.String(),
TokenID(int(';')).String(),
}
lexer := NewLexer([]byte(src))
lexer.PHPVersion = "7.2"
lexer.WithFreeFloating = true
lv := &lval{}
actual := []string{}
for {
token := lexer.Lex(lv)
if token == 0 {
break
}
actual = append(actual, TokenID(token).String())
}
assert.DeepEqual(t, expected, actual)
}
func TestInlineHtmlNopTokens(t *testing.T) {
src := `<?php
$a; ?> test <?php