scanner: fix heredoc

This commit is contained in:
z7zmey 2018-04-06 00:24:00 +03:00
parent 9ea1f05f90
commit 9a799fc3be
3 changed files with 125 additions and 31 deletions

View File

@ -8720,57 +8720,65 @@ yyrule152: // .|[ \t\n\r]
searchLabel := []byte{} searchLabel := []byte{}
tb := []lex.Char{} tb := []lex.Char{}
HEREDOCFOR:
for { for {
if c == -1 { if c == -1 {
break break
} }
nls := 0
switch c { switch c {
case '\n':
fallthrough
case '\r': case '\r':
nls = 1
c := l.Next()
if c != '\n' {
nls = 0
l.ungetChars(0)
}
fallthrough
case '\n':
if bytes.Equal(append(l.charsToBytes(heredocLabel), ';'), searchLabel) { if bytes.Equal(append(l.charsToBytes(heredocLabel), ';'), searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(heredocLabel) + 1) tb = l.ungetChars(len(heredocLabel) + 1 + nls)
break HEREDOCFOR lval.Token(l.newToken(tb))
return T_ENCAPSED_AND_WHITESPACE
} }
if bytes.Equal(l.charsToBytes(heredocLabel), searchLabel) { if bytes.Equal(l.charsToBytes(heredocLabel), searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(heredocLabel)) tb = l.ungetChars(len(heredocLabel) + nls)
break HEREDOCFOR lval.Token(l.newToken(tb))
return T_ENCAPSED_AND_WHITESPACE
} }
searchLabel = []byte{} searchLabel = []byte{}
case '$': case '$':
c = l.Next() c = l.Next()
if rune(c) == '{' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' || c >= '\u007f' && c <= 'ÿ' { if rune(c) == '{' || isValidFirstVarNameRune(rune(c)) {
tb = l.ungetChars(1) tb = l.ungetChars(1)
break HEREDOCFOR lval.Token(l.newToken(tb))
return T_ENCAPSED_AND_WHITESPACE
} }
l.ungetChars(0) l.ungetChars(0)
searchLabel = []byte{}
case '{': case '{':
c = l.Next() c = l.Next()
if rune(c) == '$' { if rune(c) == '$' {
tb = l.ungetChars(1) tb = l.ungetChars(1)
break HEREDOCFOR lval.Token(l.newToken(tb))
return T_ENCAPSED_AND_WHITESPACE
} }
l.ungetChars(0) l.ungetChars(0)
searchLabel = []byte{}
case '\\': case '\\':
c = l.Next() c = l.Next()
searchLabel = []byte{} if c == '\n' || c == '\r' {
l.ungetChars(0)
}
default: default:
searchLabel = append(searchLabel, byte(rune(c))) searchLabel = append(searchLabel, byte(rune(c)))
} }
c = l.Next() c = l.Next()
} }
goto yystate0
lval.Token(l.newToken(tb))
return T_ENCAPSED_AND_WHITESPACE
} }
yyrule153: // \${VAR_NAME} yyrule153: // \${VAR_NAME}
{ {

View File

@ -578,49 +578,65 @@ NEW_LINE (\r|\n|\r\n)
searchLabel := []byte{} searchLabel := []byte{}
tb := []lex.Char{} tb := []lex.Char{}
HEREDOCFOR:for { for {
if c == -1 { if c == -1 {
break; break;
} }
nls := 0
switch c { switch c {
case '\n': fallthrough
case '\r': case '\r':
nls = 1
c := l.Next()
if c != '\n' {
nls = 0
l.ungetChars(0)
}
fallthrough
case '\n':
if bytes.Equal(append(l.charsToBytes(heredocLabel), ';'), searchLabel) { if bytes.Equal(append(l.charsToBytes(heredocLabel), ';'), searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(heredocLabel)+1) tb = l.ungetChars(len(heredocLabel)+1+nls)
break HEREDOCFOR; lval.Token(l.newToken(tb));
return T_ENCAPSED_AND_WHITESPACE
} }
if bytes.Equal(l.charsToBytes(heredocLabel), searchLabel) { if bytes.Equal(l.charsToBytes(heredocLabel), searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(heredocLabel)) tb = l.ungetChars(len(heredocLabel)+nls)
break HEREDOCFOR; lval.Token(l.newToken(tb));
return T_ENCAPSED_AND_WHITESPACE
} }
searchLabel = []byte{} searchLabel = []byte{}
case '$': case '$':
c = l.Next(); c = l.Next();
if rune(c) == '{' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' || c >= '\u007f' && c <= 'ÿ' { if rune(c) == '{' || isValidFirstVarNameRune(rune(c)) {
tb = l.ungetChars(1) tb = l.ungetChars(1)
break HEREDOCFOR; lval.Token(l.newToken(tb));
return T_ENCAPSED_AND_WHITESPACE
} }
l.ungetChars(0) l.ungetChars(0)
searchLabel = []byte{}
case '{': case '{':
c = l.Next(); c = l.Next();
if rune(c) == '$' { if rune(c) == '$' {
tb = l.ungetChars(1) tb = l.ungetChars(1)
break HEREDOCFOR; lval.Token(l.newToken(tb));
return T_ENCAPSED_AND_WHITESPACE
} }
l.ungetChars(0) l.ungetChars(0)
searchLabel = []byte{}
case '\\': case '\\':
c = l.Next(); c = l.Next();
searchLabel = []byte{} if c == '\n' || c == '\r' {
l.ungetChars(0)
}
default: default:
searchLabel = append(searchLabel, byte(rune(c))) searchLabel = append(searchLabel, byte(rune(c)))
@ -628,9 +644,6 @@ NEW_LINE (\r|\n|\r\n)
c = l.Next() c = l.Next()
} }
lval.Token(l.newToken(tb));
return T_ENCAPSED_AND_WHITESPACE
<STRING_VAR>\${VAR_NAME} lval.Token(l.newToken(l.Token())); return T_VARIABLE <STRING_VAR>\${VAR_NAME} lval.Token(l.newToken(l.Token())); return T_VARIABLE
<STRING_VAR>->{VAR_NAME} lval.Token(l.newToken(l.ungetChars(len(l.Token())-2))); return T_OBJECT_OPERATOR <STRING_VAR>->{VAR_NAME} lval.Token(l.newToken(l.ungetChars(len(l.Token())-2))); return T_OBJECT_OPERATOR

View File

@ -693,6 +693,79 @@ CAT;
assertEqual(t, expected, actual) assertEqual(t, expected, actual)
} }
func TestHereDocTokens2(t *testing.T) {
src := `<?php
<<<CAT
$foo/
CAT;
<<<CAT
$foo/100
CAT;
<<<CAT
$/$foo
CAT;
<<<CAT
$0$foo
CAT;
<<<CAT
$foo$bar\
CAT
`
expected := []int{
scanner.T_START_HEREDOC,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC,
scanner.T_VARIABLE,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lv := &lval{}
actual := []int{}
for {
token := lexer.Lex(lv)
if token < 0 {
break
}
actual = append(actual, token)
}
assertEqual(t, expected, actual)
}
func TestInlineHtmlNopTokens(t *testing.T) { func TestInlineHtmlNopTokens(t *testing.T) {
src := `<?php src := `<?php
$a; ?> test <?php $a; ?> test <?php