maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into dev

Browse Source
This commit is contained in:
z7zmey 2018-07-24 22:24:32 +03:00
commit 0138749c6d
10 changed files with 15348 additions and 15226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1024
php5/php5.go
View File

File diff suppressed because it is too large Load Diff

2
php5/php5.y
View File

@ -377,6 +377,8 @@ top_statement:
addMeta($$, $4.Meta, meta.SemiColonToken) addMeta($$, $4.Meta, meta.SemiColonToken)
yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL) yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
yylex.(*Parser).Begin(scanner.HALT_COMPILER)
} }
| T_NAMESPACE namespace_name ';' | T_NAMESPACE namespace_name ';'
{ {

13404
php5/php5_test.go
View File

File diff suppressed because it is too large Load Diff

800
php7/php7.go
View File

File diff suppressed because it is too large Load Diff

2
php7/php7.y
View File

@ -476,6 +476,8 @@ top_statement:
addMeta($$, $4.Meta, meta.SemiColonToken) addMeta($$, $4.Meta, meta.SemiColonToken)
yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL) yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
yylex.(*Parser).Begin(scanner.HALT_COMPILER)
} }
| T_NAMESPACE namespace_name ';' | T_NAMESPACE namespace_name ';'
{ {

10371
php7/php7_test.go
View File

File diff suppressed because it is too large Load Diff

2
scanner/lexer.go
View File

@ -108,7 +108,7 @@ func (l *Lexer) popState() {
l.StateStack = l.StateStack[:len-1] l.StateStack = l.StateStack[:len-1]
} }
func (l *Lexer) begin(state int) { func (l *Lexer) Begin(state int) {
len := len(l.StateStack) len := len(l.StateStack)
l.StateStack = l.StateStack[:len-1] l.StateStack = l.StateStack[:len-1]
l.StateStack = append(l.StateStack, state) l.StateStack = append(l.StateStack, state)

5941
scanner/scanner.go
View File

File diff suppressed because it is too large Load Diff

50
scanner/scanner.l
View File

@ -24,6 +24,7 @@ const (
NOWDOC NOWDOC
HEREDOC HEREDOC
BACKQUOTE BACKQUOTE
HALT_COMPILER
) )
func isValidFirstVarNameRune(r rune) bool { func isValidFirstVarNameRune(r rune) bool {
@ -36,7 +37,7 @@ func (l *Lexer) Lex(lval Lval) int {
%} %}
%s PHP STRING STRING_VAR STRING_VAR_INDEX STRING_VAR_NAME PROPERTY HEREDOC_END NOWDOC HEREDOC BACKQUOTE %s PHP STRING STRING_VAR STRING_VAR_INDEX STRING_VAR_NAME PROPERTY HEREDOC_END NOWDOC HEREDOC BACKQUOTE HALT_COMPILER
%yyb last == '\n' || last = '\0' %yyb last == '\n' || last = '\0'
%yyt l.getCurrentState() %yyt l.getCurrentState()
@ -84,13 +85,13 @@ NEW_LINE (\r|\n|\r\n)
lval.Token(l.createToken(tb)) lval.Token(l.createToken(tb))
return int(T_INLINE_HTML) return int(T_INLINE_HTML)
<INITIAL>\<\?php([ \t]|{NEW_LINE}) l.begin(PHP);l.ungetChars(len(l.Token())-5) <INITIAL>\<\?php([ \t]|{NEW_LINE}) l.Begin(PHP);l.ungetChars(len(l.Token())-5)
<INITIAL>\<\? l.begin(PHP); <INITIAL>\<\? l.Begin(PHP);
<INITIAL>\<\?= l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_ECHO); <INITIAL>\<\?= l.Begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_ECHO);
<PHP>[ \t\n\r]+ l.addWhiteSpace(l.Token()) <PHP>[ \t\n\r]+ l.addWhiteSpace(l.Token())
<PHP>[;][ \t\n\r]*\?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';'); <PHP>[;][ \t\n\r]*\?\>{NEW_LINE}? l.Begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
<PHP>\?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';'); <PHP>\?\>{NEW_LINE}? l.Begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
<PHP>{DNUM}|{EXPONENT_DNUM} lval.Token(l.createToken(l.Token())); return int(T_DNUMBER) <PHP>{DNUM}|{EXPONENT_DNUM} lval.Token(l.createToken(l.Token())); return int(T_DNUMBER)
<PHP>{BNUM} <PHP>{BNUM}
@ -287,6 +288,8 @@ NEW_LINE (\r|\n|\r\n)
l.addComments(tb) l.addComments(tb)
<PHP>[/][*][*][/]
l.addComments(l.Token())
<PHP>([/][*])|([/][*][*]) <PHP>([/][*])|([/][*][*])
tb := l.Token() tb := l.Token()
is_doc_comment := false is_doc_comment := false
@ -295,6 +298,7 @@ NEW_LINE (\r|\n|\r\n)
l.PhpDocComment = "" l.PhpDocComment = ""
} }
c = l.Next()
for { for {
if c == -1 { if c == -1 {
break; // TODO: Unterminated comment starting line %d break; // TODO: Unterminated comment starting line %d
@ -322,16 +326,16 @@ NEW_LINE (\r|\n|\r\n)
<PHP>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_VARIABLE) <PHP>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_VARIABLE)
<PHP>{VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_STRING) <PHP>{VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_STRING)
<PHP>-> l.begin(PROPERTY);lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR); <PHP>-> l.Begin(PROPERTY);lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR);
<PROPERTY>[ \t\n\r]+ l.addWhiteSpace(l.Token()) <PROPERTY>[ \t\n\r]+ l.addWhiteSpace(l.Token())
<PROPERTY>-> lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR); <PROPERTY>-> lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR);
<PROPERTY>{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_STRING); <PROPERTY>{VAR_NAME} l.Begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_STRING);
<PROPERTY>. l.ungetChars(1);l.begin(PHP) <PROPERTY>. l.ungetChars(1);l.Begin(PHP)
<PHP>[\']([^\\\']*([\\].)*)*[\'] lval.Token(l.createToken(l.Token())); return int(T_CONSTANT_ENCAPSED_STRING); <PHP>[\']([^\\\']*(\\(.|\n))*)*[\'] lval.Token(l.createToken(l.Token())); return int(T_CONSTANT_ENCAPSED_STRING);
<PHP>` l.begin(BACKQUOTE); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) <PHP>` l.Begin(BACKQUOTE); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<BACKQUOTE>` l.begin(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) <BACKQUOTE>` l.Begin(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<PHP>[b]?\<\<\<[ \t]*({VAR_NAME}|([']{VAR_NAME}['])|(["]{VAR_NAME}["])){NEW_LINE} <PHP>[b]?\<\<\<[ \t]*({VAR_NAME}|([']{VAR_NAME}['])|(["]{VAR_NAME}["])){NEW_LINE}
tb := l.Token() tb := l.Token()
@ -362,13 +366,13 @@ NEW_LINE (\r|\n|\r\n)
case '\'' : case '\'' :
lblFirst++ lblFirst++
lblLast-- lblLast--
l.begin(NOWDOC) l.Begin(NOWDOC)
case '"' : case '"' :
lblFirst++ lblFirst++
lblLast-- lblLast--
l.begin(HEREDOC) l.Begin(HEREDOC)
default: default:
l.begin(HEREDOC) l.Begin(HEREDOC)
} }
l.heredocLabel = l.tokenString(tb[lblFirst:lblLast+1]) l.heredocLabel = l.tokenString(tb[lblFirst:lblLast+1])
@ -387,7 +391,7 @@ NEW_LINE (\r|\n|\r\n)
ungetCnt++ ungetCnt++
c = l.Next() c = l.Next()
if '\n' == rune(c) || '\r' == rune(c) { if '\n' == rune(c) || '\r' == rune(c) {
l.begin(HEREDOC_END) l.Begin(HEREDOC_END)
} }
} }
@ -407,14 +411,14 @@ NEW_LINE (\r|\n|\r\n)
if '\n' == rune(c) || '\r' == rune(c) { if '\n' == rune(c) || '\r' == rune(c) {
if l.heredocLabel + ";" == string(searchLabel) { if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END) l.Begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+1) tb = l.ungetChars(len(l.heredocLabel)+1)
tb = tb[:len(tb)-1] tb = tb[:len(tb)-1]
break; break;
} }
if l.heredocLabel == string(searchLabel) { if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END) l.Begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)) tb = l.ungetChars(len(l.heredocLabel))
tb = tb[:len(tb)-1] tb = tb[:len(tb)-1]
break; break;
@ -431,8 +435,8 @@ NEW_LINE (\r|\n|\r\n)
lval.Token(l.createToken(tb) ) lval.Token(l.createToken(tb) )
return int(T_ENCAPSED_AND_WHITESPACE) return int(T_ENCAPSED_AND_WHITESPACE)
<HEREDOC_END>{VAR_NAME}\; l.begin(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_END_HEREDOC) <HEREDOC_END>{VAR_NAME}\; l.Begin(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_END_HEREDOC)
<HEREDOC_END>{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_END_HEREDOC) <HEREDOC_END>{VAR_NAME} l.Begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_END_HEREDOC)
<PHP>[b]?[\"] <PHP>[b]?[\"]
binPrefix := l.Token()[0].Rune == 'b' binPrefix := l.Token()[0].Rune == 'b'
@ -586,7 +590,7 @@ NEW_LINE (\r|\n|\r\n)
case '\n': case '\n':
if l.heredocLabel + ";" == string(searchLabel) { if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END) l.Begin(HEREDOC_END)
l.ungetChars(len(l.heredocLabel)+1+nls) l.ungetChars(len(l.heredocLabel)+1+nls)
i := len(tb) - len(l.heredocLabel) - 3 - nls i := len(tb) - len(l.heredocLabel) - 3 - nls
@ -600,7 +604,7 @@ NEW_LINE (\r|\n|\r\n)
} }
if l.heredocLabel == string(searchLabel) { if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END) l.Begin(HEREDOC_END)
l.ungetChars(len(l.heredocLabel)+nls) l.ungetChars(len(l.heredocLabel)+nls)
i := len(tb) - len(l.heredocLabel) - 2 - nls i := len(tb) - len(l.heredocLabel) - 2 - nls
@ -667,6 +671,8 @@ NEW_LINE (\r|\n|\r\n)
<STRING_VAR_NAME>{VAR_NAME}[\[\}] l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_STRING_VARNAME) <STRING_VAR_NAME>{VAR_NAME}[\[\}] l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_STRING_VARNAME)
<STRING_VAR_NAME>. l.ungetChars(1);l.popState();l.pushState(PHP) <STRING_VAR_NAME>. l.ungetChars(1);l.popState();l.pushState(PHP)
<HALT_COMPILER>.|[ \t\n\r] // do nothing
%% %%
if _, ok := l.Abort(); ok { if _, ok := l.Abort(); ok {
// always return same $end token // always return same $end token

70
scanner/scanner_test.go
View File

@ -431,6 +431,53 @@ func TestTokens(t *testing.T) {
assertEqual(t, expected, actual) assertEqual(t, expected, actual)
} }
func TestSingleQuoteStringTokens(t *testing.T) {
src := `<?php
'str $var str'
'\''
'\'
'
'\
\''
'\\'
'\\
'
'\
\''
`
expected := []string{
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lv := &lval{}
actual := []string{}
for {
token := lexer.Lex(lv)
if token < 0 {
break
}
actual = append(actual, scanner.LexerToken(token).String())
}
assertEqual(t, expected, actual)
}
func TestTeplateStringTokens(t *testing.T) { func TestTeplateStringTokens(t *testing.T) {
src := `<?php src := `<?php
"foo $a" "foo $a"
@ -1033,13 +1080,34 @@ func TestInlineComment(t *testing.T) {
assertEqual(t, expected, actual) assertEqual(t, expected, actual)
} }
func TestInlineComment2(t *testing.T) {
src := `<?php
/*/*/`
expected := []meta.Meta{
meta.NewWhiteSpace("\n\t", position.NewPosition(1, 2, 6, 7)),
meta.NewComment("/*/*/", position.NewPosition(2, 2, 8, 12)),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Meta
assertEqual(t, expected, actual)
}
func TestEmptyInlineComment(t *testing.T) { func TestEmptyInlineComment(t *testing.T) {
src := `<?php src := `<?php
/**/` /**/ `
expected := []meta.Meta{ expected := []meta.Meta{
meta.NewWhiteSpace("\n\t", position.NewPosition(1, 2, 6, 7)), meta.NewWhiteSpace("\n\t", position.NewPosition(1, 2, 6, 7)),
meta.NewComment("/**/", position.NewPosition(2, 2, 8, 11)), meta.NewComment("/**/", position.NewPosition(2, 2, 8, 11)),
meta.NewWhiteSpace(" ", position.NewPosition(2, 2, 12, 12)),
} }
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")