maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#13: the printer prints nodes with meta(comments and whitespaces) saved by the parser

Browse Source
This commit is contained in:
z7zmey
2018-07-02 20:48:55 +03:00
parent c4c0934c51
commit 15a1a81924
30 changed files with 14110 additions and 4465 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
scanner/lexer.go
View File

@@ -38,6 +38,7 @@ type Lexer struct {
tokenBytesBuf *bytes.Buffer
TokenPool sync.Pool
WithMeta bool
lastToken *Token
}
// Rune2Class returns the rune integer id

1001
scanner/scanner.go
View File

File diff suppressed because it is too large Load Diff

14
scanner/scanner.l
View File

@@ -191,7 +191,7 @@ NEW_LINE (\r|\n|\r\n)
<PHP>use lval.Token(l.createToken(l.Token())); return T_USE
<PHP>var lval.Token(l.createToken(l.Token())); return T_VAR
<PHP>while lval.Token(l.createToken(l.Token())); return T_WHILE
<PHP>yield[ \t\n\r]+from[^a-zA-Z0-9_\x80-\xff] lval.Token(l.createToken(l.Token())); return T_YIELD_FROM
<PHP>yield[ \t\n\r]+from lval.Token(l.createToken(l.Token())); return T_YIELD_FROM
<PHP>yield lval.Token(l.createToken(l.Token())); return T_YIELD
<PHP>include lval.Token(l.createToken(l.Token())); return T_INCLUDE
<PHP>include_once lval.Token(l.createToken(l.Token())); return T_INCLUDE_ONCE
@@ -266,6 +266,7 @@ NEW_LINE (\r|\n|\r\n)
}
case '\n':
c = l.Next()
case '?':
c = l.Next()
@@ -322,7 +323,7 @@ NEW_LINE (\r|\n|\r\n)
<PHP>{VAR_NAME} lval.Token(l.createToken(l.Token())); return T_STRING
<PHP>-> l.begin(PROPERTY);lval.Token(l.createToken(l.Token())); return T_OBJECT_OPERATOR;
<PROPERTY>[ \t\n\r]+
<PROPERTY>[ \t\n\r]+ l.addWhiteSpace(l.Token())
<PROPERTY>-> lval.Token(l.createToken(l.Token())); return T_OBJECT_OPERATOR;
<PROPERTY>{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return T_STRING;
<PROPERTY>. l.ungetChars(1);l.begin(PHP)
@@ -650,6 +651,13 @@ NEW_LINE (\r|\n|\r\n)
<STRING_VAR_NAME>. l.ungetChars(1);l.popState();l.pushState(PHP)
%%
if c, ok := l.Abort(); ok { return int(c) }
if _, ok := l.Abort(); ok {
// always return same $end token
if l.lastToken == nil {
l.lastToken = l.createToken(l.Token())
}
lval.Token(l.lastToken);
return -1
}
goto yyAction
}

95
scanner/scanner_test.go
View File

@@ -904,7 +904,6 @@ func TestCommentNewLine(t *testing.T) {
expected := []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(1, 1, 6, 6)),
meta.NewComment("//test\n", position.NewPosition(1, 1, 7, 13)),
meta.NewWhiteSpace("\n", position.NewPosition(1, 1, 13, 13)),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
@@ -913,7 +912,7 @@ func TestCommentNewLine(t *testing.T) {
lexer.Lex(lv)
actual := lexer.Meta
actual := lv.Tkn.Meta
assertEqual(t, expected, actual)
}
@@ -932,7 +931,7 @@ func TestCommentNewLine1(t *testing.T) {
lexer.Lex(lv)
actual := lexer.Meta
actual := lv.Tkn.Meta
assertEqual(t, expected, actual)
}
@@ -943,7 +942,6 @@ func TestCommentNewLine2(t *testing.T) {
expected := []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(1, 1, 6, 6)),
meta.NewComment("#test\r\n", position.NewPosition(1, 1, 7, 13)),
meta.NewWhiteSpace("\n", position.NewPosition(1, 1, 13, 13)),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
@@ -952,7 +950,7 @@ func TestCommentNewLine2(t *testing.T) {
lexer.Lex(lv)
actual := lexer.Meta
actual := lv.Tkn.Meta
assertEqual(t, expected, actual)
}
@@ -972,7 +970,7 @@ func TestCommentWithPhpEndTag(t *testing.T) {
lexer.Lex(lv)
actual := lexer.Meta
actual := lv.Tkn.Meta
assertEqual(t, expected, actual)
}
@@ -992,7 +990,7 @@ func TestInlineComment(t *testing.T) {
lexer.Lex(lv)
actual := lexer.Meta
actual := lv.Tkn.Meta
assertEqual(t, expected, actual)
}
@@ -1032,7 +1030,88 @@ func TestEmptyInlineComment2(t *testing.T) {
lexer.Lex(lv)
actual := lexer.Meta
actual := lv.Tkn.Meta
assertEqual(t, expected, actual)
}
func TestMethodCallTokens(t *testing.T) {
src := `<?php
$a -> bar ( '' ) ;`
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
expected := []meta.Meta{
meta.NewWhiteSpace("\n\t", position.NewPosition(1, 2, 6, 7)),
}
lexer.Lex(lv)
actual := lv.Tkn.Meta
assertEqual(t, expected, actual)
expected = []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(2, 2, 10, 10)),
}
lexer.Lex(lv)
actual = lv.Tkn.Meta
assertEqual(t, expected, actual)
expected = []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(2, 2, 13, 13)),
}
lexer.Lex(lv)
actual = lv.Tkn.Meta
assertEqual(t, expected, actual)
expected = []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(2, 2, 17, 17)),
}
lexer.Lex(lv)
actual = lv.Tkn.Meta
assertEqual(t, expected, actual)
expected = []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(2, 2, 19, 19)),
}
lexer.Lex(lv)
actual = lv.Tkn.Meta
assertEqual(t, expected, actual)
expected = []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(2, 2, 22, 22)),
}
lexer.Lex(lv)
actual = lv.Tkn.Meta
assertEqual(t, expected, actual)
expected = []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(2, 2, 24, 24)),
}
lexer.Lex(lv)
actual = lv.Tkn.Meta
assertEqual(t, expected, actual)
}
func TestYieldFromTokens(t *testing.T) {
src := `<?php
yield from $a`
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
expected := []meta.Meta{
meta.NewWhiteSpace("\n\t", position.NewPosition(1, 2, 6, 7)),
}
lexer.Lex(lv)
actual := lv.Tkn.Meta
assertEqual(t, expected, actual)
expected = []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(2, 2, 18, 18)),
}
lexer.Lex(lv)
actual = lv.Tkn.Meta
assertEqual(t, expected, actual)
}