maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#33 comment package has renamed to meta and parser now saves whitespaces

Browse Source
This commit is contained in:
z7zmey
2018-06-30 00:51:11 +03:00
parent 36d0cf4823
commit e90df8ef5f
205 changed files with 16485 additions and 16302 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
scanner/lexer.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/z7zmey/php-parser/position"
"github.com/cznic/golex/lex"
"github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/meta"
)
// Allocate Character classes anywhere in [0x80, 0xFF].
@@ -441,10 +441,11 @@ type Lexer struct {
*lex.Lexer
StateStack []int
PhpDocComment string
Comments []*comment.Comment
Meta []meta.Meta
heredocLabel string
tokenBytesBuf *bytes.Buffer
TokenPool sync.Pool
WithMeta bool
}
// Rune2Class returns the rune integer id
@@ -477,7 +478,7 @@ func NewLexer(src io.Reader, fName string) *Lexer {
Lexer: lx,
StateStack: []int{0},
PhpDocComment: "",
Comments: nil,
Meta: nil,
heredocLabel: "",
tokenBytesBuf: &bytes.Buffer{},
TokenPool: sync.Pool{
@@ -530,7 +531,7 @@ func (l *Lexer) createToken(chars []lex.Char) *Token {
lastChar := chars[len(chars)-1]
token := l.TokenPool.Get().(*Token)
token.Comments = l.Comments
token.Meta = l.Meta
token.Value = l.tokenString(chars)
token.StartLine = l.File.Line(firstChar.Pos())
@@ -541,7 +542,11 @@ func (l *Lexer) createToken(chars []lex.Char) *Token {
return token
}
func (l *Lexer) addComment(chars []lex.Char) {
func (l *Lexer) addComments(chars []lex.Char) {
if !l.WithMeta {
return
}
firstChar := chars[0]
lastChar := chars[len(chars)-1]
@@ -552,8 +557,27 @@ func (l *Lexer) addComment(chars []lex.Char) {
int(lastChar.Pos()),
)
c := comment.NewComment(l.tokenString(chars), pos)
l.Comments = append(l.Comments, c)
c := meta.NewComment(l.tokenString(chars), pos)
l.Meta = append(l.Meta, c)
}
func (l *Lexer) addWhiteSpace(chars []lex.Char) {
if !l.WithMeta {
return
}
firstChar := chars[0]
lastChar := chars[len(chars)-1]
pos := position.NewPosition(
l.File.Line(firstChar.Pos()),
l.File.Line(lastChar.Pos()),
int(firstChar.Pos()),
int(lastChar.Pos()),
)
c := meta.NewWhiteSpace(l.tokenString(chars), pos)
l.Meta = append(l.Meta, c)
}
func (l *Lexer) tokenString(chars []lex.Char) string {

15
scanner/scanner.go
View File

@@ -32,7 +32,7 @@ func isValidFirstVarNameRune(r rune) bool {
}
func (l *Lexer) Lex(lval Lval) int {
l.Comments = nil
l.Meta = nil
c := l.Enter()
yystate0:
@@ -7556,6 +7556,7 @@ yyrule2: // .
yyrule3: // \<\?php([ \t]|{NEW_LINE})
{
l.begin(PHP)
l.ungetChars(len(l.Token()) - 5)
goto yystate0
}
yyrule4: // \<\?
@@ -7571,8 +7572,10 @@ yyrule5: // \<\?=
goto yystate0
}
yyrule6: // [ \t\n\r]+
goto yystate0
{
l.addWhiteSpace(l.Token())
goto yystate0
}
yyrule7: // [;][ \t\n\r]*\?\>{NEW_LINE}?
{
l.begin(INITIAL)
@@ -8367,7 +8370,7 @@ yyrule126: // (#|[/][/])
}
break
}
l.addComment(tb)
l.addComments(tb)
goto yystate0
}
yyrule127: // ([/][*])|([/][*][*])
@@ -8391,9 +8394,9 @@ yyrule127: // ([/][*])|([/][*][*])
}
if is_doc_comment {
l.PhpDocComment = string(l.TokenBytes(nil))
l.addComment(l.Token())
l.addComments(l.Token())
} else {
l.addComment(l.Token())
l.addComments(l.Token())
}
goto yystate0
}

12
scanner/scanner.l
View File

@@ -31,7 +31,7 @@ func isValidFirstVarNameRune(r rune) bool {
}
func (l *Lexer) Lex(lval Lval) int {
l.Comments = nil
l.Meta = nil
c := l.Enter()
%}
@@ -84,11 +84,11 @@ NEW_LINE (\r|\n|\r\n)
lval.Token(l.createToken(tb))
return T_INLINE_HTML
<INITIAL>\<\?php([ \t]|{NEW_LINE}) l.begin(PHP);
<INITIAL>\<\?php([ \t]|{NEW_LINE}) l.begin(PHP);l.ungetChars(len(l.Token())-5)
<INITIAL>\<\? l.begin(PHP);
<INITIAL>\<\?= l.begin(PHP);lval.Token(l.createToken(l.Token())); return T_ECHO;
<PHP>[ \t\n\r]+
<PHP>[ \t\n\r]+ l.addWhiteSpace(l.Token())
<PHP>[;][ \t\n\r]*\?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
<PHP>\?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
@@ -284,7 +284,7 @@ NEW_LINE (\r|\n|\r\n)
break;
}
l.addComment(tb)
l.addComments(tb)
<PHP>([/][*])|([/][*][*])
tb := l.Token()
@@ -309,9 +309,9 @@ NEW_LINE (\r|\n|\r\n)
if is_doc_comment {
l.PhpDocComment = string(l.TokenBytes(nil))
l.addComment(l.Token())
l.addComments(l.Token())
} else {
l.addComment(l.Token())
l.addComments(l.Token())
}
<PHP>{OPERATORS} lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))

74
scanner/scanner_test.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/meta"
"github.com/z7zmey/php-parser/scanner"
@@ -415,6 +415,7 @@ func TestTokens(t *testing.T) {
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
@@ -503,6 +504,7 @@ func TestTeplateStringTokens(t *testing.T) {
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
@@ -588,6 +590,7 @@ func TestBackquoteStringTokens(t *testing.T) {
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
@@ -681,6 +684,7 @@ CAT;
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
@@ -754,6 +758,7 @@ CAT
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
@@ -786,6 +791,7 @@ func TestInlineHtmlNopTokens(t *testing.T) {
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
@@ -876,16 +882,18 @@ func TestSlashAfterVariable(t *testing.T) {
func TestCommentEnd(t *testing.T) {
src := `<?php //test`
expected := []*comment.Comment{
comment.NewComment("//test", position.NewPosition(1, 1, 7, 12)),
expected := []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(1, 1, 6, 6)),
meta.NewComment("//test", position.NewPosition(1, 1, 7, 12)),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Comments
actual := lexer.Meta
assertEqual(t, expected, actual)
}
@@ -893,16 +901,19 @@ func TestCommentEnd(t *testing.T) {
func TestCommentNewLine(t *testing.T) {
src := "<?php //test\n$a"
expected := []*comment.Comment{
comment.NewComment("//test\n", position.NewPosition(1, 1, 7, 13)),
expected := []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(1, 1, 6, 6)),
meta.NewComment("//test\n", position.NewPosition(1, 1, 7, 13)),
meta.NewWhiteSpace("\n", position.NewPosition(1, 1, 13, 13)),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Comments
actual := lexer.Meta
assertEqual(t, expected, actual)
}
@@ -910,16 +921,18 @@ func TestCommentNewLine(t *testing.T) {
func TestCommentNewLine1(t *testing.T) {
src := "<?php //test\r$a"
expected := []*comment.Comment{
comment.NewComment("//test\r", position.NewPosition(1, 1, 7, 13)),
expected := []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(1, 1, 6, 6)),
meta.NewComment("//test\r", position.NewPosition(1, 1, 7, 13)),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Comments
actual := lexer.Meta
assertEqual(t, expected, actual)
}
@@ -927,16 +940,19 @@ func TestCommentNewLine1(t *testing.T) {
func TestCommentNewLine2(t *testing.T) {
src := "<?php #test\r\n$a"
expected := []*comment.Comment{
comment.NewComment("#test\r\n", position.NewPosition(1, 1, 7, 13)),
expected := []meta.Meta{
meta.NewWhiteSpace(" ", position.NewPosition(1, 1, 6, 6)),
meta.NewComment("#test\r\n", position.NewPosition(1, 1, 7, 13)),
meta.NewWhiteSpace("\n", position.NewPosition(1, 1, 13, 13)),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Comments
actual := lexer.Meta
assertEqual(t, expected, actual)
}
@@ -945,16 +961,18 @@ func TestCommentWithPhpEndTag(t *testing.T) {
src := `<?php
//test?> test`
expected := []*comment.Comment{
comment.NewComment("//test", position.NewPosition(2, 2, 8, 13)),
expected := []meta.Meta{
meta.NewWhiteSpace("\n\t", position.NewPosition(1, 2, 6, 7)),
meta.NewComment("//test", position.NewPosition(2, 2, 8, 13)),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Comments
actual := lexer.Meta
assertEqual(t, expected, actual)
}
@@ -963,16 +981,18 @@ func TestInlineComment(t *testing.T) {
src := `<?php
/*test*/`
expected := []*comment.Comment{
comment.NewComment("/*test*/", position.NewPosition(2, 2, 8, 15)),
expected := []meta.Meta{
meta.NewWhiteSpace("\n\t", position.NewPosition(1, 2, 6, 7)),
meta.NewComment("/*test*/", position.NewPosition(2, 2, 8, 15)),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Comments
actual := lexer.Meta
assertEqual(t, expected, actual)
}
@@ -981,16 +1001,18 @@ func TestEmptyInlineComment(t *testing.T) {
src := `<?php
/**/`
expected := []*comment.Comment{
comment.NewComment("/**/", position.NewPosition(2, 2, 8, 11)),
expected := []meta.Meta{
meta.NewWhiteSpace("\n\t", position.NewPosition(1, 2, 6, 7)),
meta.NewComment("/**/", position.NewPosition(2, 2, 8, 11)),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Comments
actual := lexer.Meta
assertEqual(t, expected, actual)
}
@@ -999,16 +1021,18 @@ func TestEmptyInlineComment2(t *testing.T) {
src := `<?php
/***/`
expected := []*comment.Comment{
comment.NewComment("/***/", position.NewPosition(2, 2, 8, 12)),
expected := []meta.Meta{
meta.NewWhiteSpace("\n\t", position.NewPosition(1, 2, 6, 7)),
meta.NewComment("/***/", position.NewPosition(2, 2, 8, 12)),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Comments
actual := lexer.Meta
assertEqual(t, expected, actual)
}

4
scanner/token.go
View File

@@ -1,13 +1,13 @@
package scanner
import (
"github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/meta"
)
// Token value returned by lexer
type Token struct {
Value string
Comments []*comment.Comment
Meta []meta.Meta
StartLine int
EndLine int
StartPos int

10
scanner/token_test.go
View File

@@ -4,7 +4,7 @@ import (
"reflect"
"testing"
"github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/meta"
"github.com/z7zmey/php-parser/scanner"
)
@@ -18,13 +18,13 @@ func TestToken(t *testing.T) {
EndPos: 3,
}
c := []*comment.Comment{
comment.NewComment("test comment", nil),
c := []meta.Meta{
meta.NewComment("test comment", nil),
}
tkn.Comments = c
tkn.Meta = c
if !reflect.DeepEqual(tkn.Comments, c) {
if !reflect.DeepEqual(tkn.Meta, c) {
t.Errorf("comments are not equal\n")
}