maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor tokenString

Browse Source
This commit is contained in:
z7zmey 2018-06-05 15:20:23 +03:00
parent 95c257c0f6
commit d72d3b7980
6 changed files with 39 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
position/position_test.go
View File

@ -12,7 +12,7 @@ import (
func TestNewTokenPosition(t *testing.T) { func TestNewTokenPosition(t *testing.T) {
builder := position.Builder{} builder := position.Builder{}
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) tkn := token.NewToken(`foo`, 1, 1, 0, 3)
pos := builder.NewTokenPosition(tkn) pos := builder.NewTokenPosition(tkn)
@ -24,8 +24,8 @@ func TestNewTokenPosition(t *testing.T) {
func TestNewTokensPosition(t *testing.T) { func TestNewTokensPosition(t *testing.T) {
builder := position.Builder{} builder := position.Builder{}
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) token1 := token.NewToken(`foo`, 1, 1, 0, 3)
token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6) token2 := token.NewToken(`foo`, 2, 2, 4, 6)
pos := builder.NewTokensPosition(token1, token2) pos := builder.NewTokensPosition(token1, token2)
@ -57,7 +57,7 @@ func TestNewNodePosition(t *testing.T) {
} }
func TestNewTokenNodePosition(t *testing.T) { func TestNewTokenNodePosition(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) tkn := token.NewToken(`foo`, 1, 1, 0, 3)
n := node.NewIdentifier("test node") n := node.NewIdentifier("test node")
p := &position.Positions{} p := &position.Positions{}
@ -81,7 +81,7 @@ func TestNewTokenNodePosition(t *testing.T) {
func TestNewNodeTokenPosition(t *testing.T) { func TestNewNodeTokenPosition(t *testing.T) {
n := node.NewIdentifier("test node") n := node.NewIdentifier("test node")
tkn := token.NewToken([]byte(`foo`), 2, 2, 10, 12) tkn := token.NewToken(`foo`, 2, 2, 10, 12)
p := &position.Positions{} p := &position.Positions{}
p.AddPosition(n, &position.Position{ p.AddPosition(n, &position.Position{
@ -161,7 +161,7 @@ func TestNewNodesPosition(t *testing.T) {
func TestNewNodeListTokenPosition(t *testing.T) { func TestNewNodeListTokenPosition(t *testing.T) {
n1 := node.NewIdentifier("test node") n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node") n2 := node.NewIdentifier("test node")
tkn := token.NewToken([]byte(`foo`), 3, 3, 20, 22) tkn := token.NewToken(`foo`, 3, 3, 20, 22)
builder := position.Builder{ builder := position.Builder{
Positions: &position.Positions{ Positions: &position.Positions{
@ -188,7 +188,7 @@ func TestNewNodeListTokenPosition(t *testing.T) {
} }
func TestNewTokenNodeListPosition(t *testing.T) { func TestNewTokenNodeListPosition(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 2) tkn := token.NewToken(`foo`, 1, 1, 0, 2)
n1 := node.NewIdentifier("test node") n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node") n2 := node.NewIdentifier("test node")
@ -254,8 +254,8 @@ func TestNewNodeNodeListPosition(t *testing.T) {
func TestNewOptionalListTokensPosition(t *testing.T) { func TestNewOptionalListTokensPosition(t *testing.T) {
builder := position.Builder{} builder := position.Builder{}
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) token1 := token.NewToken(`foo`, 1, 1, 0, 3)
token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6) token2 := token.NewToken(`foo`, 2, 2, 4, 6)
pos := builder.NewOptionalListTokensPosition(nil, token1, token2) pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
@ -292,8 +292,8 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
}, },
} }
token1 := token.NewToken([]byte(`foo`), 4, 4, 27, 29) token1 := token.NewToken(`foo`, 4, 4, 27, 29)
token2 := token.NewToken([]byte(`foo`), 5, 5, 30, 32) token2 := token.NewToken(`foo`, 5, 5, 30, 32)
pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2) pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2)
@ -334,7 +334,7 @@ func TestNilNodeListPos(t *testing.T) {
} }
func TestNilNodeListTokenPos(t *testing.T) { func TestNilNodeListTokenPos(t *testing.T) {
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) token1 := token.NewToken(`foo`, 1, 1, 0, 3)
builder := position.Builder{} builder := position.Builder{}
@ -367,7 +367,7 @@ func TestEmptyNodeListPos(t *testing.T) {
} }
func TestEmptyNodeListTokenPos(t *testing.T) { func TestEmptyNodeListTokenPos(t *testing.T) {
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) token1 := token.NewToken(`foo`, 1, 1, 0, 3)
builder := position.Builder{} builder := position.Builder{}

16
scanner/lexer.go
View File

@ -440,7 +440,7 @@ type Lexer struct {
StateStack []int StateStack []int
PhpDocComment string PhpDocComment string
Comments []comment.Comment Comments []comment.Comment
heredocLabel []lex.Char heredocLabel string
tokenBytesBuf *bytes.Buffer tokenBytesBuf *bytes.Buffer
} }
@ -469,7 +469,7 @@ func NewLexer(src io.Reader, fName string) *Lexer {
if err != nil { if err != nil {
panic(err) panic(err)
} }
return &Lexer{lx, []int{0}, "", nil, nil, &bytes.Buffer{}} return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}}
} }
func (l *Lexer) ungetChars(n int) []lex.Char { func (l *Lexer) ungetChars(n int) []lex.Char {
@ -520,21 +520,19 @@ func (l *Lexer) newToken(chars []lex.Char) t.Token {
startPos := int(firstChar.Pos()) startPos := int(firstChar.Pos())
endPos := int(lastChar.Pos()) endPos := int(lastChar.Pos())
return t.NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments) return t.NewToken(l.tokenString(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments)
} }
func (l *Lexer) addComment(c comment.Comment) { func (l *Lexer) addComment(c comment.Comment) {
l.Comments = append(l.Comments, c) l.Comments = append(l.Comments, c)
} }
func (l *Lexer) charsToBytes(chars []lex.Char) []byte { func (l *Lexer) tokenString(chars []lex.Char) string {
l.tokenBytesBuf.Reset()
for _, c := range chars { for _, c := range chars {
l.tokenBytesBuf.WriteRune(c.Rune) l.tokenBytesBuf.WriteRune(c.Rune)
} }
r := l.tokenBytesBuf.Bytes() return string(l.tokenBytesBuf.Bytes())
l.tokenBytesBuf.Reset()
return r
} }

18
scanner/scanner.go
View File

@ -9,7 +9,6 @@
package scanner package scanner
import ( import (
"bytes"
"fmt" "fmt"
"github.com/cznic/golex/lex" "github.com/cznic/golex/lex"
"github.com/z7zmey/php-parser/comment" "github.com/z7zmey/php-parser/comment"
@ -8530,19 +8529,18 @@ yyrule141: // [b]?\<\<\<[ \t]*({VAR_NAME}|([']{VAR_NAME}['])|(["]{VAR_NAME}["]))
default: default:
l.begin(HEREDOC) l.begin(HEREDOC)
} }
l.heredocLabel = make([]lex.Char, lblLast-lblFirst+1) l.heredocLabel = l.tokenString(tb[lblFirst : lblLast+1])
copy(l.heredocLabel, tb[lblFirst:lblLast+1])
ungetCnt := len(l.heredocLabel) ungetCnt := len(l.heredocLabel)
searchLabelAhead := []lex.Char{} searchLabelAhead := []byte{}
for i := 0; i < len(l.heredocLabel); i++ { for i := 0; i < len(l.heredocLabel); i++ {
if c == -1 { if c == -1 {
break break
} }
searchLabelAhead = append(searchLabelAhead, l.Lookahead()) searchLabelAhead = append(searchLabelAhead, byte(rune(c)))
c = l.Next() c = l.Next()
} }
if bytes.Equal(l.charsToBytes(l.heredocLabel), l.charsToBytes(searchLabelAhead)) && ';' == rune(c) { if l.heredocLabel == string(searchLabelAhead) && ';' == rune(c) {
ungetCnt++ ungetCnt++
c = l.Next() c = l.Next()
if '\n' == rune(c) || '\r' == rune(c) { if '\n' == rune(c) || '\r' == rune(c) {
@ -8564,12 +8562,12 @@ yyrule142: // .|[ \t\n\r]
break break
} }
if '\n' == rune(c) || '\r' == rune(c) { if '\n' == rune(c) || '\r' == rune(c) {
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) { if l.heredocLabel+";" == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel) + 1) tb = l.ungetChars(len(l.heredocLabel) + 1)
break break
} }
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) { if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)) tb = l.ungetChars(len(l.heredocLabel))
break break
@ -8770,13 +8768,13 @@ yyrule152: // .|[ \t\n\r]
} }
fallthrough fallthrough
case '\n': case '\n':
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) { if l.heredocLabel+";" == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel) + 1 + nls) tb = l.ungetChars(len(l.heredocLabel) + 1 + nls)
lval.Token(l.newToken(tb)) lval.Token(l.newToken(tb))
return T_ENCAPSED_AND_WHITESPACE return T_ENCAPSED_AND_WHITESPACE
} }
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) { if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel) + nls) tb = l.ungetChars(len(l.heredocLabel) + nls)
lval.Token(l.newToken(tb)) lval.Token(l.newToken(tb))

18
scanner/scanner.l
View File

@ -9,7 +9,6 @@ package scanner
import ( import (
"fmt" "fmt"
"bytes"
"github.com/cznic/golex/lex" "github.com/cznic/golex/lex"
"github.com/z7zmey/php-parser/comment" "github.com/z7zmey/php-parser/comment"
) )
@ -379,20 +378,19 @@ NEW_LINE (\r|\n|\r\n)
l.begin(HEREDOC) l.begin(HEREDOC)
} }
l.heredocLabel = make([]lex.Char, lblLast - lblFirst + 1) l.heredocLabel = l.tokenString(tb[lblFirst:lblLast+1])
copy(l.heredocLabel, tb[lblFirst:lblLast+1])
ungetCnt := len(l.heredocLabel) ungetCnt := len(l.heredocLabel)
searchLabelAhead := []lex.Char{} searchLabelAhead := []byte{}
for i := 0; i < len(l.heredocLabel); i++ { for i := 0; i < len(l.heredocLabel); i++ {
if c == -1 { if c == -1 {
break; break;
} }
searchLabelAhead = append(searchLabelAhead, l.Lookahead()) searchLabelAhead = append(searchLabelAhead, byte(rune(c)))
c = l.Next() c = l.Next()
} }
if bytes.Equal(l.charsToBytes(l.heredocLabel), l.charsToBytes(searchLabelAhead)) && ';' == rune(c) { if l.heredocLabel == string(searchLabelAhead) && ';' == rune(c) {
ungetCnt++ ungetCnt++
c = l.Next() c = l.Next()
if '\n' == rune(c) || '\r' == rune(c) { if '\n' == rune(c) || '\r' == rune(c) {
@ -415,13 +413,13 @@ NEW_LINE (\r|\n|\r\n)
} }
if '\n' == rune(c) || '\r' == rune(c) { if '\n' == rune(c) || '\r' == rune(c) {
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) { if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+1) tb = l.ungetChars(len(l.heredocLabel)+1)
break; break;
} }
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) { if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)) tb = l.ungetChars(len(l.heredocLabel))
break; break;
@ -595,14 +593,14 @@ NEW_LINE (\r|\n|\r\n)
fallthrough fallthrough
case '\n': case '\n':
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) { if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+1+nls) tb = l.ungetChars(len(l.heredocLabel)+1+nls)
lval.Token(l.newToken(tb)); lval.Token(l.newToken(tb));
return T_ENCAPSED_AND_WHITESPACE return T_ENCAPSED_AND_WHITESPACE
} }
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) { if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+nls) tb = l.ungetChars(len(l.heredocLabel)+nls)
lval.Token(l.newToken(tb)); lval.Token(l.newToken(tb));

4
token/token.go
View File

@ -16,8 +16,8 @@ type Token struct {
// NewToken Token constructor // NewToken Token constructor
// TODO: return pointer // TODO: return pointer
func NewToken(value []byte, startLine int, endLine int, startPos int, endPos int) Token { func NewToken(value string, startLine int, endLine int, startPos int, endPos int) Token {
return Token{string(value), startLine, endLine, startPos, endPos, nil} return Token{value, startLine, endLine, startPos, endPos, nil}
} }
func (t Token) String() string { func (t Token) String() string {

2
token/token_test.go
View File

@ -10,7 +10,7 @@ import (
) )
func TestToken(t *testing.T) { func TestToken(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) tkn := token.NewToken(`foo`, 1, 1, 0, 3)
c := []comment.Comment{ c := []comment.Comment{
comment.NewPlainComment("test comment"), comment.NewPlainComment("test comment"),