maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor tokenString

Browse Source
This commit is contained in:
z7zmey 2018-06-05 15:20:23 +03:00
parent 95c257c0f6
commit d72d3b7980
6 changed files with 39 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
position/position_test.go
View File

@ -12,7 +12,7 @@ import (
func TestNewTokenPosition(t *testing.T) {
builder := position.Builder{}
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
tkn := token.NewToken(`foo`, 1, 1, 0, 3)
pos := builder.NewTokenPosition(tkn)
@ -24,8 +24,8 @@ func TestNewTokenPosition(t *testing.T) {
func TestNewTokensPosition(t *testing.T) {
builder := position.Builder{}
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6)
token1 := token.NewToken(`foo`, 1, 1, 0, 3)
token2 := token.NewToken(`foo`, 2, 2, 4, 6)
pos := builder.NewTokensPosition(token1, token2)
@ -57,7 +57,7 @@ func TestNewNodePosition(t *testing.T) {
}
func TestNewTokenNodePosition(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
tkn := token.NewToken(`foo`, 1, 1, 0, 3)
n := node.NewIdentifier("test node")
p := &position.Positions{}
@ -81,7 +81,7 @@ func TestNewTokenNodePosition(t *testing.T) {
func TestNewNodeTokenPosition(t *testing.T) {
n := node.NewIdentifier("test node")
tkn := token.NewToken([]byte(`foo`), 2, 2, 10, 12)
tkn := token.NewToken(`foo`, 2, 2, 10, 12)
p := &position.Positions{}
p.AddPosition(n, &position.Position{
@ -161,7 +161,7 @@ func TestNewNodesPosition(t *testing.T) {
func TestNewNodeListTokenPosition(t *testing.T) {
n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node")
tkn := token.NewToken([]byte(`foo`), 3, 3, 20, 22)
tkn := token.NewToken(`foo`, 3, 3, 20, 22)
builder := position.Builder{
Positions: &position.Positions{
@ -188,7 +188,7 @@ func TestNewNodeListTokenPosition(t *testing.T) {
}
func TestNewTokenNodeListPosition(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 2)
tkn := token.NewToken(`foo`, 1, 1, 0, 2)
n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node")
@ -254,8 +254,8 @@ func TestNewNodeNodeListPosition(t *testing.T) {
func TestNewOptionalListTokensPosition(t *testing.T) {
builder := position.Builder{}
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6)
token1 := token.NewToken(`foo`, 1, 1, 0, 3)
token2 := token.NewToken(`foo`, 2, 2, 4, 6)
pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
@ -292,8 +292,8 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
},
}
token1 := token.NewToken([]byte(`foo`), 4, 4, 27, 29)
token2 := token.NewToken([]byte(`foo`), 5, 5, 30, 32)
token1 := token.NewToken(`foo`, 4, 4, 27, 29)
token2 := token.NewToken(`foo`, 5, 5, 30, 32)
pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2)
@ -334,7 +334,7 @@ func TestNilNodeListPos(t *testing.T) {
}
func TestNilNodeListTokenPos(t *testing.T) {
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
token1 := token.NewToken(`foo`, 1, 1, 0, 3)
builder := position.Builder{}
@ -367,7 +367,7 @@ func TestEmptyNodeListPos(t *testing.T) {
}
func TestEmptyNodeListTokenPos(t *testing.T) {
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
token1 := token.NewToken(`foo`, 1, 1, 0, 3)
builder := position.Builder{}

16
scanner/lexer.go
View File

@ -440,7 +440,7 @@ type Lexer struct {
StateStack []int
PhpDocComment string
Comments []comment.Comment
heredocLabel []lex.Char
heredocLabel string
tokenBytesBuf *bytes.Buffer
}
@ -469,7 +469,7 @@ func NewLexer(src io.Reader, fName string) *Lexer {
if err != nil {
panic(err)
}
return &Lexer{lx, []int{0}, "", nil, nil, &bytes.Buffer{}}
return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}}
}
func (l *Lexer) ungetChars(n int) []lex.Char {
@ -520,21 +520,19 @@ func (l *Lexer) newToken(chars []lex.Char) t.Token {
startPos := int(firstChar.Pos())
endPos := int(lastChar.Pos())
return t.NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments)
return t.NewToken(l.tokenString(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments)
}
func (l *Lexer) addComment(c comment.Comment) {
l.Comments = append(l.Comments, c)
}
func (l *Lexer) charsToBytes(chars []lex.Char) []byte {
func (l *Lexer) tokenString(chars []lex.Char) string {
l.tokenBytesBuf.Reset()
for _, c := range chars {
l.tokenBytesBuf.WriteRune(c.Rune)
}
r := l.tokenBytesBuf.Bytes()
l.tokenBytesBuf.Reset()
return r
return string(l.tokenBytesBuf.Bytes())
}

18
scanner/scanner.go
View File

@ -9,7 +9,6 @@
package scanner
import (
"bytes"
"fmt"
"github.com/cznic/golex/lex"
"github.com/z7zmey/php-parser/comment"
@ -8530,19 +8529,18 @@ yyrule141: // [b]?\<\<\<[ \t]*({VAR_NAME}|([']{VAR_NAME}['])|(["]{VAR_NAME}["]))
default:
l.begin(HEREDOC)
}
l.heredocLabel = make([]lex.Char, lblLast-lblFirst+1)
copy(l.heredocLabel, tb[lblFirst:lblLast+1])
l.heredocLabel = l.tokenString(tb[lblFirst : lblLast+1])
ungetCnt := len(l.heredocLabel)
searchLabelAhead := []lex.Char{}
searchLabelAhead := []byte{}
for i := 0; i < len(l.heredocLabel); i++ {
if c == -1 {
break
}
searchLabelAhead = append(searchLabelAhead, l.Lookahead())
searchLabelAhead = append(searchLabelAhead, byte(rune(c)))
c = l.Next()
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), l.charsToBytes(searchLabelAhead)) && ';' == rune(c) {
if l.heredocLabel == string(searchLabelAhead) && ';' == rune(c) {
ungetCnt++
c = l.Next()
if '\n' == rune(c) || '\r' == rune(c) {
@ -8564,12 +8562,12 @@ yyrule142: // .|[ \t\n\r]
break
}
if '\n' == rune(c) || '\r' == rune(c) {
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) {
if l.heredocLabel+";" == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel) + 1)
break
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) {
if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel))
break
@ -8770,13 +8768,13 @@ yyrule152: // .|[ \t\n\r]
}
fallthrough
case '\n':
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) {
if l.heredocLabel+";" == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel) + 1 + nls)
lval.Token(l.newToken(tb))
return T_ENCAPSED_AND_WHITESPACE
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) {
if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel) + nls)
lval.Token(l.newToken(tb))

18
scanner/scanner.l
View File

@ -9,7 +9,6 @@ package scanner
import (
"fmt"
"bytes"
"github.com/cznic/golex/lex"
"github.com/z7zmey/php-parser/comment"
)
@ -379,20 +378,19 @@ NEW_LINE (\r|\n|\r\n)
l.begin(HEREDOC)
}
l.heredocLabel = make([]lex.Char, lblLast - lblFirst + 1)
copy(l.heredocLabel, tb[lblFirst:lblLast+1])
l.heredocLabel = l.tokenString(tb[lblFirst:lblLast+1])
ungetCnt := len(l.heredocLabel)
searchLabelAhead := []lex.Char{}
searchLabelAhead := []byte{}
for i := 0; i < len(l.heredocLabel); i++ {
if c == -1 {
break;
}
searchLabelAhead = append(searchLabelAhead, l.Lookahead())
searchLabelAhead = append(searchLabelAhead, byte(rune(c)))
c = l.Next()
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), l.charsToBytes(searchLabelAhead)) && ';' == rune(c) {
if l.heredocLabel == string(searchLabelAhead) && ';' == rune(c) {
ungetCnt++
c = l.Next()
if '\n' == rune(c) || '\r' == rune(c) {
@ -415,13 +413,13 @@ NEW_LINE (\r|\n|\r\n)
}
if '\n' == rune(c) || '\r' == rune(c) {
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) {
if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+1)
break;
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) {
if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel))
break;
@ -595,14 +593,14 @@ NEW_LINE (\r|\n|\r\n)
fallthrough
case '\n':
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) {
if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+1+nls)
lval.Token(l.newToken(tb));
return T_ENCAPSED_AND_WHITESPACE
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) {
if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+nls)
lval.Token(l.newToken(tb));

4
token/token.go
View File

@ -16,8 +16,8 @@ type Token struct {
// NewToken Token constructor
// TODO: return pointer
func NewToken(value []byte, startLine int, endLine int, startPos int, endPos int) Token {
return Token{string(value), startLine, endLine, startPos, endPos, nil}
func NewToken(value string, startLine int, endLine int, startPos int, endPos int) Token {
return Token{value, startLine, endLine, startPos, endPos, nil}
}
func (t Token) String() string {

2
token/token_test.go
View File

@ -10,7 +10,7 @@ import (
)
func TestToken(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
tkn := token.NewToken(`foo`, 1, 1, 0, 3)
c := []comment.Comment{
comment.NewPlainComment("test comment"),