maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor tokenString

Browse Source
This commit is contained in:
z7zmey
2018-06-05 15:20:23 +03:00
parent 95c257c0f6
commit d72d3b7980
6 changed files with 39 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
scanner/lexer.go
View File

@@ -440,7 +440,7 @@ type Lexer struct {
StateStack []int
PhpDocComment string
Comments []comment.Comment
heredocLabel []lex.Char
heredocLabel string
tokenBytesBuf *bytes.Buffer
}
@@ -469,7 +469,7 @@ func NewLexer(src io.Reader, fName string) *Lexer {
if err != nil {
panic(err)
}
return &Lexer{lx, []int{0}, "", nil, nil, &bytes.Buffer{}}
return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}}
}
func (l *Lexer) ungetChars(n int) []lex.Char {
@@ -520,21 +520,19 @@ func (l *Lexer) newToken(chars []lex.Char) t.Token {
startPos := int(firstChar.Pos())
endPos := int(lastChar.Pos())
return t.NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments)
return t.NewToken(l.tokenString(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments)
}
func (l *Lexer) addComment(c comment.Comment) {
l.Comments = append(l.Comments, c)
}
func (l *Lexer) charsToBytes(chars []lex.Char) []byte {
func (l *Lexer) tokenString(chars []lex.Char) string {
l.tokenBytesBuf.Reset()
for _, c := range chars {
l.tokenBytesBuf.WriteRune(c.Rune)
}
r := l.tokenBytesBuf.Bytes()
l.tokenBytesBuf.Reset()
return r
return string(l.tokenBytesBuf.Bytes())
}

18
scanner/scanner.go
View File

@@ -9,7 +9,6 @@
package scanner
import (
"bytes"
"fmt"
"github.com/cznic/golex/lex"
"github.com/z7zmey/php-parser/comment"
@@ -8530,19 +8529,18 @@ yyrule141: // [b]?\<\<\<[ \t]*({VAR_NAME}|([']{VAR_NAME}['])|(["]{VAR_NAME}["]))
default:
l.begin(HEREDOC)
}
l.heredocLabel = make([]lex.Char, lblLast-lblFirst+1)
copy(l.heredocLabel, tb[lblFirst:lblLast+1])
l.heredocLabel = l.tokenString(tb[lblFirst : lblLast+1])
ungetCnt := len(l.heredocLabel)
searchLabelAhead := []lex.Char{}
searchLabelAhead := []byte{}
for i := 0; i < len(l.heredocLabel); i++ {
if c == -1 {
break
}
searchLabelAhead = append(searchLabelAhead, l.Lookahead())
searchLabelAhead = append(searchLabelAhead, byte(rune(c)))
c = l.Next()
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), l.charsToBytes(searchLabelAhead)) && ';' == rune(c) {
if l.heredocLabel == string(searchLabelAhead) && ';' == rune(c) {
ungetCnt++
c = l.Next()
if '\n' == rune(c) || '\r' == rune(c) {
@@ -8564,12 +8562,12 @@ yyrule142: // .|[ \t\n\r]
break
}
if '\n' == rune(c) || '\r' == rune(c) {
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) {
if l.heredocLabel+";" == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel) + 1)
break
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) {
if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel))
break
@@ -8770,13 +8768,13 @@ yyrule152: // .|[ \t\n\r]
}
fallthrough
case '\n':
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) {
if l.heredocLabel+";" == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel) + 1 + nls)
lval.Token(l.newToken(tb))
return T_ENCAPSED_AND_WHITESPACE
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) {
if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel) + nls)
lval.Token(l.newToken(tb))

18
scanner/scanner.l
View File

@@ -9,7 +9,6 @@ package scanner
import (
"fmt"
"bytes"
"github.com/cznic/golex/lex"
"github.com/z7zmey/php-parser/comment"
)
@@ -379,20 +378,19 @@ NEW_LINE (\r|\n|\r\n)
l.begin(HEREDOC)
}
l.heredocLabel = make([]lex.Char, lblLast - lblFirst + 1)
copy(l.heredocLabel, tb[lblFirst:lblLast+1])
l.heredocLabel = l.tokenString(tb[lblFirst:lblLast+1])
ungetCnt := len(l.heredocLabel)
searchLabelAhead := []lex.Char{}
searchLabelAhead := []byte{}
for i := 0; i < len(l.heredocLabel); i++ {
if c == -1 {
break;
}
searchLabelAhead = append(searchLabelAhead, l.Lookahead())
searchLabelAhead = append(searchLabelAhead, byte(rune(c)))
c = l.Next()
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), l.charsToBytes(searchLabelAhead)) && ';' == rune(c) {
if l.heredocLabel == string(searchLabelAhead) && ';' == rune(c) {
ungetCnt++
c = l.Next()
if '\n' == rune(c) || '\r' == rune(c) {
@@ -415,13 +413,13 @@ NEW_LINE (\r|\n|\r\n)
}
if '\n' == rune(c) || '\r' == rune(c) {
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) {
if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+1)
break;
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) {
if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel))
break;
@@ -595,14 +593,14 @@ NEW_LINE (\r|\n|\r\n)
fallthrough
case '\n':
if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) {
if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+1+nls)
lval.Token(l.newToken(tb));
return T_ENCAPSED_AND_WHITESPACE
}
if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) {
if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+nls)
lval.Token(l.newToken(tb));