From d72d3b7980876426403ac6adb4ef9b14d314b8f1 Mon Sep 17 00:00:00 2001 From: z7zmey Date: Tue, 5 Jun 2018 15:20:23 +0300 Subject: [PATCH] refactor tokenString --- position/position_test.go | 26 +++++++++++++------------- scanner/lexer.go | 16 +++++++--------- scanner/scanner.go | 18 ++++++++---------- scanner/scanner.l | 18 ++++++++---------- token/token.go | 4 ++-- token/token_test.go | 2 +- 6 files changed, 39 insertions(+), 45 deletions(-) diff --git a/position/position_test.go b/position/position_test.go index 7cbc878..933e3f7 100644 --- a/position/position_test.go +++ b/position/position_test.go @@ -12,7 +12,7 @@ import ( func TestNewTokenPosition(t *testing.T) { builder := position.Builder{} - tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) + tkn := token.NewToken(`foo`, 1, 1, 0, 3) pos := builder.NewTokenPosition(tkn) @@ -24,8 +24,8 @@ func TestNewTokenPosition(t *testing.T) { func TestNewTokensPosition(t *testing.T) { builder := position.Builder{} - token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) - token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6) + token1 := token.NewToken(`foo`, 1, 1, 0, 3) + token2 := token.NewToken(`foo`, 2, 2, 4, 6) pos := builder.NewTokensPosition(token1, token2) @@ -57,7 +57,7 @@ func TestNewNodePosition(t *testing.T) { } func TestNewTokenNodePosition(t *testing.T) { - tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) + tkn := token.NewToken(`foo`, 1, 1, 0, 3) n := node.NewIdentifier("test node") p := &position.Positions{} @@ -81,7 +81,7 @@ func TestNewTokenNodePosition(t *testing.T) { func TestNewNodeTokenPosition(t *testing.T) { n := node.NewIdentifier("test node") - tkn := token.NewToken([]byte(`foo`), 2, 2, 10, 12) + tkn := token.NewToken(`foo`, 2, 2, 10, 12) p := &position.Positions{} p.AddPosition(n, &position.Position{ @@ -161,7 +161,7 @@ func TestNewNodesPosition(t *testing.T) { func TestNewNodeListTokenPosition(t *testing.T) { n1 := node.NewIdentifier("test node") n2 := node.NewIdentifier("test node") - tkn := token.NewToken([]byte(`foo`), 3, 3, 20, 22) + tkn := token.NewToken(`foo`, 3, 3, 20, 22) builder := position.Builder{ Positions: &position.Positions{ @@ -188,7 +188,7 @@ func TestNewNodeListTokenPosition(t *testing.T) { } func TestNewTokenNodeListPosition(t *testing.T) { - tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 2) + tkn := token.NewToken(`foo`, 1, 1, 0, 2) n1 := node.NewIdentifier("test node") n2 := node.NewIdentifier("test node") @@ -254,8 +254,8 @@ func TestNewNodeNodeListPosition(t *testing.T) { func TestNewOptionalListTokensPosition(t *testing.T) { builder := position.Builder{} - token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) - token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6) + token1 := token.NewToken(`foo`, 1, 1, 0, 3) + token2 := token.NewToken(`foo`, 2, 2, 4, 6) pos := builder.NewOptionalListTokensPosition(nil, token1, token2) @@ -292,8 +292,8 @@ func TestNewOptionalListTokensPosition2(t *testing.T) { }, } - token1 := token.NewToken([]byte(`foo`), 4, 4, 27, 29) - token2 := token.NewToken([]byte(`foo`), 5, 5, 30, 32) + token1 := token.NewToken(`foo`, 4, 4, 27, 29) + token2 := token.NewToken(`foo`, 5, 5, 30, 32) pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2) @@ -334,7 +334,7 @@ func TestNilNodeListPos(t *testing.T) { } func TestNilNodeListTokenPos(t *testing.T) { - token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) + token1 := token.NewToken(`foo`, 1, 1, 0, 3) builder := position.Builder{} @@ -367,7 +367,7 @@ func TestEmptyNodeListPos(t *testing.T) { } func TestEmptyNodeListTokenPos(t *testing.T) { - token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) + token1 := token.NewToken(`foo`, 1, 1, 0, 3) builder := position.Builder{} diff --git a/scanner/lexer.go b/scanner/lexer.go index 59afec5..e50a7df 100644 --- a/scanner/lexer.go +++ b/scanner/lexer.go @@ -440,7 +440,7 @@ type Lexer struct { StateStack []int PhpDocComment string Comments []comment.Comment - heredocLabel []lex.Char + heredocLabel string tokenBytesBuf *bytes.Buffer } @@ -469,7 +469,7 @@ func NewLexer(src io.Reader, fName string) *Lexer { if err != nil { panic(err) } - return &Lexer{lx, []int{0}, "", nil, nil, &bytes.Buffer{}} + return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}} } func (l *Lexer) ungetChars(n int) []lex.Char { @@ -520,21 +520,19 @@ func (l *Lexer) newToken(chars []lex.Char) t.Token { startPos := int(firstChar.Pos()) endPos := int(lastChar.Pos()) - return t.NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments) + return t.NewToken(l.tokenString(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments) } func (l *Lexer) addComment(c comment.Comment) { l.Comments = append(l.Comments, c) } -func (l *Lexer) charsToBytes(chars []lex.Char) []byte { +func (l *Lexer) tokenString(chars []lex.Char) string { + l.tokenBytesBuf.Reset() + for _, c := range chars { l.tokenBytesBuf.WriteRune(c.Rune) } - r := l.tokenBytesBuf.Bytes() - - l.tokenBytesBuf.Reset() - - return r + return string(l.tokenBytesBuf.Bytes()) } diff --git a/scanner/scanner.go b/scanner/scanner.go index 68521d1..725a303 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -9,7 +9,6 @@ package scanner import ( - "bytes" "fmt" "github.com/cznic/golex/lex" "github.com/z7zmey/php-parser/comment" @@ -8530,19 +8529,18 @@ yyrule141: // [b]?\<\<\<[ \t]*({VAR_NAME}|([']{VAR_NAME}['])|(["]{VAR_NAME}["])) default: l.begin(HEREDOC) } - l.heredocLabel = make([]lex.Char, lblLast-lblFirst+1) - copy(l.heredocLabel, tb[lblFirst:lblLast+1]) + l.heredocLabel = l.tokenString(tb[lblFirst : lblLast+1]) ungetCnt := len(l.heredocLabel) - searchLabelAhead := []lex.Char{} + searchLabelAhead := []byte{} for i := 0; i < len(l.heredocLabel); i++ { if c == -1 { break } - searchLabelAhead = append(searchLabelAhead, l.Lookahead()) + searchLabelAhead = append(searchLabelAhead, byte(rune(c))) c = l.Next() } - if bytes.Equal(l.charsToBytes(l.heredocLabel), l.charsToBytes(searchLabelAhead)) && ';' == rune(c) { + if l.heredocLabel == string(searchLabelAhead) && ';' == rune(c) { ungetCnt++ c = l.Next() if '\n' == rune(c) || '\r' == rune(c) { @@ -8564,12 +8562,12 @@ yyrule142: // .|[ \t\n\r] break } if '\n' == rune(c) || '\r' == rune(c) { - if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) { + if l.heredocLabel+";" == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel) + 1) break } - if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) { + if l.heredocLabel == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel)) break @@ -8770,13 +8768,13 @@ yyrule152: // .|[ \t\n\r] } fallthrough case '\n': - if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) { + if l.heredocLabel+";" == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel) + 1 + nls) lval.Token(l.newToken(tb)) return T_ENCAPSED_AND_WHITESPACE } - if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) { + if l.heredocLabel == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel) + nls) lval.Token(l.newToken(tb)) diff --git a/scanner/scanner.l b/scanner/scanner.l index 8b5ea53..d107bb7 100644 --- a/scanner/scanner.l +++ b/scanner/scanner.l @@ -9,7 +9,6 @@ package scanner import ( "fmt" - "bytes" "github.com/cznic/golex/lex" "github.com/z7zmey/php-parser/comment" ) @@ -379,20 +378,19 @@ NEW_LINE (\r|\n|\r\n) l.begin(HEREDOC) } - l.heredocLabel = make([]lex.Char, lblLast - lblFirst + 1) - copy(l.heredocLabel, tb[lblFirst:lblLast+1]) + l.heredocLabel = l.tokenString(tb[lblFirst:lblLast+1]) ungetCnt := len(l.heredocLabel) - searchLabelAhead := []lex.Char{} + searchLabelAhead := []byte{} for i := 0; i < len(l.heredocLabel); i++ { if c == -1 { break; } - searchLabelAhead = append(searchLabelAhead, l.Lookahead()) + searchLabelAhead = append(searchLabelAhead, byte(rune(c))) c = l.Next() } - if bytes.Equal(l.charsToBytes(l.heredocLabel), l.charsToBytes(searchLabelAhead)) && ';' == rune(c) { + if l.heredocLabel == string(searchLabelAhead) && ';' == rune(c) { ungetCnt++ c = l.Next() if '\n' == rune(c) || '\r' == rune(c) { @@ -415,13 +413,13 @@ NEW_LINE (\r|\n|\r\n) } if '\n' == rune(c) || '\r' == rune(c) { - if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) { + if l.heredocLabel + ";" == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel)+1) break; } - if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) { + if l.heredocLabel == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel)) break; @@ -595,14 +593,14 @@ NEW_LINE (\r|\n|\r\n) fallthrough case '\n': - if bytes.Equal(append(l.charsToBytes(l.heredocLabel), ';'), searchLabel) { + if l.heredocLabel + ";" == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel)+1+nls) lval.Token(l.newToken(tb)); return T_ENCAPSED_AND_WHITESPACE } - if bytes.Equal(l.charsToBytes(l.heredocLabel), searchLabel) { + if l.heredocLabel == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel)+nls) lval.Token(l.newToken(tb)); diff --git a/token/token.go b/token/token.go index 0da6189..fe5d97c 100644 --- a/token/token.go +++ b/token/token.go @@ -16,8 +16,8 @@ type Token struct { // NewToken Token constructor // TODO: return pointer -func NewToken(value []byte, startLine int, endLine int, startPos int, endPos int) Token { - return Token{string(value), startLine, endLine, startPos, endPos, nil} +func NewToken(value string, startLine int, endLine int, startPos int, endPos int) Token { + return Token{value, startLine, endLine, startPos, endPos, nil} } func (t Token) String() string { diff --git a/token/token_test.go b/token/token_test.go index 5f61365..a890601 100644 --- a/token/token_test.go +++ b/token/token_test.go @@ -10,7 +10,7 @@ import ( ) func TestToken(t *testing.T) { - tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) + tkn := token.NewToken(`foo`, 1, 1, 0, 3) c := []comment.Comment{ comment.NewPlainComment("test comment"),