#27 reduce memory allocations for scanner.Token by using sync.Pool

scanner/lexer.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"go/token"
 	"io"
+	"sync"
 	"unicode"
 
 	"github.com/z7zmey/php-parser/position"
@@ -443,6 +444,7 @@ type Lexer struct {
 	Comments      []*comment.Comment
 	heredocLabel  string
 	tokenBytesBuf *bytes.Buffer
+	TokenPool     sync.Pool
 }
 
 // Rune2Class returns the rune integer id
@@ -470,7 +472,12 @@ func NewLexer(src io.Reader, fName string) *Lexer {
 	if err != nil {
 		panic(err)
 	}
-	return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}}
+
+	var TokenPool = sync.Pool{
+		New: func() interface{} { return &Token{} },
+	}
+
+	return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}, TokenPool}
 }
 
 func (l *Lexer) ungetChars(n int) []lex.Char {
@@ -523,7 +530,14 @@ func (l *Lexer) createToken(chars []lex.Char) *Token {
 		int(lastChar.Pos()),
 	)
 
-	return NewToken(l.tokenString(chars), pos).SetComments(l.Comments)
+	token := l.TokenPool.Get().(*Token)
+	token.Position = pos
+	token.Comments = l.Comments
+	token.Value = l.tokenString(chars)
+
+	return token
+
+	// return NewToken(l.tokenString(chars), pos).SetComments(l.Comments)
 }
 
 func (l *Lexer) addComment(chars []lex.Char) {

scanner/token.go

@@ -8,35 +8,10 @@ import (
 // Token value returned by lexer
 type Token struct {
 	Value    string
-	position *position.Position
-	comments []*comment.Comment
-}
-
-// NewToken Token constructor
-func NewToken(value string, pos *position.Position) *Token {
-	return &Token{
-		Value:    value,
-		position: pos,
-		comments: nil,
-	}
+	Position *position.Position
+	Comments []*comment.Comment
 }
 
 func (t *Token) String() string {
 	return string(t.Value)
 }
-
-// Position returns token position
-func (t *Token) Position() *position.Position {
-	return t.position
-}
-
-// Comments returns attached comments
-func (t *Token) Comments() []*comment.Comment {
-	return t.comments
-}
-
-// SetComments attach comments
-func (t *Token) SetComments(comments []*comment.Comment) *Token {
-	t.comments = comments
-	return t
-}

scanner/token_test.go

@@ -13,15 +13,18 @@ import (
 
 func TestToken(t *testing.T) {
 	pos := position.NewPosition(1, 1, 0, 3)
-	tkn := scanner.NewToken(`foo`, pos)
+	tkn := &scanner.Token{
+		Value:    `foo`,
+		Position: pos,
+	}
 
 	c := []*comment.Comment{
 		comment.NewComment("test comment", nil),
 	}
 
-	tkn.SetComments(c)
+	tkn.Comments = c
 
-	if !reflect.DeepEqual(tkn.Comments(), c) {
+	if !reflect.DeepEqual(tkn.Comments, c) {
 		t.Errorf("comments are not equal\n")
 	}
 
@@ -29,7 +32,7 @@ func TestToken(t *testing.T) {
 		t.Errorf("token value is not equal\n")
 	}
 
-	if tkn.Position() != pos {
+	if tkn.Position != pos {
 		t.Errorf("token position is not equal\n")
 	}
 }