#27 reduce memory allocations for scanner.Token by using sync.Pool

errors/error.go

@@ -17,7 +17,7 @@ type Error struct {
 func NewError(msg string, t *scanner.Token) *Error {
 	return &Error{
 		Msg: msg,
-		Pos: t.Position(),
+		Pos: t.Position,
 	}
 }
 

errors/error_test.go

@@ -27,7 +27,10 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
 
 func TestConstructor(t *testing.T) {
 	pos := position.NewPosition(1, 2, 3, 4)
-	token := scanner.NewToken(`test`, pos)
+	token := &scanner.Token{
+		Value:    `test`,
+		Position: pos,
+	}
 
 	actual := errors.NewError("message", token)
 
@@ -41,7 +44,10 @@ func TestConstructor(t *testing.T) {
 
 func TestPrint(t *testing.T) {
 	pos := position.NewPosition(1, 2, 3, 4)
-	token := scanner.NewToken(`test`, pos)
+	token := &scanner.Token{
+		Value:    `test`,
+		Position: pos,
+	}
 
 	Error := errors.NewError("message", token)
 

parser/comments.go

@@ -15,7 +15,7 @@ func (c Comments) AddComments(node node.Node, comments []*comment.Comment) {
 }
 
 func (c Comments) AddFromToken(node node.Node, token *scanner.Token, tokenName comment.TokenName) {
-	comments := token.Comments()
+	comments := token.Comments
 
 	for _, cmt := range comments {
 		cmt.SetTokenName(tokenName)

parser/position_builder.go

@@ -101,25 +101,25 @@ func (b *PositionBuilder) NewNodePosition(n node.Node) *position.Position {
 
 // NewTokenPosition returns new Position
 func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position {
-	return t.Position()
+	return t.Position
 }
 
 // NewTokensPosition returns new Position
 func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position {
 	return position.NewPosition(
-		startToken.Position().StartLine,
+		startToken.Position.StartLine,
-		endToken.Position().EndLine,
+		endToken.Position.EndLine,
-		startToken.Position().StartPos,
+		startToken.Position.StartPos,
-		endToken.Position().EndPos,
+		endToken.Position.EndPos,
 	)
 }
 
 // NewTokenNodePosition returns new Position
 func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n node.Node) *position.Position {
 	return position.NewPosition(
-		t.Position().StartLine,
+		t.Position.StartLine,
 		b.getNodeEndPos(n).endLine,
-		t.Position().StartPos,
+		t.Position.StartPos,
 		b.getNodeEndPos(n).endPos,
 	)
 }
@@ -128,9 +128,9 @@ func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n node.Node) *p
 func (b *PositionBuilder) NewNodeTokenPosition(n node.Node, t *scanner.Token) *position.Position {
 	return position.NewPosition(
 		b.getNodeStartPos(n).startLine,
-		t.Position().EndLine,
+		t.Position.EndLine,
 		b.getNodeStartPos(n).startPos,
-		t.Position().EndPos,
+		t.Position.EndPos,
 	)
 }
 
@@ -148,18 +148,18 @@ func (b *PositionBuilder) NewNodesPosition(startNode node.Node, endNode node.Nod
 func (b *PositionBuilder) NewNodeListTokenPosition(list []node.Node, t *scanner.Token) *position.Position {
 	return position.NewPosition(
 		b.getListStartPos(list).startLine,
-		t.Position().EndLine,
+		t.Position.EndLine,
 		b.getListStartPos(list).startPos,
-		t.Position().EndPos,
+		t.Position.EndPos,
 	)
 }
 
 // NewTokenNodeListPosition returns new Position
 func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []node.Node) *position.Position {
 	return position.NewPosition(
-		t.Position().StartLine,
+		t.Position.StartLine,
 		b.getListEndPos(list).endLine,
-		t.Position().StartPos,
+		t.Position.StartPos,
 		b.getListEndPos(list).endPos,
 	)
 }
@@ -188,17 +188,17 @@ func (b *PositionBuilder) NewNodeListNodePosition(list []node.Node, n node.Node)
 func (b *PositionBuilder) NewOptionalListTokensPosition(list []node.Node, t *scanner.Token, endToken *scanner.Token) *position.Position {
 	if list == nil {
 		return position.NewPosition(
-			t.Position().StartLine,
+			t.Position.StartLine,
-			endToken.Position().EndLine,
+			endToken.Position.EndLine,
-			t.Position().StartPos,
+			t.Position.StartPos,
-			endToken.Position().EndPos,
+			endToken.Position.EndPos,
 		)
 	}
 
 	return position.NewPosition(
 		b.getListStartPos(list).startLine,
-		endToken.Position().EndLine,
+		endToken.Position.EndLine,
 		b.getListStartPos(list).startPos,
-		endToken.Position().EndPos,
+		endToken.Position.EndPos,
 	)
 }

parser/position_builder_test.go

@@ -14,7 +14,10 @@ func TestNewTokenPosition(t *testing.T) {
 	builder := parser.PositionBuilder{}
 
 	tpos := position.NewPosition(1, 1, 0, 3)
-	tkn := scanner.NewToken(`foo`, tpos)
+	tkn := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos,
+	}
 
 	pos := builder.NewTokenPosition(tkn)
 
@@ -27,9 +30,15 @@ func TestNewTokensPosition(t *testing.T) {
 	builder := parser.PositionBuilder{}
 
 	tpos1 := position.NewPosition(1, 1, 0, 3)
-	token1 := scanner.NewToken(`foo`, tpos1)
+	token1 := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos1,
+	}
 	tpos2 := position.NewPosition(2, 2, 4, 6)
-	token2 := scanner.NewToken(`foo`, tpos2)
+	token2 := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos2,
+	}
 
 	pos := builder.NewTokensPosition(token1, token2)
 
@@ -62,7 +71,10 @@ func TestNewNodePosition(t *testing.T) {
 
 func TestNewTokenNodePosition(t *testing.T) {
 	tpos := position.NewPosition(1, 1, 0, 3)
-	tkn := scanner.NewToken(`foo`, tpos)
+	tkn := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos,
+	}
 	n := node.NewIdentifier("test node")
 
 	p := &parser.Positions{}
@@ -87,7 +99,10 @@ func TestNewTokenNodePosition(t *testing.T) {
 func TestNewNodeTokenPosition(t *testing.T) {
 	n := node.NewIdentifier("test node")
 	tpos := position.NewPosition(2, 2, 10, 12)
-	tkn := scanner.NewToken(`foo`, tpos)
+	tkn := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos,
+	}
 
 	p := &parser.Positions{}
 	p.AddPosition(n, &position.Position{
@@ -168,7 +183,10 @@ func TestNewNodeListTokenPosition(t *testing.T) {
 	n1 := node.NewIdentifier("test node")
 	n2 := node.NewIdentifier("test node")
 	tpos := position.NewPosition(3, 3, 20, 22)
-	tkn := scanner.NewToken(`foo`, tpos)
+	tkn := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos,
+	}
 
 	builder := parser.PositionBuilder{
 		Positions: &parser.Positions{
@@ -196,7 +214,10 @@ func TestNewNodeListTokenPosition(t *testing.T) {
 
 func TestNewTokenNodeListPosition(t *testing.T) {
 	tpos := position.NewPosition(1, 1, 0, 2)
-	tkn := scanner.NewToken(`foo`, tpos)
+	tkn := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos,
+	}
 	n1 := node.NewIdentifier("test node")
 	n2 := node.NewIdentifier("test node")
 
@@ -298,9 +319,15 @@ func TestNewOptionalListTokensPosition(t *testing.T) {
 	builder := parser.PositionBuilder{}
 
 	tpos1 := position.NewPosition(1, 1, 0, 3)
-	token1 := scanner.NewToken(`foo`, tpos1)
+	token1 := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos1,
+	}
 	tpos2 := position.NewPosition(2, 2, 4, 6)
-	token2 := scanner.NewToken(`foo`, tpos2)
+	token2 := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos2,
+	}
 
 	pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
 
@@ -338,9 +365,15 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
 	}
 
 	tpos1 := position.NewPosition(4, 4, 27, 29)
-	token1 := scanner.NewToken(`foo`, tpos1)
+	token1 := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos1,
+	}
 	tpos2 := position.NewPosition(5, 5, 30, 32)
-	token2 := scanner.NewToken(`foo`, tpos2)
+	token2 := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos2,
+	}
 
 	pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2)
 
@@ -382,11 +415,14 @@ func TestNilNodeListPos(t *testing.T) {
 
 func TestNilNodeListTokenPos(t *testing.T) {
 	tpos := position.NewPosition(1, 1, 0, 3)
-	token1 := scanner.NewToken(`foo`, tpos)
+	token := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos,
+	}
 
 	builder := parser.PositionBuilder{}
 
-	pos := builder.NewNodeListTokenPosition(nil, token1)
+	pos := builder.NewNodeListTokenPosition(nil, token)
 
 	if pos.String() != `Pos{Line: -1-1 Pos: -1-3}` {
 		t.Errorf("token value is not equal\n")
@@ -416,11 +452,14 @@ func TestEmptyNodeListPos(t *testing.T) {
 
 func TestEmptyNodeListTokenPos(t *testing.T) {
 	tpos := position.NewPosition(1, 1, 0, 3)
-	token1 := scanner.NewToken(`foo`, tpos)
+	token := &scanner.Token{
+		Value:    `foo`,
+		Position: tpos,
+	}
 
 	builder := parser.PositionBuilder{}
 
-	pos := builder.NewNodeListTokenPosition([]node.Node{}, token1)
+	pos := builder.NewNodeListTokenPosition([]node.Node{}, token)
 
 	if pos.String() != `Pos{Line: -1-1 Pos: -1-3}` {
 		t.Errorf("token value is not equal\n")

php5/parser.go

@@ -117,3 +117,13 @@ func firstNode(nn []node.Node) node.Node {
 func isDollar(r rune) bool {
 	return r == '$'
 }
+
+func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
+	for i := 1; i < len(yyDollar); i++ {
+		if yyDollar[i].token != nil {
+			p.TokenPool.Put(yyDollar[i].token)
+		}
+		yyDollar[i].token = nil
+	}
+	yyVAL.token = nil
+}

php7/parser.go

@@ -106,3 +106,13 @@ func firstNode(nn []node.Node) node.Node {
 func isDollar(r rune) bool {
 	return r == '$'
 }
+
+func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
+	for i := 1; i < len(yyDollar); i++ {
+		if yyDollar[i].token != nil {
+			p.TokenPool.Put(yyDollar[i].token)
+		}
+		yyDollar[i].token = nil
+	}
+	yyVAL.token = nil
+}

scanner/lexer.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"go/token"
 	"io"
+	"sync"
 	"unicode"
 
 	"github.com/z7zmey/php-parser/position"
@@ -443,6 +444,7 @@ type Lexer struct {
 	Comments      []*comment.Comment
 	heredocLabel  string
 	tokenBytesBuf *bytes.Buffer
+	TokenPool     sync.Pool
 }
 
 // Rune2Class returns the rune integer id
@@ -470,7 +472,12 @@ func NewLexer(src io.Reader, fName string) *Lexer {
 	if err != nil {
 		panic(err)
 	}
-	return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}}
+
+	var TokenPool = sync.Pool{
+		New: func() interface{} { return &Token{} },
+	}
+
+	return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}, TokenPool}
 }
 
 func (l *Lexer) ungetChars(n int) []lex.Char {
@@ -523,7 +530,14 @@ func (l *Lexer) createToken(chars []lex.Char) *Token {
 		int(lastChar.Pos()),
 	)
 
-	return NewToken(l.tokenString(chars), pos).SetComments(l.Comments)
+	token := l.TokenPool.Get().(*Token)
+	token.Position = pos
+	token.Comments = l.Comments
+	token.Value = l.tokenString(chars)
+
+	return token
+
+	// return NewToken(l.tokenString(chars), pos).SetComments(l.Comments)
 }
 
 func (l *Lexer) addComment(chars []lex.Char) {

scanner/token.go

@@ -8,35 +8,10 @@ import (
 // Token value returned by lexer
 type Token struct {
 	Value    string
-	position *position.Position
+	Position *position.Position
-	comments []*comment.Comment
+	Comments []*comment.Comment
-}
-
-// NewToken Token constructor
-func NewToken(value string, pos *position.Position) *Token {
-	return &Token{
-		Value:    value,
-		position: pos,
-		comments: nil,
-	}
 }
 
 func (t *Token) String() string {
 	return string(t.Value)
 }
-
-// Position returns token position
-func (t *Token) Position() *position.Position {
-	return t.position
-}
-
-// Comments returns attached comments
-func (t *Token) Comments() []*comment.Comment {
-	return t.comments
-}
-
-// SetComments attach comments
-func (t *Token) SetComments(comments []*comment.Comment) *Token {
-	t.comments = comments
-	return t
-}

scanner/token_test.go

@@ -13,15 +13,18 @@ import (
 
 func TestToken(t *testing.T) {
 	pos := position.NewPosition(1, 1, 0, 3)
-	tkn := scanner.NewToken(`foo`, pos)
+	tkn := &scanner.Token{
+		Value:    `foo`,
+		Position: pos,
+	}
 
 	c := []*comment.Comment{
 		comment.NewComment("test comment", nil),
 	}
 
-	tkn.SetComments(c)
+	tkn.Comments = c
 
-	if !reflect.DeepEqual(tkn.Comments(), c) {
+	if !reflect.DeepEqual(tkn.Comments, c) {
 		t.Errorf("comments are not equal\n")
 	}
 
@@ -29,7 +32,7 @@ func TestToken(t *testing.T) {
 		t.Errorf("token value is not equal\n")
 	}
 
-	if tkn.Position() != pos {
+	if tkn.Position != pos {
 		t.Errorf("token position is not equal\n")
 	}
 }