#40 reduce memory allocations for position.Position by using sync.Pool

parser/position_builder.go

@@ -1,6 +1,8 @@
 package parser
 
 import (
+	"sync"
+
 	"github.com/z7zmey/php-parser/node"
 	"github.com/z7zmey/php-parser/position"
 	"github.com/z7zmey/php-parser/scanner"
@@ -9,6 +11,7 @@ import (
 // PositionBuilder provide functions to constuct positions
 type PositionBuilder struct {
 	Positions    *Positions
+	PositionPool *sync.Pool
 }
 
 type startPos struct {
@@ -81,124 +84,140 @@ func (b *PositionBuilder) getNodeEndPos(n node.Node) endPos {
 
 // NewNodeListPosition returns new Position
 func (b *PositionBuilder) NewNodeListPosition(list []node.Node) *position.Position {
-	return position.NewPosition(
+	pos := b.PositionPool.Get().(*position.Position)
-		b.getListStartPos(list).startLine,
+	pos.StartLine = b.getListStartPos(list).startLine
-		b.getListEndPos(list).endLine,
+	pos.EndLine = b.getListEndPos(list).endLine
-		b.getListStartPos(list).startPos,
+	pos.StartPos = b.getListStartPos(list).startPos
-		b.getListEndPos(list).endPos,
+	pos.EndPos = b.getListEndPos(list).endPos
-	)
+
+	return pos
 }
 
 // NewNodePosition returns new Position
 func (b *PositionBuilder) NewNodePosition(n node.Node) *position.Position {
-	return position.NewPosition(
+	pos := b.PositionPool.Get().(*position.Position)
-		b.getNodeStartPos(n).startLine,
+	pos.StartLine = b.getNodeStartPos(n).startLine
-		b.getNodeEndPos(n).endLine,
+	pos.EndLine = b.getNodeEndPos(n).endLine
-		b.getNodeStartPos(n).startPos,
+	pos.StartPos = b.getNodeStartPos(n).startPos
-		b.getNodeEndPos(n).endPos,
+	pos.EndPos = b.getNodeEndPos(n).endPos
-	)
+
+	return pos
 }
 
 // NewTokenPosition returns new Position
 func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position {
-	return t.Position
+	pos := b.PositionPool.Get().(*position.Position)
+	pos.StartLine = t.Position.StartLine
+	pos.EndLine = t.Position.EndLine
+	pos.StartPos = t.Position.StartPos
+	pos.EndPos = t.Position.EndPos
+
+	return pos
 }
 
 // NewTokensPosition returns new Position
 func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position {
-	return position.NewPosition(
+	pos := b.PositionPool.Get().(*position.Position)
-		startToken.Position.StartLine,
+	pos.StartLine = startToken.Position.StartLine
-		endToken.Position.EndLine,
+	pos.EndLine = endToken.Position.EndLine
-		startToken.Position.StartPos,
+	pos.StartPos = startToken.Position.StartPos
-		endToken.Position.EndPos,
+	pos.EndPos = endToken.Position.EndPos
-	)
+
+	return pos
 }
 
 // NewTokenNodePosition returns new Position
 func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n node.Node) *position.Position {
-	return position.NewPosition(
+	pos := b.PositionPool.Get().(*position.Position)
-		t.Position.StartLine,
+	pos.StartLine = t.Position.StartLine
-		b.getNodeEndPos(n).endLine,
+	pos.EndLine = b.getNodeEndPos(n).endLine
-		t.Position.StartPos,
+	pos.StartPos = t.Position.StartPos
-		b.getNodeEndPos(n).endPos,
+	pos.EndPos = b.getNodeEndPos(n).endPos
-	)
+
+	return pos
 }
 
 // NewNodeTokenPosition returns new Position
 func (b *PositionBuilder) NewNodeTokenPosition(n node.Node, t *scanner.Token) *position.Position {
-	return position.NewPosition(
+	pos := b.PositionPool.Get().(*position.Position)
-		b.getNodeStartPos(n).startLine,
+	pos.StartLine = b.getNodeStartPos(n).startLine
-		t.Position.EndLine,
+	pos.EndLine = t.Position.EndLine
-		b.getNodeStartPos(n).startPos,
+	pos.StartPos = b.getNodeStartPos(n).startPos
-		t.Position.EndPos,
+	pos.EndPos = t.Position.EndPos
-	)
+
+	return pos
 }
 
 // NewNodesPosition returns new Position
 func (b *PositionBuilder) NewNodesPosition(startNode node.Node, endNode node.Node) *position.Position {
-	return position.NewPosition(
+	pos := b.PositionPool.Get().(*position.Position)
-		b.getNodeStartPos(startNode).startLine,
+	pos.StartLine = b.getNodeStartPos(startNode).startLine
-		b.getNodeEndPos(endNode).endLine,
+	pos.EndLine = b.getNodeEndPos(endNode).endLine
-		b.getNodeStartPos(startNode).startPos,
+	pos.StartPos = b.getNodeStartPos(startNode).startPos
-		b.getNodeEndPos(endNode).endPos,
+	pos.EndPos = b.getNodeEndPos(endNode).endPos
-	)
+
+	return pos
 }
 
 // NewNodeListTokenPosition returns new Position
 func (b *PositionBuilder) NewNodeListTokenPosition(list []node.Node, t *scanner.Token) *position.Position {
-	return position.NewPosition(
+	pos := b.PositionPool.Get().(*position.Position)
-		b.getListStartPos(list).startLine,
+	pos.StartLine = b.getListStartPos(list).startLine
-		t.Position.EndLine,
+	pos.EndLine = t.Position.EndLine
-		b.getListStartPos(list).startPos,
+	pos.StartPos = b.getListStartPos(list).startPos
-		t.Position.EndPos,
+	pos.EndPos = t.Position.EndPos
-	)
+
+	return pos
 }
 
 // NewTokenNodeListPosition returns new Position
 func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []node.Node) *position.Position {
-	return position.NewPosition(
+	pos := b.PositionPool.Get().(*position.Position)
-		t.Position.StartLine,
+	pos.StartLine = t.Position.StartLine
-		b.getListEndPos(list).endLine,
+	pos.EndLine = b.getListEndPos(list).endLine
-		t.Position.StartPos,
+	pos.StartPos = t.Position.StartPos
-		b.getListEndPos(list).endPos,
+	pos.EndPos = b.getListEndPos(list).endPos
-	)
+
+	return pos
 }
 
 // NewNodeNodeListPosition returns new Position
 func (b *PositionBuilder) NewNodeNodeListPosition(n node.Node, list []node.Node) *position.Position {
-	return position.NewPosition(
+	pos := b.PositionPool.Get().(*position.Position)
-		b.getNodeStartPos(n).startLine,
+	pos.StartLine = b.getNodeStartPos(n).startLine
-		b.getListEndPos(list).endLine,
+	pos.EndLine = b.getListEndPos(list).endLine
-		b.getNodeStartPos(n).startPos,
+	pos.StartPos = b.getNodeStartPos(n).startPos
-		b.getListEndPos(list).endPos,
+	pos.EndPos = b.getListEndPos(list).endPos
-	)
+
+	return pos
 }
 
 // NewNodeListNodePosition returns new Position
 func (b *PositionBuilder) NewNodeListNodePosition(list []node.Node, n node.Node) *position.Position {
-	return position.NewPosition(
+	pos := b.PositionPool.Get().(*position.Position)
-		b.getListStartPos(list).startLine,
+	pos.StartLine = b.getListStartPos(list).startLine
-		b.getNodeEndPos(n).endLine,
+	pos.EndLine = b.getNodeEndPos(n).endLine
-		b.getListStartPos(list).startPos,
+	pos.StartPos = b.getListStartPos(list).startPos
-		b.getNodeEndPos(n).endPos,
+	pos.EndPos = b.getNodeEndPos(n).endPos
-	)
+
+	return pos
 }
 
 // NewOptionalListTokensPosition returns new Position
 func (b *PositionBuilder) NewOptionalListTokensPosition(list []node.Node, t *scanner.Token, endToken *scanner.Token) *position.Position {
+	pos := b.PositionPool.Get().(*position.Position)
+
 	if list == nil {
-		return position.NewPosition(
+		pos.StartLine = t.Position.StartLine
-			t.Position.StartLine,
+		pos.EndLine = endToken.Position.EndLine
-			endToken.Position.EndLine,
+		pos.StartPos = t.Position.StartPos
-			t.Position.StartPos,
+		pos.EndPos = endToken.Position.EndPos
-			endToken.Position.EndPos,
+		return pos
-		)
 	}
 
-	return position.NewPosition(
+	pos.StartLine = b.getListStartPos(list).startLine
-		b.getListStartPos(list).startLine,
+	pos.EndLine = endToken.Position.EndLine
-		endToken.Position.EndLine,
+	pos.StartPos = b.getListStartPos(list).startPos
-		b.getListStartPos(list).startPos,
+	pos.EndPos = endToken.Position.EndPos
-		endToken.Position.EndPos,
+	return pos
-	)
 }

parser/position_builder_test.go

@@ -1,6 +1,7 @@
 package parser_test
 
 import (
+	"sync"
 	"testing"
 
 	"github.com/z7zmey/php-parser/node"
@@ -11,7 +12,11 @@ import (
 )
 
 func TestNewTokenPosition(t *testing.T) {
-	builder := parser.PositionBuilder{}
+	builder := parser.PositionBuilder{
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
+	}
 
 	tpos := position.NewPosition(1, 1, 0, 3)
 	tkn := &scanner.Token{
@@ -27,7 +32,11 @@ func TestNewTokenPosition(t *testing.T) {
 }
 
 func TestNewTokensPosition(t *testing.T) {
-	builder := parser.PositionBuilder{}
+	builder := parser.PositionBuilder{
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
+	}
 
 	tpos1 := position.NewPosition(1, 1, 0, 3)
 	token1 := &scanner.Token{
@@ -60,6 +69,9 @@ func TestNewNodePosition(t *testing.T) {
 
 	builder := parser.PositionBuilder{
 		Positions: p,
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	pos := builder.NewNodePosition(n)
@@ -87,6 +99,9 @@ func TestNewTokenNodePosition(t *testing.T) {
 
 	builder := parser.PositionBuilder{
 		Positions: p,
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	pos := builder.NewTokenNodePosition(tkn, n)
@@ -114,6 +129,9 @@ func TestNewNodeTokenPosition(t *testing.T) {
 
 	builder := parser.PositionBuilder{
 		Positions: p,
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	pos := builder.NewNodeTokenPosition(n, tkn)
@@ -142,6 +160,9 @@ func TestNewNodeListPosition(t *testing.T) {
 				EndPos:    19,
 			},
 		},
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	pos := builder.NewNodeListPosition([]node.Node{n1, n2})
@@ -170,6 +191,9 @@ func TestNewNodesPosition(t *testing.T) {
 				EndPos:    19,
 			},
 		},
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	pos := builder.NewNodesPosition(n1, n2)
@@ -203,6 +227,9 @@ func TestNewNodeListTokenPosition(t *testing.T) {
 				EndPos:    19,
 			},
 		},
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	pos := builder.NewNodeListTokenPosition([]node.Node{n1, n2}, tkn)
@@ -236,6 +263,9 @@ func TestNewTokenNodeListPosition(t *testing.T) {
 				EndPos:    20,
 			},
 		},
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	pos := builder.NewTokenNodeListPosition(tkn, []node.Node{n1, n2})
@@ -271,6 +301,9 @@ func TestNewNodeNodeListPosition(t *testing.T) {
 				EndPos:    26,
 			},
 		},
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	pos := builder.NewNodeNodeListPosition(n1, []node.Node{n2, n3})
@@ -306,6 +339,9 @@ func TestNewNodeListNodePosition(t *testing.T) {
 				EndPos:    26,
 			},
 		},
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	pos := builder.NewNodeListNodePosition([]node.Node{n1, n2}, n3)
@@ -316,7 +352,11 @@ func TestNewNodeListNodePosition(t *testing.T) {
 }
 
 func TestNewOptionalListTokensPosition(t *testing.T) {
-	builder := parser.PositionBuilder{}
+	builder := parser.PositionBuilder{
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
+	}
 
 	tpos1 := position.NewPosition(1, 1, 0, 3)
 	token1 := &scanner.Token{
@@ -362,6 +402,9 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
 				EndPos:    26,
 			},
 		},
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	tpos1 := position.NewPosition(4, 4, 27, 29)
@@ -383,7 +426,11 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
 }
 
 func TestNilNodePos(t *testing.T) {
-	builder := parser.PositionBuilder{}
+	builder := parser.PositionBuilder{
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
+	}
 
 	pos := builder.NewNodesPosition(nil, nil)
 
@@ -404,6 +451,9 @@ func TestNilNodeListPos(t *testing.T) {
 				EndPos:    8,
 			},
 		},
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	pos := builder.NewNodeNodeListPosition(n1, nil)
@@ -420,7 +470,11 @@ func TestNilNodeListTokenPos(t *testing.T) {
 		Position: tpos,
 	}
 
-	builder := parser.PositionBuilder{}
+	builder := parser.PositionBuilder{
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
+	}
 
 	pos := builder.NewNodeListTokenPosition(nil, token)
 
@@ -441,6 +495,9 @@ func TestEmptyNodeListPos(t *testing.T) {
 				EndPos:    8,
 			},
 		},
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
 
 	pos := builder.NewNodeNodeListPosition(n1, []node.Node{})
@@ -457,7 +514,11 @@ func TestEmptyNodeListTokenPos(t *testing.T) {
 		Position: tpos,
 	}
 
-	builder := parser.PositionBuilder{}
+	builder := parser.PositionBuilder{
+		PositionPool: &sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
+	}
 
 	pos := builder.NewNodeListTokenPosition([]node.Node{}, token)
 

php5/parser.go

@@ -62,6 +62,7 @@ func (l *Parser) Parse() int {
 	l.positions = parser.Positions{}
 	l.positionBuilder = &parser.PositionBuilder{
 		Positions:    &l.positions,
+		PositionPool: &l.PositionPool,
 	}
 
 	// parse
@@ -122,6 +123,7 @@ func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
 	for i := 1; i < len(yyDollar); i++ {
 		if yyDollar[i].token != nil {
 			p.TokenPool.Put(yyDollar[i].token)
+			p.PositionPool.Put(yyDollar[i].token.Position)
 		}
 		yyDollar[i].token = nil
 	}

php7/parser.go

@@ -61,6 +61,7 @@ func (l *Parser) Parse() int {
 	l.positions = parser.Positions{}
 	l.positionBuilder = &parser.PositionBuilder{
 		Positions:    &l.positions,
+		PositionPool: &l.PositionPool,
 	}
 
 	// parse
@@ -111,6 +112,7 @@ func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
 	for i := 1; i < len(yyDollar); i++ {
 		if yyDollar[i].token != nil {
 			p.TokenPool.Put(yyDollar[i].token)
+			p.PositionPool.Put(yyDollar[i].token.Position)
 		}
 		yyDollar[i].token = nil
 	}

scanner/lexer.go

@@ -445,6 +445,7 @@ type Lexer struct {
 	heredocLabel  string
 	tokenBytesBuf *bytes.Buffer
 	TokenPool     sync.Pool
+	PositionPool  sync.Pool
 }
 
 // Rune2Class returns the rune integer id
@@ -473,11 +474,20 @@ func NewLexer(src io.Reader, fName string) *Lexer {
 		panic(err)
 	}
 
-	var TokenPool = sync.Pool{
+	return &Lexer{
+		Lexer:         lx,
+		StateStack:    []int{0},
+		PhpDocComment: "",
+		Comments:      nil,
+		heredocLabel:  "",
+		tokenBytesBuf: &bytes.Buffer{},
+		TokenPool: sync.Pool{
 			New: func() interface{} { return &Token{} },
+		},
+		PositionPool: sync.Pool{
+			New: func() interface{} { return &position.Position{} },
+		},
 	}
-
-	return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}, TokenPool}
 }
 
 func (l *Lexer) ungetChars(n int) []lex.Char {
@@ -523,12 +533,12 @@ func (l *Lexer) createToken(chars []lex.Char) *Token {
 	firstChar := chars[0]
 	lastChar := chars[len(chars)-1]
 
-	pos := position.NewPosition(
+	pos := l.PositionPool.Get().(*position.Position)
-		l.File.Line(firstChar.Pos()),
+
-		l.File.Line(lastChar.Pos()),
+	pos.StartLine = l.File.Line(firstChar.Pos())
-		int(firstChar.Pos()),
+	pos.EndLine = l.File.Line(lastChar.Pos())
-		int(lastChar.Pos()),
+	pos.StartPos = int(firstChar.Pos())
-	)
+	pos.EndPos = int(lastChar.Pos())
 
 	token := l.TokenPool.Get().(*Token)
 	token.Position = pos
@@ -536,8 +546,6 @@ func (l *Lexer) createToken(chars []lex.Char) *Token {
 	token.Value = l.tokenString(chars)
 
 	return token
-
-	// return NewToken(l.tokenString(chars), pos).SetComments(l.Comments)
 }
 
 func (l *Lexer) addComment(chars []lex.Char) {