maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#40 reduce memory allocations for position.Position by using sync.Pool

Browse Source
This commit is contained in:
z7zmey 2018-06-12 21:14:11 +03:00
parent 2737e98559
commit bfc346ebab
5 changed files with 186 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

165
parser/position_builder.go
View File

@ -1,6 +1,8 @@
package parser
import (
"sync"
"github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/scanner"
@ -9,6 +11,7 @@ import (
// PositionBuilder provide functions to constuct positions
type PositionBuilder struct {
Positions *Positions
PositionPool *sync.Pool
}
type startPos struct {
@ -81,124 +84,140 @@ func (b *PositionBuilder) getNodeEndPos(n node.Node) endPos {
// NewNodeListPosition returns new Position
func (b *PositionBuilder) NewNodeListPosition(list []node.Node) *position.Position {
return position.NewPosition(
b.getListStartPos(list).startLine,
b.getListEndPos(list).endLine,
b.getListStartPos(list).startPos,
b.getListEndPos(list).endPos,
)
pos := b.PositionPool.Get().(*position.Position)
pos.StartLine = b.getListStartPos(list).startLine
pos.EndLine = b.getListEndPos(list).endLine
pos.StartPos = b.getListStartPos(list).startPos
pos.EndPos = b.getListEndPos(list).endPos
return pos
}
// NewNodePosition returns new Position
func (b *PositionBuilder) NewNodePosition(n node.Node) *position.Position {
return position.NewPosition(
b.getNodeStartPos(n).startLine,
b.getNodeEndPos(n).endLine,
b.getNodeStartPos(n).startPos,
b.getNodeEndPos(n).endPos,
)
pos := b.PositionPool.Get().(*position.Position)
pos.StartLine = b.getNodeStartPos(n).startLine
pos.EndLine = b.getNodeEndPos(n).endLine
pos.StartPos = b.getNodeStartPos(n).startPos
pos.EndPos = b.getNodeEndPos(n).endPos
return pos
}
// NewTokenPosition returns new Position
func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position {
return t.Position
pos := b.PositionPool.Get().(*position.Position)
pos.StartLine = t.Position.StartLine
pos.EndLine = t.Position.EndLine
pos.StartPos = t.Position.StartPos
pos.EndPos = t.Position.EndPos
return pos
}
// NewTokensPosition returns new Position
func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position {
return position.NewPosition(
startToken.Position.StartLine,
endToken.Position.EndLine,
startToken.Position.StartPos,
endToken.Position.EndPos,
)
pos := b.PositionPool.Get().(*position.Position)
pos.StartLine = startToken.Position.StartLine
pos.EndLine = endToken.Position.EndLine
pos.StartPos = startToken.Position.StartPos
pos.EndPos = endToken.Position.EndPos
return pos
}
// NewTokenNodePosition returns new Position
func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n node.Node) *position.Position {
return position.NewPosition(
t.Position.StartLine,
b.getNodeEndPos(n).endLine,
t.Position.StartPos,
b.getNodeEndPos(n).endPos,
)
pos := b.PositionPool.Get().(*position.Position)
pos.StartLine = t.Position.StartLine
pos.EndLine = b.getNodeEndPos(n).endLine
pos.StartPos = t.Position.StartPos
pos.EndPos = b.getNodeEndPos(n).endPos
return pos
}
// NewNodeTokenPosition returns new Position
func (b *PositionBuilder) NewNodeTokenPosition(n node.Node, t *scanner.Token) *position.Position {
return position.NewPosition(
b.getNodeStartPos(n).startLine,
t.Position.EndLine,
b.getNodeStartPos(n).startPos,
t.Position.EndPos,
)
pos := b.PositionPool.Get().(*position.Position)
pos.StartLine = b.getNodeStartPos(n).startLine
pos.EndLine = t.Position.EndLine
pos.StartPos = b.getNodeStartPos(n).startPos
pos.EndPos = t.Position.EndPos
return pos
}
// NewNodesPosition returns new Position
func (b *PositionBuilder) NewNodesPosition(startNode node.Node, endNode node.Node) *position.Position {
return position.NewPosition(
b.getNodeStartPos(startNode).startLine,
b.getNodeEndPos(endNode).endLine,
b.getNodeStartPos(startNode).startPos,
b.getNodeEndPos(endNode).endPos,
)
pos := b.PositionPool.Get().(*position.Position)
pos.StartLine = b.getNodeStartPos(startNode).startLine
pos.EndLine = b.getNodeEndPos(endNode).endLine
pos.StartPos = b.getNodeStartPos(startNode).startPos
pos.EndPos = b.getNodeEndPos(endNode).endPos
return pos
}
// NewNodeListTokenPosition returns new Position
func (b *PositionBuilder) NewNodeListTokenPosition(list []node.Node, t *scanner.Token) *position.Position {
return position.NewPosition(
b.getListStartPos(list).startLine,
t.Position.EndLine,
b.getListStartPos(list).startPos,
t.Position.EndPos,
)
pos := b.PositionPool.Get().(*position.Position)
pos.StartLine = b.getListStartPos(list).startLine
pos.EndLine = t.Position.EndLine
pos.StartPos = b.getListStartPos(list).startPos
pos.EndPos = t.Position.EndPos
return pos
}
// NewTokenNodeListPosition returns new Position
func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []node.Node) *position.Position {
return position.NewPosition(
t.Position.StartLine,
b.getListEndPos(list).endLine,
t.Position.StartPos,
b.getListEndPos(list).endPos,
)
pos := b.PositionPool.Get().(*position.Position)
pos.StartLine = t.Position.StartLine
pos.EndLine = b.getListEndPos(list).endLine
pos.StartPos = t.Position.StartPos
pos.EndPos = b.getListEndPos(list).endPos
return pos
}
// NewNodeNodeListPosition returns new Position
func (b *PositionBuilder) NewNodeNodeListPosition(n node.Node, list []node.Node) *position.Position {
return position.NewPosition(
b.getNodeStartPos(n).startLine,
b.getListEndPos(list).endLine,
b.getNodeStartPos(n).startPos,
b.getListEndPos(list).endPos,
)
pos := b.PositionPool.Get().(*position.Position)
pos.StartLine = b.getNodeStartPos(n).startLine
pos.EndLine = b.getListEndPos(list).endLine
pos.StartPos = b.getNodeStartPos(n).startPos
pos.EndPos = b.getListEndPos(list).endPos
return pos
}
// NewNodeListNodePosition returns new Position
func (b *PositionBuilder) NewNodeListNodePosition(list []node.Node, n node.Node) *position.Position {
return position.NewPosition(
b.getListStartPos(list).startLine,
b.getNodeEndPos(n).endLine,
b.getListStartPos(list).startPos,
b.getNodeEndPos(n).endPos,
)
pos := b.PositionPool.Get().(*position.Position)
pos.StartLine = b.getListStartPos(list).startLine
pos.EndLine = b.getNodeEndPos(n).endLine
pos.StartPos = b.getListStartPos(list).startPos
pos.EndPos = b.getNodeEndPos(n).endPos
return pos
}
// NewOptionalListTokensPosition returns new Position
func (b *PositionBuilder) NewOptionalListTokensPosition(list []node.Node, t *scanner.Token, endToken *scanner.Token) *position.Position {
pos := b.PositionPool.Get().(*position.Position)
if list == nil {
return position.NewPosition(
t.Position.StartLine,
endToken.Position.EndLine,
t.Position.StartPos,
endToken.Position.EndPos,
)
pos.StartLine = t.Position.StartLine
pos.EndLine = endToken.Position.EndLine
pos.StartPos = t.Position.StartPos
pos.EndPos = endToken.Position.EndPos
return pos
}
return position.NewPosition(
b.getListStartPos(list).startLine,
endToken.Position.EndLine,
b.getListStartPos(list).startPos,
endToken.Position.EndPos,
)
pos.StartLine = b.getListStartPos(list).startLine
pos.EndLine = endToken.Position.EndLine
pos.StartPos = b.getListStartPos(list).startPos
pos.EndPos = endToken.Position.EndPos
return pos
}

73
parser/position_builder_test.go
View File

@ -1,6 +1,7 @@
package parser_test
import (
"sync"
"testing"
"github.com/z7zmey/php-parser/node"
@ -11,7 +12,11 @@ import (
)
func TestNewTokenPosition(t *testing.T) {
builder := parser.PositionBuilder{}
builder := parser.PositionBuilder{
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
tpos := position.NewPosition(1, 1, 0, 3)
tkn := &scanner.Token{
@ -27,7 +32,11 @@ func TestNewTokenPosition(t *testing.T) {
}
func TestNewTokensPosition(t *testing.T) {
builder := parser.PositionBuilder{}
builder := parser.PositionBuilder{
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
tpos1 := position.NewPosition(1, 1, 0, 3)
token1 := &scanner.Token{
@ -60,6 +69,9 @@ func TestNewNodePosition(t *testing.T) {
builder := parser.PositionBuilder{
Positions: p,
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodePosition(n)
@ -87,6 +99,9 @@ func TestNewTokenNodePosition(t *testing.T) {
builder := parser.PositionBuilder{
Positions: p,
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewTokenNodePosition(tkn, n)
@ -114,6 +129,9 @@ func TestNewNodeTokenPosition(t *testing.T) {
builder := parser.PositionBuilder{
Positions: p,
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodeTokenPosition(n, tkn)
@ -142,6 +160,9 @@ func TestNewNodeListPosition(t *testing.T) {
EndPos: 19,
},
},
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodeListPosition([]node.Node{n1, n2})
@ -170,6 +191,9 @@ func TestNewNodesPosition(t *testing.T) {
EndPos: 19,
},
},
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodesPosition(n1, n2)
@ -203,6 +227,9 @@ func TestNewNodeListTokenPosition(t *testing.T) {
EndPos: 19,
},
},
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodeListTokenPosition([]node.Node{n1, n2}, tkn)
@ -236,6 +263,9 @@ func TestNewTokenNodeListPosition(t *testing.T) {
EndPos: 20,
},
},
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewTokenNodeListPosition(tkn, []node.Node{n1, n2})
@ -271,6 +301,9 @@ func TestNewNodeNodeListPosition(t *testing.T) {
EndPos: 26,
},
},
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodeNodeListPosition(n1, []node.Node{n2, n3})
@ -306,6 +339,9 @@ func TestNewNodeListNodePosition(t *testing.T) {
EndPos: 26,
},
},
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodeListNodePosition([]node.Node{n1, n2}, n3)
@ -316,7 +352,11 @@ func TestNewNodeListNodePosition(t *testing.T) {
}
func TestNewOptionalListTokensPosition(t *testing.T) {
builder := parser.PositionBuilder{}
builder := parser.PositionBuilder{
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
tpos1 := position.NewPosition(1, 1, 0, 3)
token1 := &scanner.Token{
@ -362,6 +402,9 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
EndPos: 26,
},
},
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
tpos1 := position.NewPosition(4, 4, 27, 29)
@ -383,7 +426,11 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
}
func TestNilNodePos(t *testing.T) {
builder := parser.PositionBuilder{}
builder := parser.PositionBuilder{
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodesPosition(nil, nil)
@ -404,6 +451,9 @@ func TestNilNodeListPos(t *testing.T) {
EndPos: 8,
},
},
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodeNodeListPosition(n1, nil)
@ -420,7 +470,11 @@ func TestNilNodeListTokenPos(t *testing.T) {
Position: tpos,
}
builder := parser.PositionBuilder{}
builder := parser.PositionBuilder{
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodeListTokenPosition(nil, token)
@ -441,6 +495,9 @@ func TestEmptyNodeListPos(t *testing.T) {
EndPos: 8,
},
},
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodeNodeListPosition(n1, []node.Node{})
@ -457,7 +514,11 @@ func TestEmptyNodeListTokenPos(t *testing.T) {
Position: tpos,
}
builder := parser.PositionBuilder{}
builder := parser.PositionBuilder{
PositionPool: &sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
pos := builder.NewNodeListTokenPosition([]node.Node{}, token)

2
php5/parser.go
View File

@ -62,6 +62,7 @@ func (l *Parser) Parse() int {
l.positions = parser.Positions{}
l.positionBuilder = &parser.PositionBuilder{
Positions: &l.positions,
PositionPool: &l.PositionPool,
}
// parse
@ -122,6 +123,7 @@ func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
for i := 1; i < len(yyDollar); i++ {
if yyDollar[i].token != nil {
p.TokenPool.Put(yyDollar[i].token)
p.PositionPool.Put(yyDollar[i].token.Position)
}
yyDollar[i].token = nil
}

2
php7/parser.go
View File

@ -61,6 +61,7 @@ func (l *Parser) Parse() int {
l.positions = parser.Positions{}
l.positionBuilder = &parser.PositionBuilder{
Positions: &l.positions,
PositionPool: &l.PositionPool,
}
// parse
@ -111,6 +112,7 @@ func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
for i := 1; i < len(yyDollar); i++ {
if yyDollar[i].token != nil {
p.TokenPool.Put(yyDollar[i].token)
p.PositionPool.Put(yyDollar[i].token.Position)
}
yyDollar[i].token = nil
}

30
scanner/lexer.go
View File

@ -445,6 +445,7 @@ type Lexer struct {
heredocLabel string
tokenBytesBuf *bytes.Buffer
TokenPool sync.Pool
PositionPool sync.Pool
}
// Rune2Class returns the rune integer id
@ -473,11 +474,20 @@ func NewLexer(src io.Reader, fName string) *Lexer {
panic(err)
}
var TokenPool = sync.Pool{
return &Lexer{
Lexer: lx,
StateStack: []int{0},
PhpDocComment: "",
Comments: nil,
heredocLabel: "",
tokenBytesBuf: &bytes.Buffer{},
TokenPool: sync.Pool{
New: func() interface{} { return &Token{} },
},
PositionPool: sync.Pool{
New: func() interface{} { return &position.Position{} },
},
}
return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}, TokenPool}
}
func (l *Lexer) ungetChars(n int) []lex.Char {
@ -523,12 +533,12 @@ func (l *Lexer) createToken(chars []lex.Char) *Token {
firstChar := chars[0]
lastChar := chars[len(chars)-1]
pos := position.NewPosition(
l.File.Line(firstChar.Pos()),
l.File.Line(lastChar.Pos()),
int(firstChar.Pos()),
int(lastChar.Pos()),
)
pos := l.PositionPool.Get().(*position.Position)
pos.StartLine = l.File.Line(firstChar.Pos())
pos.EndLine = l.File.Line(lastChar.Pos())
pos.StartPos = int(firstChar.Pos())
pos.EndPos = int(lastChar.Pos())
token := l.TokenPool.Get().(*Token)
token.Position = pos
@ -536,8 +546,6 @@ func (l *Lexer) createToken(chars []lex.Char) *Token {
token.Value = l.tokenString(chars)
return token
// return NewToken(l.tokenString(chars), pos).SetComments(l.Comments)
}
func (l *Lexer) addComment(chars []lex.Char) {