maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#27 reduce memory allocations for scanner.Token by using sync.Pool

Browse Source
This commit is contained in:
z7zmey
2018-06-11 02:41:12 +03:00
parent 43451a070f
commit 2737e98559
14 changed files with 5631 additions and 1414 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
parser/comments.go
View File

@@ -15,7 +15,7 @@ func (c Comments) AddComments(node node.Node, comments []*comment.Comment) {
}
func (c Comments) AddFromToken(node node.Node, token *scanner.Token, tokenName comment.TokenName) {
comments := token.Comments()
comments := token.Comments
for _, cmt := range comments {
cmt.SetTokenName(tokenName)

38
parser/position_builder.go
View File

@@ -101,25 +101,25 @@ func (b *PositionBuilder) NewNodePosition(n node.Node) *position.Position {
// NewTokenPosition returns new Position
func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position {
return t.Position()
return t.Position
}
// NewTokensPosition returns new Position
func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position {
return position.NewPosition(
startToken.Position().StartLine,
endToken.Position().EndLine,
startToken.Position().StartPos,
endToken.Position().EndPos,
startToken.Position.StartLine,
endToken.Position.EndLine,
startToken.Position.StartPos,
endToken.Position.EndPos,
)
}
// NewTokenNodePosition returns new Position
func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n node.Node) *position.Position {
return position.NewPosition(
t.Position().StartLine,
t.Position.StartLine,
b.getNodeEndPos(n).endLine,
t.Position().StartPos,
t.Position.StartPos,
b.getNodeEndPos(n).endPos,
)
}
@@ -128,9 +128,9 @@ func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n node.Node) *p
func (b *PositionBuilder) NewNodeTokenPosition(n node.Node, t *scanner.Token) *position.Position {
return position.NewPosition(
b.getNodeStartPos(n).startLine,
t.Position().EndLine,
t.Position.EndLine,
b.getNodeStartPos(n).startPos,
t.Position().EndPos,
t.Position.EndPos,
)
}
@@ -148,18 +148,18 @@ func (b *PositionBuilder) NewNodesPosition(startNode node.Node, endNode node.Nod
func (b *PositionBuilder) NewNodeListTokenPosition(list []node.Node, t *scanner.Token) *position.Position {
return position.NewPosition(
b.getListStartPos(list).startLine,
t.Position().EndLine,
t.Position.EndLine,
b.getListStartPos(list).startPos,
t.Position().EndPos,
t.Position.EndPos,
)
}
// NewTokenNodeListPosition returns new Position
func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []node.Node) *position.Position {
return position.NewPosition(
t.Position().StartLine,
t.Position.StartLine,
b.getListEndPos(list).endLine,
t.Position().StartPos,
t.Position.StartPos,
b.getListEndPos(list).endPos,
)
}
@@ -188,17 +188,17 @@ func (b *PositionBuilder) NewNodeListNodePosition(list []node.Node, n node.Node)
func (b *PositionBuilder) NewOptionalListTokensPosition(list []node.Node, t *scanner.Token, endToken *scanner.Token) *position.Position {
if list == nil {
return position.NewPosition(
t.Position().StartLine,
endToken.Position().EndLine,
t.Position().StartPos,
endToken.Position().EndPos,
t.Position.StartLine,
endToken.Position.EndLine,
t.Position.StartPos,
endToken.Position.EndPos,
)
}
return position.NewPosition(
b.getListStartPos(list).startLine,
endToken.Position().EndLine,
endToken.Position.EndLine,
b.getListStartPos(list).startPos,
endToken.Position().EndPos,
endToken.Position.EndPos,
)
}

69
parser/position_builder_test.go
View File

@@ -14,7 +14,10 @@ func TestNewTokenPosition(t *testing.T) {
builder := parser.PositionBuilder{}
tpos := position.NewPosition(1, 1, 0, 3)
tkn := scanner.NewToken(`foo`, tpos)
tkn := &scanner.Token{
Value: `foo`,
Position: tpos,
}
pos := builder.NewTokenPosition(tkn)
@@ -27,9 +30,15 @@ func TestNewTokensPosition(t *testing.T) {
builder := parser.PositionBuilder{}
tpos1 := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken(`foo`, tpos1)
token1 := &scanner.Token{
Value: `foo`,
Position: tpos1,
}
tpos2 := position.NewPosition(2, 2, 4, 6)
token2 := scanner.NewToken(`foo`, tpos2)
token2 := &scanner.Token{
Value: `foo`,
Position: tpos2,
}
pos := builder.NewTokensPosition(token1, token2)
@@ -62,7 +71,10 @@ func TestNewNodePosition(t *testing.T) {
func TestNewTokenNodePosition(t *testing.T) {
tpos := position.NewPosition(1, 1, 0, 3)
tkn := scanner.NewToken(`foo`, tpos)
tkn := &scanner.Token{
Value: `foo`,
Position: tpos,
}
n := node.NewIdentifier("test node")
p := &parser.Positions{}
@@ -87,7 +99,10 @@ func TestNewTokenNodePosition(t *testing.T) {
func TestNewNodeTokenPosition(t *testing.T) {
n := node.NewIdentifier("test node")
tpos := position.NewPosition(2, 2, 10, 12)
tkn := scanner.NewToken(`foo`, tpos)
tkn := &scanner.Token{
Value: `foo`,
Position: tpos,
}
p := &parser.Positions{}
p.AddPosition(n, &position.Position{
@@ -168,7 +183,10 @@ func TestNewNodeListTokenPosition(t *testing.T) {
n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node")
tpos := position.NewPosition(3, 3, 20, 22)
tkn := scanner.NewToken(`foo`, tpos)
tkn := &scanner.Token{
Value: `foo`,
Position: tpos,
}
builder := parser.PositionBuilder{
Positions: &parser.Positions{
@@ -196,7 +214,10 @@ func TestNewNodeListTokenPosition(t *testing.T) {
func TestNewTokenNodeListPosition(t *testing.T) {
tpos := position.NewPosition(1, 1, 0, 2)
tkn := scanner.NewToken(`foo`, tpos)
tkn := &scanner.Token{
Value: `foo`,
Position: tpos,
}
n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node")
@@ -298,9 +319,15 @@ func TestNewOptionalListTokensPosition(t *testing.T) {
builder := parser.PositionBuilder{}
tpos1 := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken(`foo`, tpos1)
token1 := &scanner.Token{
Value: `foo`,
Position: tpos1,
}
tpos2 := position.NewPosition(2, 2, 4, 6)
token2 := scanner.NewToken(`foo`, tpos2)
token2 := &scanner.Token{
Value: `foo`,
Position: tpos2,
}
pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
@@ -338,9 +365,15 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
}
tpos1 := position.NewPosition(4, 4, 27, 29)
token1 := scanner.NewToken(`foo`, tpos1)
token1 := &scanner.Token{
Value: `foo`,
Position: tpos1,
}
tpos2 := position.NewPosition(5, 5, 30, 32)
token2 := scanner.NewToken(`foo`, tpos2)
token2 := &scanner.Token{
Value: `foo`,
Position: tpos2,
}
pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2)
@@ -382,11 +415,14 @@ func TestNilNodeListPos(t *testing.T) {
func TestNilNodeListTokenPos(t *testing.T) {
tpos := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken(`foo`, tpos)
token := &scanner.Token{
Value: `foo`,
Position: tpos,
}
builder := parser.PositionBuilder{}
pos := builder.NewNodeListTokenPosition(nil, token1)
pos := builder.NewNodeListTokenPosition(nil, token)
if pos.String() != `Pos{Line: -1-1 Pos: -1-3}` {
t.Errorf("token value is not equal\n")
@@ -416,11 +452,14 @@ func TestEmptyNodeListPos(t *testing.T) {
func TestEmptyNodeListTokenPos(t *testing.T) {
tpos := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken(`foo`, tpos)
token := &scanner.Token{
Value: `foo`,
Position: tpos,
}
builder := parser.PositionBuilder{}
pos := builder.NewNodeListTokenPosition([]node.Node{}, token1)
pos := builder.NewNodeListTokenPosition([]node.Node{}, token)
if pos.String() != `Pos{Line: -1-1 Pos: -1-3}` {
t.Errorf("token value is not equal\n")