maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#27 reduce memory allocations for scanner.Token by using sync.Pool

Browse Source
This commit is contained in:
z7zmey 2018-06-11 02:41:12 +03:00
parent 43451a070f
commit 2737e98559
14 changed files with 5631 additions and 1414 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
errors/error.go
View File

@ -17,7 +17,7 @@ type Error struct {
func NewError(msg string, t *scanner.Token) *Error {
return &Error{
Msg: msg,
Pos: t.Position(),
Pos: t.Position,
}
}

10
errors/error_test.go
View File

@ -27,7 +27,10 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
func TestConstructor(t *testing.T) {
pos := position.NewPosition(1, 2, 3, 4)
token := scanner.NewToken(`test`, pos)
token := &scanner.Token{
Value: `test`,
Position: pos,
}
actual := errors.NewError("message", token)
@ -41,7 +44,10 @@ func TestConstructor(t *testing.T) {
func TestPrint(t *testing.T) {
pos := position.NewPosition(1, 2, 3, 4)
token := scanner.NewToken(`test`, pos)
token := &scanner.Token{
Value: `test`,
Position: pos,
}
Error := errors.NewError("message", token)

2
parser/comments.go
View File

@ -15,7 +15,7 @@ func (c Comments) AddComments(node node.Node, comments []*comment.Comment) {
}
func (c Comments) AddFromToken(node node.Node, token *scanner.Token, tokenName comment.TokenName) {
comments := token.Comments()
comments := token.Comments
for _, cmt := range comments {
cmt.SetTokenName(tokenName)

38
parser/position_builder.go
View File

@ -101,25 +101,25 @@ func (b *PositionBuilder) NewNodePosition(n node.Node) *position.Position {
// NewTokenPosition returns new Position
func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position {
return t.Position()
return t.Position
}
// NewTokensPosition returns new Position
func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position {
return position.NewPosition(
startToken.Position().StartLine,
endToken.Position().EndLine,
startToken.Position().StartPos,
endToken.Position().EndPos,
startToken.Position.StartLine,
endToken.Position.EndLine,
startToken.Position.StartPos,
endToken.Position.EndPos,
)
}
// NewTokenNodePosition returns new Position
func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n node.Node) *position.Position {
return position.NewPosition(
t.Position().StartLine,
t.Position.StartLine,
b.getNodeEndPos(n).endLine,
t.Position().StartPos,
t.Position.StartPos,
b.getNodeEndPos(n).endPos,
)
}
@ -128,9 +128,9 @@ func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n node.Node) *p
func (b *PositionBuilder) NewNodeTokenPosition(n node.Node, t *scanner.Token) *position.Position {
return position.NewPosition(
b.getNodeStartPos(n).startLine,
t.Position().EndLine,
t.Position.EndLine,
b.getNodeStartPos(n).startPos,
t.Position().EndPos,
t.Position.EndPos,
)
}
@ -148,18 +148,18 @@ func (b *PositionBuilder) NewNodesPosition(startNode node.Node, endNode node.Nod
func (b *PositionBuilder) NewNodeListTokenPosition(list []node.Node, t *scanner.Token) *position.Position {
return position.NewPosition(
b.getListStartPos(list).startLine,
t.Position().EndLine,
t.Position.EndLine,
b.getListStartPos(list).startPos,
t.Position().EndPos,
t.Position.EndPos,
)
}
// NewTokenNodeListPosition returns new Position
func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []node.Node) *position.Position {
return position.NewPosition(
t.Position().StartLine,
t.Position.StartLine,
b.getListEndPos(list).endLine,
t.Position().StartPos,
t.Position.StartPos,
b.getListEndPos(list).endPos,
)
}
@ -188,17 +188,17 @@ func (b *PositionBuilder) NewNodeListNodePosition(list []node.Node, n node.Node)
func (b *PositionBuilder) NewOptionalListTokensPosition(list []node.Node, t *scanner.Token, endToken *scanner.Token) *position.Position {
if list == nil {
return position.NewPosition(
t.Position().StartLine,
endToken.Position().EndLine,
t.Position().StartPos,
endToken.Position().EndPos,
t.Position.StartLine,
endToken.Position.EndLine,
t.Position.StartPos,
endToken.Position.EndPos,
)
}
return position.NewPosition(
b.getListStartPos(list).startLine,
endToken.Position().EndLine,
endToken.Position.EndLine,
b.getListStartPos(list).startPos,
endToken.Position().EndPos,
endToken.Position.EndPos,
)
}

69
parser/position_builder_test.go
View File

@ -14,7 +14,10 @@ func TestNewTokenPosition(t *testing.T) {
builder := parser.PositionBuilder{}
tpos := position.NewPosition(1, 1, 0, 3)
tkn := scanner.NewToken(`foo`, tpos)
tkn := &scanner.Token{
Value: `foo`,
Position: tpos,
}
pos := builder.NewTokenPosition(tkn)
@ -27,9 +30,15 @@ func TestNewTokensPosition(t *testing.T) {
builder := parser.PositionBuilder{}
tpos1 := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken(`foo`, tpos1)
token1 := &scanner.Token{
Value: `foo`,
Position: tpos1,
}
tpos2 := position.NewPosition(2, 2, 4, 6)
token2 := scanner.NewToken(`foo`, tpos2)
token2 := &scanner.Token{
Value: `foo`,
Position: tpos2,
}
pos := builder.NewTokensPosition(token1, token2)
@ -62,7 +71,10 @@ func TestNewNodePosition(t *testing.T) {
func TestNewTokenNodePosition(t *testing.T) {
tpos := position.NewPosition(1, 1, 0, 3)
tkn := scanner.NewToken(`foo`, tpos)
tkn := &scanner.Token{
Value: `foo`,
Position: tpos,
}
n := node.NewIdentifier("test node")
p := &parser.Positions{}
@ -87,7 +99,10 @@ func TestNewTokenNodePosition(t *testing.T) {
func TestNewNodeTokenPosition(t *testing.T) {
n := node.NewIdentifier("test node")
tpos := position.NewPosition(2, 2, 10, 12)
tkn := scanner.NewToken(`foo`, tpos)
tkn := &scanner.Token{
Value: `foo`,
Position: tpos,
}
p := &parser.Positions{}
p.AddPosition(n, &position.Position{
@ -168,7 +183,10 @@ func TestNewNodeListTokenPosition(t *testing.T) {
n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node")
tpos := position.NewPosition(3, 3, 20, 22)
tkn := scanner.NewToken(`foo`, tpos)
tkn := &scanner.Token{
Value: `foo`,
Position: tpos,
}
builder := parser.PositionBuilder{
Positions: &parser.Positions{
@ -196,7 +214,10 @@ func TestNewNodeListTokenPosition(t *testing.T) {
func TestNewTokenNodeListPosition(t *testing.T) {
tpos := position.NewPosition(1, 1, 0, 2)
tkn := scanner.NewToken(`foo`, tpos)
tkn := &scanner.Token{
Value: `foo`,
Position: tpos,
}
n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node")
@ -298,9 +319,15 @@ func TestNewOptionalListTokensPosition(t *testing.T) {
builder := parser.PositionBuilder{}
tpos1 := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken(`foo`, tpos1)
token1 := &scanner.Token{
Value: `foo`,
Position: tpos1,
}
tpos2 := position.NewPosition(2, 2, 4, 6)
token2 := scanner.NewToken(`foo`, tpos2)
token2 := &scanner.Token{
Value: `foo`,
Position: tpos2,
}
pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
@ -338,9 +365,15 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
}
tpos1 := position.NewPosition(4, 4, 27, 29)
token1 := scanner.NewToken(`foo`, tpos1)
token1 := &scanner.Token{
Value: `foo`,
Position: tpos1,
}
tpos2 := position.NewPosition(5, 5, 30, 32)
token2 := scanner.NewToken(`foo`, tpos2)
token2 := &scanner.Token{
Value: `foo`,
Position: tpos2,
}
pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2)
@ -382,11 +415,14 @@ func TestNilNodeListPos(t *testing.T) {
func TestNilNodeListTokenPos(t *testing.T) {
tpos := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken(`foo`, tpos)
token := &scanner.Token{
Value: `foo`,
Position: tpos,
}
builder := parser.PositionBuilder{}
pos := builder.NewNodeListTokenPosition(nil, token1)
pos := builder.NewNodeListTokenPosition(nil, token)
if pos.String() != `Pos{Line: -1-1 Pos: -1-3}` {
t.Errorf("token value is not equal\n")
@ -416,11 +452,14 @@ func TestEmptyNodeListPos(t *testing.T) {
func TestEmptyNodeListTokenPos(t *testing.T) {
tpos := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken(`foo`, tpos)
token := &scanner.Token{
Value: `foo`,
Position: tpos,
}
builder := parser.PositionBuilder{}
pos := builder.NewNodeListTokenPosition([]node.Node{}, token1)
pos := builder.NewNodeListTokenPosition([]node.Node{}, token)
if pos.String() != `Pos{Line: -1-1 Pos: -1-3}` {
t.Errorf("token value is not equal\n")

10
php5/parser.go
View File

@ -117,3 +117,13 @@ func firstNode(nn []node.Node) node.Node {
func isDollar(r rune) bool {
return r == '$'
}
func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
for i := 1; i < len(yyDollar); i++ {
if yyDollar[i].token != nil {
p.TokenPool.Put(yyDollar[i].token)
}
yyDollar[i].token = nil
}
yyVAL.token = nil
}

2205
php5/php5.go
View File

File diff suppressed because it is too large Load Diff

1574
php5/php5.y
View File

File diff suppressed because it is too large Load Diff

10
php7/parser.go
View File

@ -106,3 +106,13 @@ func firstNode(nn []node.Node) node.Node {
func isDollar(r rune) bool {
return r == '$'
}
func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
for i := 1; i < len(yyDollar); i++ {
if yyDollar[i].token != nil {
p.TokenPool.Put(yyDollar[i].token)
}
yyDollar[i].token = nil
}
yyVAL.token = nil
}

1783
php7/php7.go
View File

File diff suppressed because it is too large Load Diff

1284
php7/php7.y
View File

File diff suppressed because it is too large Load Diff

18
scanner/lexer.go
View File

@ -6,6 +6,7 @@ import (
"bytes"
"go/token"
"io"
"sync"
"unicode"
"github.com/z7zmey/php-parser/position"
@ -443,6 +444,7 @@ type Lexer struct {
Comments []*comment.Comment
heredocLabel string
tokenBytesBuf *bytes.Buffer
TokenPool sync.Pool
}
// Rune2Class returns the rune integer id
@ -470,7 +472,12 @@ func NewLexer(src io.Reader, fName string) *Lexer {
if err != nil {
panic(err)
}
return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}}
var TokenPool = sync.Pool{
New: func() interface{} { return &Token{} },
}
return &Lexer{lx, []int{0}, "", nil, "", &bytes.Buffer{}, TokenPool}
}
func (l *Lexer) ungetChars(n int) []lex.Char {
@ -523,7 +530,14 @@ func (l *Lexer) createToken(chars []lex.Char) *Token {
int(lastChar.Pos()),
)
return NewToken(l.tokenString(chars), pos).SetComments(l.Comments)
token := l.TokenPool.Get().(*Token)
token.Position = pos
token.Comments = l.Comments
token.Value = l.tokenString(chars)
return token
// return NewToken(l.tokenString(chars), pos).SetComments(l.Comments)
}
func (l *Lexer) addComment(chars []lex.Char) {

29
scanner/token.go
View File

@ -8,35 +8,10 @@ import (
// Token value returned by lexer
type Token struct {
Value string
position *position.Position
comments []*comment.Comment
}
// NewToken Token constructor
func NewToken(value string, pos *position.Position) *Token {
return &Token{
Value: value,
position: pos,
comments: nil,
}
Position *position.Position
Comments []*comment.Comment
}
func (t *Token) String() string {
return string(t.Value)
}
// Position returns token position
func (t *Token) Position() *position.Position {
return t.position
}
// Comments returns attached comments
func (t *Token) Comments() []*comment.Comment {
return t.comments
}
// SetComments attach comments
func (t *Token) SetComments(comments []*comment.Comment) *Token {
t.comments = comments
return t
}

11
scanner/token_test.go
View File

@ -13,15 +13,18 @@ import (
func TestToken(t *testing.T) {
pos := position.NewPosition(1, 1, 0, 3)
tkn := scanner.NewToken(`foo`, pos)
tkn := &scanner.Token{
Value: `foo`,
Position: pos,
}
c := []*comment.Comment{
comment.NewComment("test comment", nil),
}
tkn.SetComments(c)
tkn.Comments = c
if !reflect.DeepEqual(tkn.Comments(), c) {
if !reflect.DeepEqual(tkn.Comments, c) {
t.Errorf("comments are not equal\n")
}
@ -29,7 +32,7 @@ func TestToken(t *testing.T) {
t.Errorf("token value is not equal\n")
}
if tkn.Position() != pos {
if tkn.Position != pos {
t.Errorf("token position is not equal\n")
}
}