[refactoring] remove scanner token

pkg/token/pool.go

@@ -0,0 +1,29 @@
+package token
+
+const DefaultBlockSize = 1024
+
+type Pool struct {
+	block []Token
+	off   int
+}
+
+func NewPool(blockSize int) *Pool {
+	return &Pool{
+		block: make([]Token, blockSize),
+	}
+}
+
+func (p *Pool) Get() *Token {
+	if len(p.block) == 0 {
+		return nil
+	}
+
+	if len(p.block) == p.off {
+		p.block = make([]Token, len(p.block))
+		p.off = 0
+	}
+
+	p.off++
+
+	return &p.block[p.off-1]
+}

pkg/token/pool_bench_test.go

@@ -0,0 +1,173 @@
+package token
+
+import (
+	"testing"
+)
+
+const amount = 100000
+
+func BenchmarkPlain(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		buf := make([]*Token, 0, amount)
+
+		for i := 0; i < amount; i++ {
+			buf = append(buf, &Token{})
+		}
+	}
+}
+
+func BenchmarkSlice128(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		buf := make([]*Token, 0, amount)
+		slc := make([]Token, 0, 128)
+
+		for i := 0; i < amount; i++ {
+			slc = append(slc, Token{})
+			buf = append(buf, &slc[len(slc)-1])
+		}
+	}
+}
+
+func BenchmarkSlice512(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		buf := make([]*Token, 0, amount)
+		slc := make([]Token, 0, 512)
+
+		for i := 0; i < amount; i++ {
+			slc = append(slc, Token{})
+			buf = append(buf, &slc[len(slc)-1])
+		}
+	}
+}
+
+func BenchmarkSlice1024(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		buf := make([]*Token, 0, amount)
+		slc := make([]Token, 0, 1024)
+
+		for i := 0; i < amount; i++ {
+			slc = append(slc, Token{})
+			buf = append(buf, &slc[len(slc)-1])
+		}
+	}
+}
+
+func BenchmarkSlice2048(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		buf := make([]*Token, 0, amount)
+		slc := make([]Token, 0, 2048)
+
+		for i := 0; i < amount; i++ {
+			slc = append(slc, Token{})
+			buf = append(buf, &slc[len(slc)-1])
+		}
+	}
+}
+
+func BenchmarkBlockAppend128(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		buf := make([]*Token, 0, amount)
+		slc := make([]Token, 0, 128)
+
+		for i := 0; i < amount; i++ {
+			if len(slc) == 128 {
+				slc = make([]Token, 0, 128)
+			}
+
+			slc = append(slc, Token{})
+			buf = append(buf, &slc[len(slc)-1])
+		}
+	}
+}
+
+func BenchmarkBlockAppend512(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		buf := make([]*Token, 0, amount)
+		slc := make([]Token, 0, 512)
+
+		for i := 0; i < amount; i++ {
+			if len(slc) == 512 {
+				slc = make([]Token, 0, 512)
+			}
+
+			slc = append(slc, Token{})
+			buf = append(buf, &slc[len(slc)-1])
+		}
+	}
+}
+
+func BenchmarkBlockAppend1024(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		buf := make([]*Token, 0, amount)
+		slc := make([]Token, 0, 1024)
+
+		for i := 0; i < amount; i++ {
+			if len(slc) == 1024 {
+				slc = make([]Token, 0, 1024)
+			}
+
+			slc = append(slc, Token{})
+			buf = append(buf, &slc[len(slc)-1])
+		}
+	}
+}
+
+func BenchmarkBlockAppend2048(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		buf := make([]*Token, 0, amount)
+		slc := make([]Token, 0, 2048)
+
+		for i := 0; i < amount; i++ {
+			if len(slc) == 2048 {
+				slc = make([]Token, 0, 2048)
+			}
+
+			slc = append(slc, Token{})
+			buf = append(buf, &slc[len(slc)-1])
+		}
+	}
+}
+
+func BenchmarkPool128(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		pool := NewPool(128)
+		buf := make([]*Token, 0, amount)
+
+		for i := 0; i < amount; i++ {
+			buf = append(buf, pool.Get())
+		}
+	}
+}
+
+func BenchmarkPool512(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		pool := NewPool(512)
+		buf := make([]*Token, 0, amount)
+
+		for i := 0; i < amount; i++ {
+			buf = append(buf, pool.Get())
+		}
+	}
+}
+
+func BenchmarkPool1024(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		pool := NewPool(1024)
+		buf := make([]*Token, 0, amount)
+
+		for i := 0; i < amount; i++ {
+			buf = append(buf, pool.Get())
+		}
+	}
+}
+
+func BenchmarkPool2048(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		pool := NewPool(2048)
+		buf := make([]*Token, 0, amount)
+
+		for i := 0; i < amount; i++ {
+			buf = append(buf, pool.Get())
+		}
+	}
+}

pkg/token/position.go

@@ -62,7 +62,7 @@ const (
 	CloseParenthesisToken
 )
 
-type Collection map[Position][]Token
+type Collection map[Position][]*Token
 
 func (c Collection) IsEmpty() bool {
 	for _, v := range c {

pkg/token/token.go

@@ -1,5 +1,7 @@
 package token
 
+import "github.com/z7zmey/php-parser/pkg/position"
+
 //go:generate stringer -type=ID -output ./token_string.go
 type ID int
 
@@ -145,6 +147,13 @@ const (
 )
 
 type Token struct {
-	ID    ID
-	Value []byte
+	ID            ID
+	Value         []byte
+	Position      *position.Position
+	SkippedTokens []*Token
+	Skipped       []byte
+}
+
+func (t *Token) GetPosition() *position.Position {
+	return t.Position
 }