maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[refactoring] remove scanner token

Browse Source
This commit is contained in:
Vadym Slizov
2020-08-17 20:31:04 +03:00
parent 394092269a
commit 97747c5ac0
36 changed files with 6591 additions and 9813 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/ast/node.go
View File

@@ -16,6 +16,10 @@ func (n *Node) GetNode() *Node {
return n
}
func (n *Node) GetPosition() *position.Position {
return n.Position
}
// Root node
type Root struct {
Node

2
pkg/ast/visitor/dump.go
View File

@@ -121,7 +121,7 @@ func (v *Dump) printNode(n *ast.Node) {
key := token.Position(k)
v.printIndent(v.indent + 2)
v.print("token." + key.String() + ": []token.Token{\n")
v.print("token." + key.String() + ": []*token.Token{\n")
for _, tkn := range n.Tokens[key] {
v.printIndent(v.indent + 3)

4
pkg/ast/visitor/dump_test.go
View File

@@ -13,7 +13,7 @@ func ExampleDump() {
stxTree := &ast.Root{
Node: ast.Node{
Tokens: token.Collection{
token.Start: []token.Token{
token.Start: []*token.Token{
{
ID: token.T_WHITESPACE,
Value: []byte(" "),
@@ -44,7 +44,7 @@ func ExampleDump() {
//&ast.Root{
// Node: ast.Node{
// Tokens: token.Collection{
// token.Start: []token.Token{
// token.Start: []*token.Token{
// {
// ID: token.T_WHITESPACE,
// Value: []byte(" "),

14
pkg/ast/visitor/filter_tokens.go Normal file
View File

@@ -0,0 +1,14 @@
package visitor
import (
"github.com/z7zmey/php-parser/pkg/ast"
)
type FilterTokens struct {
Null
}
func (v *FilterTokens) EnterNode(n ast.Vertex) bool {
n.GetNode().Tokens = nil
return true
}

2
pkg/parser/parser.go
View File

@@ -29,7 +29,7 @@ func Parse(src []byte, ver string, cfg Config) (ast.Vertex, error) {
return nil, err
}
lexer := scanner.NewLexer(src, ver, cfg.WithTokens, cfg.ErrorHandlerFunc)
lexer := scanner.NewLexer(src, ver, cfg.ErrorHandlerFunc)
if r == -1 {
parser = php5.NewParser(lexer, cfg.ErrorHandlerFunc)

29
pkg/position/pool.go Normal file
View File

@@ -0,0 +1,29 @@
package position
const DefaultBlockSize = 1024
type Pool struct {
block []Position
off int
}
func NewPool(blockSize int) *Pool {
return &Pool{
block: make([]Position, blockSize),
}
}
func (p *Pool) Get() *Position {
if len(p.block) == 0 {
return nil
}
if len(p.block) == p.off {
p.block = make([]Position, len(p.block))
p.off = 0
}
p.off++
return &p.block[p.off-1]
}

2
pkg/printer/printer_parsed_php5_test.go
View File

@@ -12,7 +12,7 @@ import (
)
func parsePhp5(src string) ast.Vertex {
lexer := scanner.NewLexer([]byte(src), "5.6", true, nil)
lexer := scanner.NewLexer([]byte(src), "5.6", nil)
php5parser := php5.NewParser(lexer, nil)
php5parser.Parse()

4
pkg/printer/printer_parsed_php7_test.go
View File

@@ -29,7 +29,7 @@ abstract class Bar extends Baz
// parse
lexer := scanner.NewLexer([]byte(src), "7.4", true, nil)
lexer := scanner.NewLexer([]byte(src), "7.4", nil)
php7parser := php7.NewParser(lexer, nil)
php7parser.Parse()
@@ -61,7 +61,7 @@ abstract class Bar extends Baz
}
func parse(src string) ast.Vertex {
lexer := scanner.NewLexer([]byte(src), "7.4", true, nil)
lexer := scanner.NewLexer([]byte(src), "7.4", nil)
php7parser := php7.NewParser(lexer, nil)
php7parser.Parse()

4
pkg/printer/printer_test.go
View File

@@ -75,7 +75,7 @@ func TestPrinterPrintFileInlineHtml(t *testing.T) {
Expr: &ast.ExprVariable{
Node: ast.Node{
Tokens: token.Collection{
token.Start: []token.Token{
token.Start: []*token.Token{
{
ID: token.ID('$'),
Value: []byte("$"),
@@ -93,7 +93,7 @@ func TestPrinterPrintFileInlineHtml(t *testing.T) {
Expr: &ast.ExprVariable{
Node: ast.Node{
Tokens: token.Collection{
token.Start: []token.Token{
token.Start: []*token.Token{
{
ID: token.ID('$'),
Value: []byte("$"),

29
pkg/token/pool.go Normal file
View File

@@ -0,0 +1,29 @@
package token
const DefaultBlockSize = 1024
type Pool struct {
block []Token
off int
}
func NewPool(blockSize int) *Pool {
return &Pool{
block: make([]Token, blockSize),
}
}
func (p *Pool) Get() *Token {
if len(p.block) == 0 {
return nil
}
if len(p.block) == p.off {
p.block = make([]Token, len(p.block))
p.off = 0
}
p.off++
return &p.block[p.off-1]
}

173
pkg/token/pool_bench_test.go Normal file
View File

@@ -0,0 +1,173 @@
package token
import (
"testing"
)
const amount = 100000
func BenchmarkPlain(b *testing.B) {
for n := 0; n < b.N; n++ {
buf := make([]*Token, 0, amount)
for i := 0; i < amount; i++ {
buf = append(buf, &Token{})
}
}
}
func BenchmarkSlice128(b *testing.B) {
for n := 0; n < b.N; n++ {
buf := make([]*Token, 0, amount)
slc := make([]Token, 0, 128)
for i := 0; i < amount; i++ {
slc = append(slc, Token{})
buf = append(buf, &slc[len(slc)-1])
}
}
}
func BenchmarkSlice512(b *testing.B) {
for n := 0; n < b.N; n++ {
buf := make([]*Token, 0, amount)
slc := make([]Token, 0, 512)
for i := 0; i < amount; i++ {
slc = append(slc, Token{})
buf = append(buf, &slc[len(slc)-1])
}
}
}
func BenchmarkSlice1024(b *testing.B) {
for n := 0; n < b.N; n++ {
buf := make([]*Token, 0, amount)
slc := make([]Token, 0, 1024)
for i := 0; i < amount; i++ {
slc = append(slc, Token{})
buf = append(buf, &slc[len(slc)-1])
}
}
}
func BenchmarkSlice2048(b *testing.B) {
for n := 0; n < b.N; n++ {
buf := make([]*Token, 0, amount)
slc := make([]Token, 0, 2048)
for i := 0; i < amount; i++ {
slc = append(slc, Token{})
buf = append(buf, &slc[len(slc)-1])
}
}
}
func BenchmarkBlockAppend128(b *testing.B) {
for n := 0; n < b.N; n++ {
buf := make([]*Token, 0, amount)
slc := make([]Token, 0, 128)
for i := 0; i < amount; i++ {
if len(slc) == 128 {
slc = make([]Token, 0, 128)
}
slc = append(slc, Token{})
buf = append(buf, &slc[len(slc)-1])
}
}
}
func BenchmarkBlockAppend512(b *testing.B) {
for n := 0; n < b.N; n++ {
buf := make([]*Token, 0, amount)
slc := make([]Token, 0, 512)
for i := 0; i < amount; i++ {
if len(slc) == 512 {
slc = make([]Token, 0, 512)
}
slc = append(slc, Token{})
buf = append(buf, &slc[len(slc)-1])
}
}
}
func BenchmarkBlockAppend1024(b *testing.B) {
for n := 0; n < b.N; n++ {
buf := make([]*Token, 0, amount)
slc := make([]Token, 0, 1024)
for i := 0; i < amount; i++ {
if len(slc) == 1024 {
slc = make([]Token, 0, 1024)
}
slc = append(slc, Token{})
buf = append(buf, &slc[len(slc)-1])
}
}
}
func BenchmarkBlockAppend2048(b *testing.B) {
for n := 0; n < b.N; n++ {
buf := make([]*Token, 0, amount)
slc := make([]Token, 0, 2048)
for i := 0; i < amount; i++ {
if len(slc) == 2048 {
slc = make([]Token, 0, 2048)
}
slc = append(slc, Token{})
buf = append(buf, &slc[len(slc)-1])
}
}
}
func BenchmarkPool128(b *testing.B) {
for n := 0; n < b.N; n++ {
pool := NewPool(128)
buf := make([]*Token, 0, amount)
for i := 0; i < amount; i++ {
buf = append(buf, pool.Get())
}
}
}
func BenchmarkPool512(b *testing.B) {
for n := 0; n < b.N; n++ {
pool := NewPool(512)
buf := make([]*Token, 0, amount)
for i := 0; i < amount; i++ {
buf = append(buf, pool.Get())
}
}
}
func BenchmarkPool1024(b *testing.B) {
for n := 0; n < b.N; n++ {
pool := NewPool(1024)
buf := make([]*Token, 0, amount)
for i := 0; i < amount; i++ {
buf = append(buf, pool.Get())
}
}
}
func BenchmarkPool2048(b *testing.B) {
for n := 0; n < b.N; n++ {
pool := NewPool(2048)
buf := make([]*Token, 0, amount)
for i := 0; i < amount; i++ {
buf = append(buf, pool.Get())
}
}
}

2
pkg/token/position.go
View File

@@ -62,7 +62,7 @@ const (
CloseParenthesisToken
)
type Collection map[Position][]Token
type Collection map[Position][]*Token
func (c Collection) IsEmpty() bool {
for _, v := range c {

13
pkg/token/token.go
View File

@@ -1,5 +1,7 @@
package token
import "github.com/z7zmey/php-parser/pkg/position"
//go:generate stringer -type=ID -output ./token_string.go
type ID int
@@ -145,6 +147,13 @@ const (
)
type Token struct {
ID ID
Value []byte
ID ID
Value []byte
Position *position.Position
SkippedTokens []*Token
Skipped []byte
}
func (t *Token) GetPosition() *position.Position {
return t.Position
}