maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[refactoring] use position.Position to store token position

Browse Source
This commit is contained in:
Vadym Slizov 2020-05-17 23:50:23 +03:00
parent f4c15f4671
commit e98607f5d0
7 changed files with 126 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
internal/php5/parser.go
View File

@ -8,7 +8,6 @@ import (
"github.com/z7zmey/php-parser/internal/scanner" "github.com/z7zmey/php-parser/internal/scanner"
"github.com/z7zmey/php-parser/pkg/ast" "github.com/z7zmey/php-parser/pkg/ast"
"github.com/z7zmey/php-parser/pkg/errors" "github.com/z7zmey/php-parser/pkg/errors"
"github.com/z7zmey/php-parser/pkg/position"
"github.com/z7zmey/php-parser/pkg/token" "github.com/z7zmey/php-parser/pkg/token"
) )
@ -45,14 +44,9 @@ func (l *Parser) Lex(lval *yySymType) int {
} }
func (l *Parser) Error(msg string) { func (l *Parser) Error(msg string) {
pos := &position.Position{ var pos = l.currentToken.Position
StartLine: l.currentToken.StartLine,
EndLine: l.currentToken.EndLine,
StartPos: l.currentToken.StartPos,
EndPos: l.currentToken.EndPos,
}
l.Lexer.AddError(errors.NewError(msg, pos)) l.Lexer.AddError(errors.NewError(msg, &pos))
} }
func (l *Parser) WithTokens() { func (l *Parser) WithTokens() {

BIN
internal/php5/php5.go
View File

Binary file not shown.

10
internal/php7/parser.go
View File

@ -7,7 +7,6 @@ import (
"github.com/z7zmey/php-parser/internal/scanner" "github.com/z7zmey/php-parser/internal/scanner"
"github.com/z7zmey/php-parser/pkg/ast" "github.com/z7zmey/php-parser/pkg/ast"
"github.com/z7zmey/php-parser/pkg/errors" "github.com/z7zmey/php-parser/pkg/errors"
"github.com/z7zmey/php-parser/pkg/position"
"github.com/z7zmey/php-parser/pkg/token" "github.com/z7zmey/php-parser/pkg/token"
) )
@ -43,14 +42,9 @@ func (l *Parser) Lex(lval *yySymType) int {
} }
func (l *Parser) Error(msg string) { func (l *Parser) Error(msg string) {
pos := &position.Position{ var pos = l.currentToken.Position
StartLine: l.currentToken.StartLine,
EndLine: l.currentToken.EndLine,
StartPos: l.currentToken.StartPos,
EndPos: l.currentToken.EndPos,
}
l.Lexer.AddError(errors.NewError(msg, pos)) l.Lexer.AddError(errors.NewError(msg, &pos))
} }
func (l *Parser) WithTokens() { func (l *Parser) WithTokens() {

44
internal/positionbuilder/position_builder.go
View File

@ -100,29 +100,29 @@ func (b *PositionBuilder) NewNodePosition(n ast.Vertex) *position.Position {
// NewTokenPosition returns new Position // NewTokenPosition returns new Position
func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position { func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position {
return &position.Position{ return &position.Position{
StartLine: t.StartLine, StartLine: t.Position.StartLine,
EndLine: t.EndLine, EndLine: t.Position.EndLine,
StartPos: t.StartPos, StartPos: t.Position.StartPos,
EndPos: t.EndPos, EndPos: t.Position.EndPos,
} }
} }
// NewTokensPosition returns new Position // NewTokensPosition returns new Position
func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position { func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position {
return &position.Position{ return &position.Position{
StartLine: startToken.StartLine, StartLine: startToken.Position.StartLine,
EndLine: endToken.EndLine, EndLine: endToken.Position.EndLine,
StartPos: startToken.StartPos, StartPos: startToken.Position.StartPos,
EndPos: endToken.EndPos, EndPos: endToken.Position.EndPos,
} }
} }
// NewTokenNodePosition returns new Position // NewTokenNodePosition returns new Position
func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n ast.Vertex) *position.Position { func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n ast.Vertex) *position.Position {
return &position.Position{ return &position.Position{
StartLine: t.StartLine, StartLine: t.Position.StartLine,
EndLine: b.getNodeEndPos(n).endLine, EndLine: b.getNodeEndPos(n).endLine,
StartPos: t.StartPos, StartPos: t.Position.StartPos,
EndPos: b.getNodeEndPos(n).endPos, EndPos: b.getNodeEndPos(n).endPos,
} }
} }
@ -131,9 +131,9 @@ func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n ast.Vertex) *
func (b *PositionBuilder) NewNodeTokenPosition(n ast.Vertex, t *scanner.Token) *position.Position { func (b *PositionBuilder) NewNodeTokenPosition(n ast.Vertex, t *scanner.Token) *position.Position {
return &position.Position{ return &position.Position{
StartLine: b.getNodeStartPos(n).startLine, StartLine: b.getNodeStartPos(n).startLine,
EndLine: t.EndLine, EndLine: t.Position.EndLine,
StartPos: b.getNodeStartPos(n).startPos, StartPos: b.getNodeStartPos(n).startPos,
EndPos: t.EndPos, EndPos: t.Position.EndPos,
} }
} }
@ -151,18 +151,18 @@ func (b *PositionBuilder) NewNodesPosition(startNode ast.Vertex, endNode ast.Ver
func (b *PositionBuilder) NewNodeListTokenPosition(list []ast.Vertex, t *scanner.Token) *position.Position { func (b *PositionBuilder) NewNodeListTokenPosition(list []ast.Vertex, t *scanner.Token) *position.Position {
return &position.Position{ return &position.Position{
StartLine: b.getListStartPos(list).startLine, StartLine: b.getListStartPos(list).startLine,
EndLine: t.EndLine, EndLine: t.Position.EndLine,
StartPos: b.getListStartPos(list).startPos, StartPos: b.getListStartPos(list).startPos,
EndPos: t.EndPos, EndPos: t.Position.EndPos,
} }
} }
// NewTokenNodeListPosition returns new Position // NewTokenNodeListPosition returns new Position
func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []ast.Vertex) *position.Position { func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []ast.Vertex) *position.Position {
return &position.Position{ return &position.Position{
StartLine: t.StartLine, StartLine: t.Position.StartLine,
EndLine: b.getListEndPos(list).endLine, EndLine: b.getListEndPos(list).endLine,
StartPos: t.StartPos, StartPos: t.Position.StartPos,
EndPos: b.getListEndPos(list).endPos, EndPos: b.getListEndPos(list).endPos,
} }
} }
@ -191,17 +191,17 @@ func (b *PositionBuilder) NewNodeListNodePosition(list []ast.Vertex, n ast.Verte
func (b *PositionBuilder) NewOptionalListTokensPosition(list []ast.Vertex, t *scanner.Token, endToken *scanner.Token) *position.Position { func (b *PositionBuilder) NewOptionalListTokensPosition(list []ast.Vertex, t *scanner.Token, endToken *scanner.Token) *position.Position {
if list == nil { if list == nil {
return &position.Position{ return &position.Position{
StartLine: t.StartLine, StartLine: t.Position.StartLine,
EndLine: endToken.EndLine, EndLine: endToken.Position.EndLine,
StartPos: t.StartPos, StartPos: t.Position.StartPos,
EndPos: endToken.EndPos, EndPos: endToken.Position.EndPos,
} }
} }
return &position.Position{ return &position.Position{
StartLine: b.getListStartPos(list).startLine, StartLine: b.getListStartPos(list).startLine,
EndLine: endToken.EndLine, EndLine: endToken.Position.EndLine,
StartPos: b.getListStartPos(list).startPos, StartPos: b.getListStartPos(list).startPos,
EndPos: endToken.EndPos, EndPos: endToken.Position.EndPos,
} }
} }

156
internal/positionbuilder/position_builder_test.go
View File

@ -14,11 +14,13 @@ func TestNewTokenPosition(t *testing.T) {
builder := positionbuilder.PositionBuilder{} builder := positionbuilder.PositionBuilder{}
tkn := &scanner.Token{ tkn := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 1, Position: position.Position{
EndLine: 1, StartLine: 1,
StartPos: 0, EndLine: 1,
EndPos: 3, StartPos: 0,
EndPos: 3,
},
} }
pos := builder.NewTokenPosition(tkn) pos := builder.NewTokenPosition(tkn)
@ -32,18 +34,22 @@ func TestNewTokensPosition(t *testing.T) {
builder := positionbuilder.PositionBuilder{} builder := positionbuilder.PositionBuilder{}
token1 := &scanner.Token{ token1 := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 1, Position: position.Position{
EndLine: 1, StartLine: 1,
StartPos: 0, EndLine: 1,
EndPos: 3, StartPos: 0,
EndPos: 3,
},
} }
token2 := &scanner.Token{ token2 := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 2, Position: position.Position{
EndLine: 2, StartLine: 2,
StartPos: 4, EndLine: 2,
EndPos: 6, StartPos: 4,
EndPos: 6,
},
} }
pos := builder.NewTokensPosition(token1, token2) pos := builder.NewTokensPosition(token1, token2)
@ -72,11 +78,13 @@ func TestNewNodePosition(t *testing.T) {
func TestNewTokenNodePosition(t *testing.T) { func TestNewTokenNodePosition(t *testing.T) {
tkn := &scanner.Token{ tkn := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 1, Position: position.Position{
EndLine: 1, StartLine: 1,
StartPos: 0, EndLine: 1,
EndPos: 3, StartPos: 0,
EndPos: 3,
},
} }
n := &ast.Identifier{ n := &ast.Identifier{
Node: ast.Node{ Node: ast.Node{
@ -109,11 +117,13 @@ func TestNewNodeTokenPosition(t *testing.T) {
} }
tkn := &scanner.Token{ tkn := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 2, Position: position.Position{
EndLine: 2, StartLine: 2,
StartPos: 10, EndLine: 2,
EndPos: 12, StartPos: 10,
EndPos: 12,
},
} }
builder := positionbuilder.PositionBuilder{} builder := positionbuilder.PositionBuilder{}
@ -207,11 +217,13 @@ func TestNewNodeListTokenPosition(t *testing.T) {
} }
tkn := &scanner.Token{ tkn := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 3, Position: position.Position{
EndLine: 3, StartLine: 3,
StartPos: 20, EndLine: 3,
EndPos: 22, StartPos: 20,
EndPos: 22,
},
} }
builder := positionbuilder.PositionBuilder{} builder := positionbuilder.PositionBuilder{}
@ -223,11 +235,13 @@ func TestNewNodeListTokenPosition(t *testing.T) {
func TestNewTokenNodeListPosition(t *testing.T) { func TestNewTokenNodeListPosition(t *testing.T) {
tkn := &scanner.Token{ tkn := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 1, Position: position.Position{
EndLine: 1, StartLine: 1,
StartPos: 0, EndLine: 1,
EndPos: 2, StartPos: 0,
EndPos: 2,
},
} }
n1 := &ast.Identifier{ n1 := &ast.Identifier{
@ -343,18 +357,22 @@ func TestNewOptionalListTokensPosition(t *testing.T) {
builder := positionbuilder.PositionBuilder{} builder := positionbuilder.PositionBuilder{}
token1 := &scanner.Token{ token1 := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 1, Position: position.Position{
EndLine: 1, StartLine: 1,
StartPos: 0, EndLine: 1,
EndPos: 3, StartPos: 0,
EndPos: 3,
},
} }
token2 := &scanner.Token{ token2 := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 2, Position: position.Position{
EndLine: 2, StartLine: 2,
StartPos: 4, EndLine: 2,
EndPos: 6, StartPos: 4,
EndPos: 6,
},
} }
pos := builder.NewOptionalListTokensPosition(nil, token1, token2) pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
@ -387,18 +405,22 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
builder := positionbuilder.PositionBuilder{} builder := positionbuilder.PositionBuilder{}
token1 := &scanner.Token{ token1 := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 4, Position: position.Position{
EndLine: 4, StartLine: 4,
StartPos: 27, EndLine: 4,
EndPos: 29, StartPos: 27,
EndPos: 29,
},
} }
token2 := &scanner.Token{ token2 := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 5, Position: position.Position{
EndLine: 5, StartLine: 5,
StartPos: 30, EndLine: 5,
EndPos: 32, StartPos: 30,
EndPos: 32,
},
} }
pos := builder.NewOptionalListTokensPosition([]ast.Vertex{n2, n3}, token1, token2) pos := builder.NewOptionalListTokensPosition([]ast.Vertex{n2, n3}, token1, token2)
@ -435,11 +457,13 @@ func TestNilNodeListPos(t *testing.T) {
func TestNilNodeListTokenPos(t *testing.T) { func TestNilNodeListTokenPos(t *testing.T) {
token := &scanner.Token{ token := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 1, Position: position.Position{
EndLine: 1, StartLine: 1,
StartPos: 0, EndLine: 1,
EndPos: 3, StartPos: 0,
EndPos: 3,
},
} }
builder := positionbuilder.PositionBuilder{} builder := positionbuilder.PositionBuilder{}
@ -470,11 +494,13 @@ func TestEmptyNodeListPos(t *testing.T) {
func TestEmptyNodeListTokenPos(t *testing.T) { func TestEmptyNodeListTokenPos(t *testing.T) {
token := &scanner.Token{ token := &scanner.Token{
Value: []byte(`foo`), Value: []byte(`foo`),
StartLine: 1, Position: position.Position{
EndLine: 1, StartLine: 1,
StartPos: 0, EndLine: 1,
EndPos: 3, StartPos: 0,
EndPos: 3,
},
} }
builder := positionbuilder.PositionBuilder{} builder := positionbuilder.PositionBuilder{}

8
internal/scanner/lexer.go
View File

@ -78,10 +78,10 @@ func (l *Lexer) SetErrors(e []*errors.Error) {
} }
func (lex *Lexer) setTokenPosition(token *Token) { func (lex *Lexer) setTokenPosition(token *Token) {
token.StartLine = lex.NewLines.GetLine(lex.ts) token.Position.StartLine = lex.NewLines.GetLine(lex.ts)
token.EndLine = lex.NewLines.GetLine(lex.te - 1) token.Position.EndLine = lex.NewLines.GetLine(lex.te - 1)
token.StartPos = lex.ts token.Position.StartPos = lex.ts
token.EndPos = lex.te token.Position.EndPos = lex.te
} }
func (lex *Lexer) addToken(id TokenID, ps, pe int) { func (lex *Lexer) addToken(id TokenID, ps, pe int) {

12
internal/scanner/token.go
View File

@ -1,16 +1,14 @@
package scanner package scanner
import ( import (
"github.com/z7zmey/php-parser/pkg/position"
"github.com/z7zmey/php-parser/pkg/token" "github.com/z7zmey/php-parser/pkg/token"
) )
// Token value returned by lexer // Token value returned by lexer
type Token struct { type Token struct {
ID TokenID ID TokenID
Value []byte Value []byte
Tokens []token.Token Tokens []token.Token
StartLine int Position position.Position
EndLine int
StartPos int
EndPos int
} }