maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[refactoring] use position.Position to store token position

Browse Source
This commit is contained in:
Vadym Slizov 2020-05-17 23:50:23 +03:00
parent f4c15f4671
commit e98607f5d0
7 changed files with 126 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
internal/php5/parser.go
View File

@ -8,7 +8,6 @@ import (
"github.com/z7zmey/php-parser/internal/scanner"
"github.com/z7zmey/php-parser/pkg/ast"
"github.com/z7zmey/php-parser/pkg/errors"
"github.com/z7zmey/php-parser/pkg/position"
"github.com/z7zmey/php-parser/pkg/token"
)
@ -45,14 +44,9 @@ func (l *Parser) Lex(lval *yySymType) int {
}
func (l *Parser) Error(msg string) {
pos := &position.Position{
StartLine: l.currentToken.StartLine,
EndLine: l.currentToken.EndLine,
StartPos: l.currentToken.StartPos,
EndPos: l.currentToken.EndPos,
}
var pos = l.currentToken.Position
l.Lexer.AddError(errors.NewError(msg, pos))
l.Lexer.AddError(errors.NewError(msg, &pos))
}
func (l *Parser) WithTokens() {

BIN
internal/php5/php5.go
View File

Binary file not shown.

10
internal/php7/parser.go
View File

@ -7,7 +7,6 @@ import (
"github.com/z7zmey/php-parser/internal/scanner"
"github.com/z7zmey/php-parser/pkg/ast"
"github.com/z7zmey/php-parser/pkg/errors"
"github.com/z7zmey/php-parser/pkg/position"
"github.com/z7zmey/php-parser/pkg/token"
)
@ -43,14 +42,9 @@ func (l *Parser) Lex(lval *yySymType) int {
}
func (l *Parser) Error(msg string) {
pos := &position.Position{
StartLine: l.currentToken.StartLine,
EndLine: l.currentToken.EndLine,
StartPos: l.currentToken.StartPos,
EndPos: l.currentToken.EndPos,
}
var pos = l.currentToken.Position
l.Lexer.AddError(errors.NewError(msg, pos))
l.Lexer.AddError(errors.NewError(msg, &pos))
}
func (l *Parser) WithTokens() {

44
internal/positionbuilder/position_builder.go
View File

@ -100,29 +100,29 @@ func (b *PositionBuilder) NewNodePosition(n ast.Vertex) *position.Position {
// NewTokenPosition returns new Position
func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position {
return &position.Position{
StartLine: t.StartLine,
EndLine: t.EndLine,
StartPos: t.StartPos,
EndPos: t.EndPos,
StartLine: t.Position.StartLine,
EndLine: t.Position.EndLine,
StartPos: t.Position.StartPos,
EndPos: t.Position.EndPos,
}
}
// NewTokensPosition returns new Position
func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position {
return &position.Position{
StartLine: startToken.StartLine,
EndLine: endToken.EndLine,
StartPos: startToken.StartPos,
EndPos: endToken.EndPos,
StartLine: startToken.Position.StartLine,
EndLine: endToken.Position.EndLine,
StartPos: startToken.Position.StartPos,
EndPos: endToken.Position.EndPos,
}
}
// NewTokenNodePosition returns new Position
func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n ast.Vertex) *position.Position {
return &position.Position{
StartLine: t.StartLine,
StartLine: t.Position.StartLine,
EndLine: b.getNodeEndPos(n).endLine,
StartPos: t.StartPos,
StartPos: t.Position.StartPos,
EndPos: b.getNodeEndPos(n).endPos,
}
}
@ -131,9 +131,9 @@ func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n ast.Vertex) *
func (b *PositionBuilder) NewNodeTokenPosition(n ast.Vertex, t *scanner.Token) *position.Position {
return &position.Position{
StartLine: b.getNodeStartPos(n).startLine,
EndLine: t.EndLine,
EndLine: t.Position.EndLine,
StartPos: b.getNodeStartPos(n).startPos,
EndPos: t.EndPos,
EndPos: t.Position.EndPos,
}
}
@ -151,18 +151,18 @@ func (b *PositionBuilder) NewNodesPosition(startNode ast.Vertex, endNode ast.Ver
func (b *PositionBuilder) NewNodeListTokenPosition(list []ast.Vertex, t *scanner.Token) *position.Position {
return &position.Position{
StartLine: b.getListStartPos(list).startLine,
EndLine: t.EndLine,
EndLine: t.Position.EndLine,
StartPos: b.getListStartPos(list).startPos,
EndPos: t.EndPos,
EndPos: t.Position.EndPos,
}
}
// NewTokenNodeListPosition returns new Position
func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []ast.Vertex) *position.Position {
return &position.Position{
StartLine: t.StartLine,
StartLine: t.Position.StartLine,
EndLine: b.getListEndPos(list).endLine,
StartPos: t.StartPos,
StartPos: t.Position.StartPos,
EndPos: b.getListEndPos(list).endPos,
}
}
@ -191,17 +191,17 @@ func (b *PositionBuilder) NewNodeListNodePosition(list []ast.Vertex, n ast.Verte
func (b *PositionBuilder) NewOptionalListTokensPosition(list []ast.Vertex, t *scanner.Token, endToken *scanner.Token) *position.Position {
if list == nil {
return &position.Position{
StartLine: t.StartLine,
EndLine: endToken.EndLine,
StartPos: t.StartPos,
EndPos: endToken.EndPos,
StartLine: t.Position.StartLine,
EndLine: endToken.Position.EndLine,
StartPos: t.Position.StartPos,
EndPos: endToken.Position.EndPos,
}
}
return &position.Position{
StartLine: b.getListStartPos(list).startLine,
EndLine: endToken.EndLine,
EndLine: endToken.Position.EndLine,
StartPos: b.getListStartPos(list).startPos,
EndPos: endToken.EndPos,
EndPos: endToken.Position.EndPos,
}
}

26
internal/positionbuilder/position_builder_test.go
View File

@ -15,10 +15,12 @@ func TestNewTokenPosition(t *testing.T) {
tkn := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
pos := builder.NewTokenPosition(tkn)
@ -33,17 +35,21 @@ func TestNewTokensPosition(t *testing.T) {
token1 := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
token2 := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 2,
EndLine: 2,
StartPos: 4,
EndPos: 6,
},
}
pos := builder.NewTokensPosition(token1, token2)
@ -73,10 +79,12 @@ func TestNewNodePosition(t *testing.T) {
func TestNewTokenNodePosition(t *testing.T) {
tkn := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
n := &ast.Identifier{
Node: ast.Node{
@ -110,10 +118,12 @@ func TestNewNodeTokenPosition(t *testing.T) {
tkn := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 2,
EndLine: 2,
StartPos: 10,
EndPos: 12,
},
}
builder := positionbuilder.PositionBuilder{}
@ -208,10 +218,12 @@ func TestNewNodeListTokenPosition(t *testing.T) {
tkn := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 3,
EndLine: 3,
StartPos: 20,
EndPos: 22,
},
}
builder := positionbuilder.PositionBuilder{}
@ -224,10 +236,12 @@ func TestNewNodeListTokenPosition(t *testing.T) {
func TestNewTokenNodeListPosition(t *testing.T) {
tkn := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 2,
},
}
n1 := &ast.Identifier{
@ -344,17 +358,21 @@ func TestNewOptionalListTokensPosition(t *testing.T) {
token1 := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
token2 := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 2,
EndLine: 2,
StartPos: 4,
EndPos: 6,
},
}
pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
@ -388,17 +406,21 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
token1 := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 4,
EndLine: 4,
StartPos: 27,
EndPos: 29,
},
}
token2 := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 5,
EndLine: 5,
StartPos: 30,
EndPos: 32,
},
}
pos := builder.NewOptionalListTokensPosition([]ast.Vertex{n2, n3}, token1, token2)
@ -436,10 +458,12 @@ func TestNilNodeListPos(t *testing.T) {
func TestNilNodeListTokenPos(t *testing.T) {
token := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
builder := positionbuilder.PositionBuilder{}
@ -471,10 +495,12 @@ func TestEmptyNodeListPos(t *testing.T) {
func TestEmptyNodeListTokenPos(t *testing.T) {
token := &scanner.Token{
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
builder := positionbuilder.PositionBuilder{}

8
internal/scanner/lexer.go
View File

@ -78,10 +78,10 @@ func (l *Lexer) SetErrors(e []*errors.Error) {
}
func (lex *Lexer) setTokenPosition(token *Token) {
token.StartLine = lex.NewLines.GetLine(lex.ts)
token.EndLine = lex.NewLines.GetLine(lex.te - 1)
token.StartPos = lex.ts
token.EndPos = lex.te
token.Position.StartLine = lex.NewLines.GetLine(lex.ts)
token.Position.EndLine = lex.NewLines.GetLine(lex.te - 1)
token.Position.StartPos = lex.ts
token.Position.EndPos = lex.te
}
func (lex *Lexer) addToken(id TokenID, ps, pe int) {

6
internal/scanner/token.go
View File

@ -1,6 +1,7 @@
package scanner
import (
"github.com/z7zmey/php-parser/pkg/position"
"github.com/z7zmey/php-parser/pkg/token"
)
@ -9,8 +10,5 @@ type Token struct {
ID TokenID
Value []byte
Tokens []token.Token
StartLine int
EndLine int
StartPos int
EndPos int
Position position.Position
}