maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[refactoring] use position.Position to store token position

Browse Source
This commit is contained in:
Vadym Slizov 2020-05-17 23:50:23 +03:00
parent f4c15f4671
commit e98607f5d0
7 changed files with 126 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
internal/php5/parser.go
View File

@ -8,7 +8,6 @@ import (
"github.com/z7zmey/php-parser/internal/scanner"
"github.com/z7zmey/php-parser/pkg/ast"
"github.com/z7zmey/php-parser/pkg/errors"
"github.com/z7zmey/php-parser/pkg/position"
"github.com/z7zmey/php-parser/pkg/token"
)
@ -45,14 +44,9 @@ func (l *Parser) Lex(lval *yySymType) int {
}
func (l *Parser) Error(msg string) {
pos := &position.Position{
StartLine: l.currentToken.StartLine,
EndLine: l.currentToken.EndLine,
StartPos: l.currentToken.StartPos,
EndPos: l.currentToken.EndPos,
}
var pos = l.currentToken.Position
l.Lexer.AddError(errors.NewError(msg, pos))
l.Lexer.AddError(errors.NewError(msg, &pos))
}
func (l *Parser) WithTokens() {

BIN
internal/php5/php5.go
View File

Binary file not shown.

10
internal/php7/parser.go
View File

@ -7,7 +7,6 @@ import (
"github.com/z7zmey/php-parser/internal/scanner"
"github.com/z7zmey/php-parser/pkg/ast"
"github.com/z7zmey/php-parser/pkg/errors"
"github.com/z7zmey/php-parser/pkg/position"
"github.com/z7zmey/php-parser/pkg/token"
)
@ -43,14 +42,9 @@ func (l *Parser) Lex(lval *yySymType) int {
}
func (l *Parser) Error(msg string) {
pos := &position.Position{
StartLine: l.currentToken.StartLine,
EndLine: l.currentToken.EndLine,
StartPos: l.currentToken.StartPos,
EndPos: l.currentToken.EndPos,
}
var pos = l.currentToken.Position
l.Lexer.AddError(errors.NewError(msg, pos))
l.Lexer.AddError(errors.NewError(msg, &pos))
}
func (l *Parser) WithTokens() {

44
internal/positionbuilder/position_builder.go
View File

@ -100,29 +100,29 @@ func (b *PositionBuilder) NewNodePosition(n ast.Vertex) *position.Position {
// NewTokenPosition returns new Position
func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position {
return &position.Position{
StartLine: t.StartLine,
EndLine: t.EndLine,
StartPos: t.StartPos,
EndPos: t.EndPos,
StartLine: t.Position.StartLine,
EndLine: t.Position.EndLine,
StartPos: t.Position.StartPos,
EndPos: t.Position.EndPos,
}
}
// NewTokensPosition returns new Position
func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position {
return &position.Position{
StartLine: startToken.StartLine,
EndLine: endToken.EndLine,
StartPos: startToken.StartPos,
EndPos: endToken.EndPos,
StartLine: startToken.Position.StartLine,
EndLine: endToken.Position.EndLine,
StartPos: startToken.Position.StartPos,
EndPos: endToken.Position.EndPos,
}
}
// NewTokenNodePosition returns new Position
func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n ast.Vertex) *position.Position {
return &position.Position{
StartLine: t.StartLine,
StartLine: t.Position.StartLine,
EndLine: b.getNodeEndPos(n).endLine,
StartPos: t.StartPos,
StartPos: t.Position.StartPos,
EndPos: b.getNodeEndPos(n).endPos,
}
}
@ -131,9 +131,9 @@ func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n ast.Vertex) *
func (b *PositionBuilder) NewNodeTokenPosition(n ast.Vertex, t *scanner.Token) *position.Position {
return &position.Position{
StartLine: b.getNodeStartPos(n).startLine,
EndLine: t.EndLine,
EndLine: t.Position.EndLine,
StartPos: b.getNodeStartPos(n).startPos,
EndPos: t.EndPos,
EndPos: t.Position.EndPos,
}
}
@ -151,18 +151,18 @@ func (b *PositionBuilder) NewNodesPosition(startNode ast.Vertex, endNode ast.Ver
func (b *PositionBuilder) NewNodeListTokenPosition(list []ast.Vertex, t *scanner.Token) *position.Position {
return &position.Position{
StartLine: b.getListStartPos(list).startLine,
EndLine: t.EndLine,
EndLine: t.Position.EndLine,
StartPos: b.getListStartPos(list).startPos,
EndPos: t.EndPos,
EndPos: t.Position.EndPos,
}
}
// NewTokenNodeListPosition returns new Position
func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []ast.Vertex) *position.Position {
return &position.Position{
StartLine: t.StartLine,
StartLine: t.Position.StartLine,
EndLine: b.getListEndPos(list).endLine,
StartPos: t.StartPos,
StartPos: t.Position.StartPos,
EndPos: b.getListEndPos(list).endPos,
}
}
@ -191,17 +191,17 @@ func (b *PositionBuilder) NewNodeListNodePosition(list []ast.Vertex, n ast.Verte
func (b *PositionBuilder) NewOptionalListTokensPosition(list []ast.Vertex, t *scanner.Token, endToken *scanner.Token) *position.Position {
if list == nil {
return &position.Position{
StartLine: t.StartLine,
EndLine: endToken.EndLine,
StartPos: t.StartPos,
EndPos: endToken.EndPos,
StartLine: t.Position.StartLine,
EndLine: endToken.Position.EndLine,
StartPos: t.Position.StartPos,
EndPos: endToken.Position.EndPos,
}
}
return &position.Position{
StartLine: b.getListStartPos(list).startLine,
EndLine: endToken.EndLine,
EndLine: endToken.Position.EndLine,
StartPos: b.getListStartPos(list).startPos,
EndPos: endToken.EndPos,
EndPos: endToken.Position.EndPos,
}
}

156
internal/positionbuilder/position_builder_test.go
View File

@ -14,11 +14,13 @@ func TestNewTokenPosition(t *testing.T) {
builder := positionbuilder.PositionBuilder{}
tkn := &scanner.Token{
Value: []byte(`foo`),
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
pos := builder.NewTokenPosition(tkn)
@ -32,18 +34,22 @@ func TestNewTokensPosition(t *testing.T) {
builder := positionbuilder.PositionBuilder{}
token1 := &scanner.Token{
Value: []byte(`foo`),
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
token2 := &scanner.Token{
Value: []byte(`foo`),
StartLine: 2,
EndLine: 2,
StartPos: 4,
EndPos: 6,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 2,
EndLine: 2,
StartPos: 4,
EndPos: 6,
},
}
pos := builder.NewTokensPosition(token1, token2)
@ -72,11 +78,13 @@ func TestNewNodePosition(t *testing.T) {
func TestNewTokenNodePosition(t *testing.T) {
tkn := &scanner.Token{
Value: []byte(`foo`),
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
n := &ast.Identifier{
Node: ast.Node{
@ -109,11 +117,13 @@ func TestNewNodeTokenPosition(t *testing.T) {
}
tkn := &scanner.Token{
Value: []byte(`foo`),
StartLine: 2,
EndLine: 2,
StartPos: 10,
EndPos: 12,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 2,
EndLine: 2,
StartPos: 10,
EndPos: 12,
},
}
builder := positionbuilder.PositionBuilder{}
@ -207,11 +217,13 @@ func TestNewNodeListTokenPosition(t *testing.T) {
}
tkn := &scanner.Token{
Value: []byte(`foo`),
StartLine: 3,
EndLine: 3,
StartPos: 20,
EndPos: 22,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 3,
EndLine: 3,
StartPos: 20,
EndPos: 22,
},
}
builder := positionbuilder.PositionBuilder{}
@ -223,11 +235,13 @@ func TestNewNodeListTokenPosition(t *testing.T) {
func TestNewTokenNodeListPosition(t *testing.T) {
tkn := &scanner.Token{
Value: []byte(`foo`),
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 2,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 2,
},
}
n1 := &ast.Identifier{
@ -343,18 +357,22 @@ func TestNewOptionalListTokensPosition(t *testing.T) {
builder := positionbuilder.PositionBuilder{}
token1 := &scanner.Token{
Value: []byte(`foo`),
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
token2 := &scanner.Token{
Value: []byte(`foo`),
StartLine: 2,
EndLine: 2,
StartPos: 4,
EndPos: 6,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 2,
EndLine: 2,
StartPos: 4,
EndPos: 6,
},
}
pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
@ -387,18 +405,22 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
builder := positionbuilder.PositionBuilder{}
token1 := &scanner.Token{
Value: []byte(`foo`),
StartLine: 4,
EndLine: 4,
StartPos: 27,
EndPos: 29,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 4,
EndLine: 4,
StartPos: 27,
EndPos: 29,
},
}
token2 := &scanner.Token{
Value: []byte(`foo`),
StartLine: 5,
EndLine: 5,
StartPos: 30,
EndPos: 32,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 5,
EndLine: 5,
StartPos: 30,
EndPos: 32,
},
}
pos := builder.NewOptionalListTokensPosition([]ast.Vertex{n2, n3}, token1, token2)
@ -435,11 +457,13 @@ func TestNilNodeListPos(t *testing.T) {
func TestNilNodeListTokenPos(t *testing.T) {
token := &scanner.Token{
Value: []byte(`foo`),
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
builder := positionbuilder.PositionBuilder{}
@ -470,11 +494,13 @@ func TestEmptyNodeListPos(t *testing.T) {
func TestEmptyNodeListTokenPos(t *testing.T) {
token := &scanner.Token{
Value: []byte(`foo`),
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
Value: []byte(`foo`),
Position: position.Position{
StartLine: 1,
EndLine: 1,
StartPos: 0,
EndPos: 3,
},
}
builder := positionbuilder.PositionBuilder{}

8
internal/scanner/lexer.go
View File

@ -78,10 +78,10 @@ func (l *Lexer) SetErrors(e []*errors.Error) {
}
func (lex *Lexer) setTokenPosition(token *Token) {
token.StartLine = lex.NewLines.GetLine(lex.ts)
token.EndLine = lex.NewLines.GetLine(lex.te - 1)
token.StartPos = lex.ts
token.EndPos = lex.te
token.Position.StartLine = lex.NewLines.GetLine(lex.ts)
token.Position.EndLine = lex.NewLines.GetLine(lex.te - 1)
token.Position.StartPos = lex.ts
token.Position.EndPos = lex.te
}
func (lex *Lexer) addToken(id TokenID, ps, pe int) {

12
internal/scanner/token.go
View File

@ -1,16 +1,14 @@
package scanner
import (
"github.com/z7zmey/php-parser/pkg/position"
"github.com/z7zmey/php-parser/pkg/token"
)
// Token value returned by lexer
type Token struct {
ID TokenID
Value []byte
Tokens []token.Token
StartLine int
EndLine int
StartPos int
EndPos int
ID TokenID
Value []byte
Tokens []token.Token
Position position.Position
}