maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add start column and end column to position struct

Browse Source
This commit is contained in:
Laytan Laats
2023-03-26 01:54:00 +01:00
parent 92019441d0
commit 7c12f73974
19 changed files with 316 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
internal/php8/lexer.go
View File

@@ -4,6 +4,7 @@ import (
"bytes"
"strings"
pos "github.com/VKCOM/php-parser/internal/position"
"github.com/VKCOM/php-parser/pkg/conf"
"github.com/VKCOM/php-parser/pkg/errors"
"github.com/VKCOM/php-parser/pkg/position"
@@ -16,7 +17,11 @@ type Lexer struct {
phpVersion *version.Version
errHandlerFunc func(*errors.Error)
p, pe, cs int
// p: current position being lexed/checked.
// pe: length in bytes of input.
p, pe, cs int
// ts: start position of the current token.
// te: end position of the current token.
ts, te, act int
stack []int
top int
@@ -24,7 +29,7 @@ type Lexer struct {
heredocLabel []byte
tokenPool *token.Pool
positionPool *position.Pool
newLines NewLines
newLines pos.NewLines
}
func NewLexer(data []byte, config conf.Config) *Lexer {
@@ -36,9 +41,9 @@ func NewLexer(data []byte, config conf.Config) *Lexer {
pe: len(data),
stack: make([]int, 0),
tokenPool: token.NewPool(position.DefaultBlockSize),
positionPool: position.NewPool(token.DefaultBlockSize),
newLines: NewLines{make([]int, 0, 128)},
tokenPool: token.NewPool(token.DefaultBlockSize),
positionPool: position.NewPool(position.DefaultBlockSize),
newLines: pos.NewNewLines(),
}
initLexer(lex)
@@ -49,10 +54,15 @@ func NewLexer(data []byte, config conf.Config) *Lexer {
func (lex *Lexer) setTokenPosition(token *token.Token) {
pos := lex.positionPool.Get()
pos.StartLine = lex.newLines.GetLine(lex.ts)
pos.EndLine = lex.newLines.GetLine(lex.te - 1)
sl, slb := lex.newLines.GetLine(lex.ts)
el, elb := lex.newLines.GetLine(lex.te - 1)
pos.StartLine = sl
pos.EndLine = el
pos.StartPos = lex.ts
pos.EndPos = lex.te
pos.StartCol = lex.ts - slb
pos.EndCol = lex.te - elb
token.Position = pos
}
@@ -249,11 +259,15 @@ func (lex *Lexer) error(msg string) {
return
}
sl, slb := lex.newLines.GetLine(lex.ts)
el, elb := lex.newLines.GetLine(lex.te - 1)
pos := position.NewPosition(
lex.newLines.GetLine(lex.ts),
lex.newLines.GetLine(lex.te-1),
sl,
el,
lex.ts,
lex.te,
lex.ts-slb,
lex.te-elb,
)
lex.errHandlerFunc(errors.NewError(msg, pos))

25
internal/php8/newline.go
View File

@@ -1,25 +0,0 @@
package php8
type NewLines struct {
data []int
}
func (nl *NewLines) Append(p int) {
if len(nl.data) == 0 || nl.data[len(nl.data)-1] < p {
nl.data = append(nl.data, p)
}
}
func (nl *NewLines) GetLine(p int) int {
line := len(nl.data) + 1
for i := len(nl.data) - 1; i >= 0; i-- {
if p < nl.data[i] {
line = i + 1
} else {
break
}
}
return line
}

2
internal/php8/parser_php8_2_test.go
View File

@@ -409,7 +409,7 @@ trait Foo {
}
`
suite.Expected = `&ast.Root{
suite.Expected = `&ast.Root{
Stmts: []ast.Vertex{
&ast.StmtTrait{
Name: &ast.Identifier{

28
internal/php8/parser_php8_test.go
View File

@@ -1435,7 +1435,7 @@ func TestRealCastParserError(t *testing.T) {
suite.Expected = []*errors.Error{
{
Msg: "The (real) cast has been removed, use (float) instead",
Pos: position.NewPosition(2, 2, 7, 13),
Pos: position.NewPosition(2, 2, 7, 13, 1, 7),
},
}
@@ -1455,7 +1455,7 @@ func TestUnsetCastParserError(t *testing.T) {
suite.Expected = []*errors.Error{
{
Msg: "The (unset) cast is no longer supported",
Pos: position.NewPosition(2, 2, 7, 14),
Pos: position.NewPosition(2, 2, 7, 14, 1, 1),
},
}
@@ -1980,15 +1980,15 @@ new $a{10}; // Error
suite.Expected = []*errors.Error{
{
Msg: "Array and string offset access syntax with curly braces is no longer supported",
Pos: position.NewPosition(3, 3, 26, 27),
Pos: position.NewPosition(3, 3, 26, 27, 1, 1),
},
{
Msg: "Array and string offset access syntax with curly braces is no longer supported",
Pos: position.NewPosition(9, 9, 97, 98),
Pos: position.NewPosition(9, 9, 97, 98, 1, 1),
},
{
Msg: "Array and string offset access syntax with curly braces is no longer supported",
Pos: position.NewPosition(12, 12, 137, 138),
Pos: position.NewPosition(12, 12, 137, 138, 1, 1),
},
}
@@ -3020,19 +3020,19 @@ class Foo {
suite.Expected = []*errors.Error{
{
Msg: "syntax error: unexpected T_STATIC, expecting T_VARIABLE",
Pos: position.NewPosition(2, 2, 18, 24),
Pos: position.NewPosition(2, 2, 18, 24, 1, 1),
},
{
Msg: "syntax error: unexpected T_STATIC, expecting T_VARIABLE",
Pos: position.NewPosition(3, 3, 48, 54),
Pos: position.NewPosition(3, 3, 48, 54, 1, 1),
},
{
Msg: "syntax error: unexpected '|', expecting T_VARIABLE",
Pos: position.NewPosition(8, 8, 154, 155),
Pos: position.NewPosition(8, 8, 154, 155, 1, 1),
},
{
Msg: "syntax error: unexpected '}'",
Pos: position.NewPosition(10, 10, 182, 183),
Pos: position.NewPosition(10, 10, 182, 183, 1, 1),
},
}
@@ -3051,15 +3051,15 @@ function f(int|string|null $a) {} // ok
suite.Expected = []*errors.Error{
{
Msg: "syntax error: unexpected '|', expecting T_VARIABLE",
Pos: position.NewPosition(2, 2, 22, 23),
Pos: position.NewPosition(2, 2, 22, 23, 1, 1),
},
{
Msg: "syntax error: unexpected '('",
Pos: position.NewPosition(3, 3, 49, 50),
Pos: position.NewPosition(3, 3, 49, 50, 1, 1),
},
{
Msg: "syntax error: unexpected T_VARIABLE",
Pos: position.NewPosition(3, 3, 62, 64),
Pos: position.NewPosition(3, 3, 62, 64, 1, 1),
},
}
@@ -4537,7 +4537,7 @@ use \ Foo \ Boo;
suite.Expected = []*errors.Error{
{
Msg: "syntax error: unexpected T_NS_SEPARATOR",
Pos: position.NewPosition(2, 2, 11, 12),
Pos: position.NewPosition(2, 2, 11, 12, 1, 1),
},
}
@@ -4554,7 +4554,7 @@ function f(\Foo \ Boo $a) {}
suite.Expected = []*errors.Error{
{
Msg: "syntax error: unexpected T_NS_SEPARATOR, expecting T_VARIABLE",
Pos: position.NewPosition(2, 2, 23, 24),
Pos: position.NewPosition(2, 2, 23, 24, 1, 1),
},
}

5
internal/php8/parser_test.go
View File

@@ -14,6 +14,11 @@ import (
"github.com/VKCOM/php-parser/pkg/version"
)
func TestMain(m *testing.M) {
// Ignore StartCol and EndCol in equality checks, these tests were written before they were added.
position.CheckColEquality = false
}
func TestIdentifier(t *testing.T) {
src := `<? $foo;`

16
internal/php8/scanner_php8_test.go
View File

@@ -366,40 +366,40 @@ $a;
{
ID: php8.T_OPEN_TAG,
Value: []byte("<?php"),
Position: &position.Position{StartLine: 1, EndLine: 1, EndPos: 5},
Position: &position.Position{StartLine: 1, EndLine: 1, EndPos: 5, StartCol: 0, EndCol: 5},
},
{
ID: php8.T_WHITESPACE,
Value: []byte("\n"),
Position: &position.Position{StartLine: 1, EndLine: 1, StartPos: 5, EndPos: 6},
Position: &position.Position{StartLine: 1, EndLine: 1, StartPos: 5, EndPos: 6, StartCol: 5, EndCol: 6},
},
{
ID: php8.T_COMMENT,
Value: []byte("#\n"),
Position: &position.Position{StartLine: 2, EndLine: 2, StartPos: 6, EndPos: 8},
Position: &position.Position{StartLine: 2, EndLine: 2, StartPos: 6, EndPos: 8, StartCol: 0, EndCol: 2},
},
{
ID: php8.T_COMMENT,
Value: []byte("# Hello\n"),
Position: &position.Position{StartLine: 3, EndLine: 3, StartPos: 8, EndPos: 16},
Position: &position.Position{StartLine: 3, EndLine: 3, StartPos: 8, EndPos: 16, StartCol: 0, EndCol: 8},
},
{
ID: php8.T_COMMENT,
Value: []byte("#\n"),
Position: &position.Position{StartLine: 4, EndLine: 4, StartPos: 16, EndPos: 18},
Position: &position.Position{StartLine: 4, EndLine: 4, StartPos: 16, EndPos: 18, StartCol: 0, EndCol: 2},
},
{
ID: php8.T_WHITESPACE,
Value: []byte("\n"),
Position: &position.Position{StartLine: 5, EndLine: 5, StartPos: 18, EndPos: 19},
Position: &position.Position{StartLine: 5, EndLine: 5, StartPos: 18, EndPos: 19, StartCol: 0, EndCol: 1},
},
},
Position: &position.Position{StartLine: 6, EndLine: 6, StartPos: 19, EndPos: 21},
Position: &position.Position{StartLine: 6, EndLine: 6, StartPos: 19, EndPos: 21, StartCol: 0, EndCol: 2},
},
{
ID: ';',
Value: []byte(";"),
Position: &position.Position{StartLine: 6, EndLine: 6, StartPos: 21, EndPos: 22},
Position: &position.Position{StartLine: 6, EndLine: 6, StartPos: 21, EndPos: 22, StartCol: 2, EndCol: 3},
},
}
suite.Run()