maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

scanner.NewToken returns pointer, and scanner.Token saves position as position.Position

Browse Source
This commit is contained in:
z7zmey 2018-04-15 22:56:20 +03:00
parent bc15825663
commit 435dc5c706
14 changed files with 137 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
errors/error.go
View File

@ -10,19 +10,14 @@ import (
// Error parsing error
type Error struct {
Msg string
Pos position.Position
Pos *position.Position
}
// NewError creates and returns new Error
func NewError(msg string, t scanner.Token) *Error {
func NewError(msg string, t *scanner.Token) *Error {
return &Error{
Msg: msg,
Pos: position.Position{
StartLine: t.StartLine,
EndLine: t.EndLine,
StartPos: t.StartPos,
EndPos: t.EndPos,
},
Pos: t.Position(),
}
}

25
errors/error_test.go
View File

@ -26,37 +26,22 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
}
func TestConstructor(t *testing.T) {
token := scanner.Token{
Value: "test",
StartLine: 1,
EndLine: 2,
StartPos: 3,
EndPos: 4,
}
pos := position.NewPosition(1, 2, 3, 4)
token := scanner.NewToken([]byte(`test`), pos)
actual := errors.NewError("message", token)
expected := &errors.Error{
Msg: "message",
Pos: position.Position{
StartLine: 1,
EndLine: 2,
StartPos: 3,
EndPos: 4,
},
Pos: pos,
}
assertEqual(t, expected, actual)
}
func TestPrint(t *testing.T) {
token := scanner.Token{
Value: "test",
StartLine: 1,
EndLine: 2,
StartPos: 3,
EndPos: 4,
}
pos := position.NewPosition(1, 2, 3, 4)
token := scanner.NewToken([]byte(`test`), pos)
Error := errors.NewError("message", token)

57
parser/position_builder.go
View File

@ -100,42 +100,37 @@ func (b *PositionBuilder) NewNodePosition(n node.Node) *position.Position {
}
// NewTokenPosition returns new Position
func (b *PositionBuilder) NewTokenPosition(t scanner.Token) *position.Position {
return position.NewPosition(
t.StartLine,
t.EndLine,
t.StartPos,
t.EndPos,
)
func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position {
return t.Position()
}
// NewTokensPosition returns new Position
func (b *PositionBuilder) NewTokensPosition(startToken scanner.Token, endToken scanner.Token) *position.Position {
func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position {
return position.NewPosition(
startToken.StartLine,
endToken.EndLine,
startToken.StartPos,
endToken.EndPos,
startToken.Position().StartLine,
endToken.Position().EndLine,
startToken.Position().StartPos,
endToken.Position().EndPos,
)
}
// NewTokenNodePosition returns new Position
func (b *PositionBuilder) NewTokenNodePosition(t scanner.Token, n node.Node) *position.Position {
func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n node.Node) *position.Position {
return position.NewPosition(
t.StartLine,
t.Position().StartLine,
b.getNodeEndPos(n).endLine,
t.StartPos,
t.Position().StartPos,
b.getNodeEndPos(n).endPos,
)
}
// NewNodeTokenPosition returns new Position
func (b *PositionBuilder) NewNodeTokenPosition(n node.Node, t scanner.Token) *position.Position {
func (b *PositionBuilder) NewNodeTokenPosition(n node.Node, t *scanner.Token) *position.Position {
return position.NewPosition(
b.getNodeStartPos(n).startLine,
t.EndLine,
t.Position().EndLine,
b.getNodeStartPos(n).startPos,
t.EndPos,
t.Position().EndPos,
)
}
@ -150,21 +145,21 @@ func (b *PositionBuilder) NewNodesPosition(startNode node.Node, endNode node.Nod
}
// NewNodeListTokenPosition returns new Position
func (b *PositionBuilder) NewNodeListTokenPosition(list []node.Node, t scanner.Token) *position.Position {
func (b *PositionBuilder) NewNodeListTokenPosition(list []node.Node, t *scanner.Token) *position.Position {
return position.NewPosition(
b.getListStartPos(list).startLine,
t.EndLine,
t.Position().EndLine,
b.getListStartPos(list).startPos,
t.EndPos,
t.Position().EndPos,
)
}
// NewTokenNodeListPosition returns new Position
func (b *PositionBuilder) NewTokenNodeListPosition(t scanner.Token, list []node.Node) *position.Position {
func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []node.Node) *position.Position {
return position.NewPosition(
t.StartLine,
t.Position().StartLine,
b.getListEndPos(list).endLine,
t.StartPos,
t.Position().StartPos,
b.getListEndPos(list).endPos,
)
}
@ -180,20 +175,20 @@ func (b *PositionBuilder) NewNodeNodeListPosition(n node.Node, list []node.Node)
}
// NewOptionalListTokensPosition returns new Position
func (b *PositionBuilder) NewOptionalListTokensPosition(list []node.Node, t scanner.Token, endToken scanner.Token) *position.Position {
func (b *PositionBuilder) NewOptionalListTokensPosition(list []node.Node, t *scanner.Token, endToken *scanner.Token) *position.Position {
if list == nil {
return position.NewPosition(
t.StartLine,
endToken.EndLine,
t.StartPos,
endToken.EndPos,
t.Position().StartLine,
endToken.Position().EndLine,
t.Position().StartPos,
endToken.Position().EndPos,
)
}
return position.NewPosition(
b.getListStartPos(list).startLine,
endToken.EndLine,
endToken.Position().EndLine,
b.getListStartPos(list).startPos,
endToken.EndPos,
endToken.Position().EndPos,
)
}

39
parser/position_builder_test.go
View File

@ -13,7 +13,8 @@ import (
func TestNewTokenPosition(t *testing.T) {
builder := parser.PositionBuilder{}
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
tpos := position.NewPosition(1, 1, 0, 3)
tkn := scanner.NewToken([]byte(`foo`), tpos)
pos := builder.NewTokenPosition(tkn)
@ -25,8 +26,10 @@ func TestNewTokenPosition(t *testing.T) {
func TestNewTokensPosition(t *testing.T) {
builder := parser.PositionBuilder{}
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6)
tpos1 := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken([]byte(`foo`), tpos1)
tpos2 := position.NewPosition(2, 2, 4, 6)
token2 := scanner.NewToken([]byte(`foo`), tpos2)
pos := builder.NewTokensPosition(token1, token2)
@ -58,7 +61,8 @@ func TestNewNodePosition(t *testing.T) {
}
func TestNewTokenNodePosition(t *testing.T) {
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
tpos := position.NewPosition(1, 1, 0, 3)
tkn := scanner.NewToken([]byte(`foo`), tpos)
n := node.NewIdentifier("test node")
p := &parser.Positions{}
@ -82,7 +86,8 @@ func TestNewTokenNodePosition(t *testing.T) {
func TestNewNodeTokenPosition(t *testing.T) {
n := node.NewIdentifier("test node")
tkn := scanner.NewToken([]byte(`foo`), 2, 2, 10, 12)
tpos := position.NewPosition(2, 2, 10, 12)
tkn := scanner.NewToken([]byte(`foo`), tpos)
p := &parser.Positions{}
p.AddPosition(n, &position.Position{
@ -162,7 +167,8 @@ func TestNewNodesPosition(t *testing.T) {
func TestNewNodeListTokenPosition(t *testing.T) {
n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node")
tkn := scanner.NewToken([]byte(`foo`), 3, 3, 20, 22)
tpos := position.NewPosition(3, 3, 20, 22)
tkn := scanner.NewToken([]byte(`foo`), tpos)
builder := parser.PositionBuilder{
Positions: &parser.Positions{
@ -189,7 +195,8 @@ func TestNewNodeListTokenPosition(t *testing.T) {
}
func TestNewTokenNodeListPosition(t *testing.T) {
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 2)
tpos := position.NewPosition(1, 1, 0, 2)
tkn := scanner.NewToken([]byte(`foo`), tpos)
n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node")
@ -255,8 +262,10 @@ func TestNewNodeNodeListPosition(t *testing.T) {
func TestNewOptionalListTokensPosition(t *testing.T) {
builder := parser.PositionBuilder{}
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6)
tpos1 := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken([]byte(`foo`), tpos1)
tpos2 := position.NewPosition(2, 2, 4, 6)
token2 := scanner.NewToken([]byte(`foo`), tpos2)
pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
@ -293,8 +302,10 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
},
}
token1 := scanner.NewToken([]byte(`foo`), 4, 4, 27, 29)
token2 := scanner.NewToken([]byte(`foo`), 5, 5, 30, 32)
tpos1 := position.NewPosition(4, 4, 27, 29)
token1 := scanner.NewToken([]byte(`foo`), tpos1)
tpos2 := position.NewPosition(5, 5, 30, 32)
token2 := scanner.NewToken([]byte(`foo`), tpos2)
pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2)
@ -335,7 +346,8 @@ func TestNilNodeListPos(t *testing.T) {
}
func TestNilNodeListTokenPos(t *testing.T) {
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
tpos := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken([]byte(`foo`), tpos)
builder := parser.PositionBuilder{}
@ -368,7 +380,8 @@ func TestEmptyNodeListPos(t *testing.T) {
}
func TestEmptyNodeListTokenPos(t *testing.T) {
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
tpos := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken([]byte(`foo`), tpos)
builder := parser.PositionBuilder{}

6
php5/parser.go
View File

@ -10,7 +10,7 @@ import (
"github.com/z7zmey/php-parser/scanner"
)
func (lval *yySymType) Token(t scanner.Token) {
func (lval *yySymType) Token(t *scanner.Token) {
lval.token = t
}
@ -45,12 +45,12 @@ func NewParser(src io.Reader, path string) *Parser {
// Lex proxy to lexer Lex
func (l *Parser) Lex(lval *yySymType) int {
t := l.Lexer.Lex(lval)
l.lastToken = &lval.token
l.lastToken = lval.token
return t
}
func (l *Parser) Error(msg string) {
l.errors = append(l.errors, errors.NewError(msg, *l.lastToken))
l.errors = append(l.errors, errors.NewError(msg, l.lastToken))
}
// Parse the php7 Parser entrypoint

17
php5/php5.go
View File

@ -23,7 +23,7 @@ import (
type yySymType struct {
yys int
node node.Node
token scanner.Token
token *scanner.Token
boolWithToken boolWithToken
list []node.Node
foreachVariable foreachVariable
@ -345,6 +345,7 @@ const yyErrCode = 2
const yyInitialStackSize = 16
//line php5/php5.y:3825
type foreachVariable struct {
node node.Node
byRef bool
@ -352,7 +353,7 @@ type foreachVariable struct {
type nodesWithEndToken struct {
nodes []node.Node
endToken scanner.Token
endToken *scanner.Token
}
type boolWithToken struct {
@ -3132,7 +3133,7 @@ yydefault:
yyDollar = yyS[yypt-1 : yypt+1]
//line php5/php5.y:910
{
yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token}
yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
}
case 91:
yyDollar = yyS[yypt-0 : yypt+1]
@ -3144,7 +3145,7 @@ yydefault:
yyDollar = yyS[yypt-1 : yypt+1]
//line php5/php5.y:917
{
yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token}
yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
}
case 93:
yyDollar = yyS[yypt-9 : yypt+1]
@ -3557,10 +3558,10 @@ yydefault:
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewNodeTokenPosition(yyDollar[1].node, yyDollar[4].token))
yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
} else if yyDollar[2].boolWithToken.value == true {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[2].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[2].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
} else if yyDollar[3].boolWithToken.value == true {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[3].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[3].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
} else {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenPosition(yyDollar[4].token))
@ -3585,10 +3586,10 @@ yydefault:
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewNodesPosition(yyDollar[1].node, yyDollar[6].node))
yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
} else if yyDollar[2].boolWithToken.value == true {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[2].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[2].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
} else if yyDollar[3].boolWithToken.value == true {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[3].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[3].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
} else {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[4].token, yyDollar[6].node))

16
php5/php5.y
View File

@ -20,7 +20,7 @@ import (
%union{
node node.Node
token scanner.Token
token *scanner.Token
boolWithToken boolWithToken
list []node.Node
foreachVariable foreachVariable
@ -907,14 +907,14 @@ is_reference:
/* empty */
{ $$ = boolWithToken{false, nil} }
| '&'
{ $$ = boolWithToken{true, &$1} }
{ $$ = boolWithToken{true, $1} }
;
is_variadic:
/* empty */
{ $$ = boolWithToken{false, nil} }
| T_ELLIPSIS
{ $$ = boolWithToken{true, &$1} }
{ $$ = boolWithToken{true, $1} }
;
unticked_function_declaration_statement:
@ -1255,10 +1255,10 @@ parameter:
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewNodeTokenPosition($1, $4))
yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
} else if $2.value == true {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$2.token, $4))
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($2.token, $4))
yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
} else if $3.value == true {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$3.token, $4))
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($3.token, $4))
yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
} else {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenPosition($4))
@ -1281,10 +1281,10 @@ parameter:
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewNodesPosition($1, $6))
yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
} else if $2.value == true {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$2.token, $6))
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($2.token, $6))
yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
} else if $3.value == true {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$3.token, $6))
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($3.token, $6))
yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
} else {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($4, $6))
@ -3831,7 +3831,7 @@ type foreachVariable struct {
type nodesWithEndToken struct {
nodes []node.Node
endToken scanner.Token
endToken *scanner.Token
}
type boolWithToken struct {

6
php7/parser.go
View File

@ -10,7 +10,7 @@ import (
"github.com/z7zmey/php-parser/scanner"
)
func (lval *yySymType) Token(t scanner.Token) {
func (lval *yySymType) Token(t *scanner.Token) {
lval.token = t
}
@ -45,12 +45,12 @@ func NewParser(src io.Reader, path string) *Parser {
// Lex proxy to lexer Lex
func (l *Parser) Lex(lval *yySymType) int {
t := l.Lexer.Lex(lval)
l.lastToken = &lval.token
l.lastToken = lval.token
return t
}
func (l *Parser) Error(msg string) {
l.errors = append(l.errors, errors.NewError(msg, *l.lastToken))
l.errors = append(l.errors, errors.NewError(msg, l.lastToken))
}
// Parse the php7 Parser entrypoint

19
php7/php7.go
View File

@ -23,7 +23,7 @@ import (
type yySymType struct {
yys int
node node.Node
token scanner.Token
token *scanner.Token
boolWithToken boolWithToken
list []node.Node
foreachVariable foreachVariable
@ -345,6 +345,7 @@ const yyErrCode = 2
const yyInitialStackSize = 16
//line php7/php7.y:2635
type foreachVariable struct {
node node.Node
byRef bool
@ -352,7 +353,7 @@ type foreachVariable struct {
type nodesWithEndToken struct {
nodes []node.Node
endToken scanner.Token
endToken *scanner.Token
}
type boolWithToken struct {
@ -3242,7 +3243,7 @@ yydefault:
yyDollar = yyS[yypt-1 : yypt+1]
//line php7/php7.y:764
{
yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token}
yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
}
case 167:
yyDollar = yyS[yypt-0 : yypt+1]
@ -3254,7 +3255,7 @@ yydefault:
yyDollar = yyS[yypt-1 : yypt+1]
//line php7/php7.y:769
{
yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token}
yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
}
case 169:
yyDollar = yyS[yypt-9 : yypt+1]
@ -3623,11 +3624,11 @@ yydefault:
yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
} else if yyDollar[2].boolWithToken.value == true {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[2].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[2].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
} else if yyDollar[3].boolWithToken.value == true {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[3].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[3].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
} else {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
@ -3653,11 +3654,11 @@ yydefault:
yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
} else if yyDollar[2].boolWithToken.value == true {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[2].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[2].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
} else if yyDollar[3].boolWithToken.value == true {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[3].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[3].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
} else {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
@ -4893,7 +4894,7 @@ yydefault:
yyDollar = yyS[yypt-1 : yypt+1]
//line php7/php7.y:1963
{
yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token}
yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
}
case 374:
yyDollar = yyS[yypt-0 : yypt+1]

18
php7/php7.y
View File

@ -20,7 +20,7 @@ import (
%union{
node node.Node
token scanner.Token
token *scanner.Token
boolWithToken boolWithToken
list []node.Node
foreachVariable foreachVariable
@ -761,12 +761,12 @@ function_declaration_statement:
is_reference:
/* empty */ { $$ = boolWithToken{false, nil} }
| '&' { $$ = boolWithToken{true, &$1} }
| '&' { $$ = boolWithToken{true, $1} }
;
is_variadic:
/* empty */ { $$ = boolWithToken{false, nil} }
| T_ELLIPSIS { $$ = boolWithToken{true, &$1} }
| T_ELLIPSIS { $$ = boolWithToken{true, $1} }
;
class_declaration_statement:
@ -1044,11 +1044,11 @@ parameter:
yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
} else if $2.value == true {
$$ = node.NewParameter($1, variable, nil, $2.value, $3.value)
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$2.token, $4))
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($2.token, $4))
yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
} else if $3.value == true {
$$ = node.NewParameter($1, variable, nil, $2.value, $3.value)
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$3.token, $4))
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($3.token, $4))
yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
} else {
$$ = node.NewParameter($1, variable, nil, $2.value, $3.value)
@ -1072,11 +1072,11 @@ parameter:
yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
} else if $2.value == true {
$$ = node.NewParameter($1, variable, $6, $2.value, $3.value)
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$2.token, $6))
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($2.token, $6))
yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
} else if $3.value == true {
$$ = node.NewParameter($1, variable, $6, $2.value, $3.value)
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$3.token, $6))
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($3.token, $6))
yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
} else {
$$ = node.NewParameter($1, variable, $6, $2.value, $3.value)
@ -1960,7 +1960,7 @@ backup_doc_comment:
returns_ref:
/* empty */ { $$ = boolWithToken{false, nil} }
| '&' { $$ = boolWithToken{true, &$1} }
| '&' { $$ = boolWithToken{true, $1} }
;
lexical_vars:
@ -2641,7 +2641,7 @@ type foreachVariable struct {
type nodesWithEndToken struct {
nodes []node.Node
endToken scanner.Token
endToken *scanner.Token
}
type boolWithToken struct {

16
scanner/lexer.go
View File

@ -432,7 +432,7 @@ const T_POW = 57481
// Lval parsers yySymType must implement this interface
type Lval interface {
Token(tkn Token)
Token(tkn *Token)
}
// Lexer php lexer
@ -510,16 +510,18 @@ func (l *Lexer) getCurrentState() int {
return l.StateStack[len(l.StateStack)-1]
}
func (l *Lexer) newToken(chars []lex.Char) Token {
func (l *Lexer) newToken(chars []lex.Char) *Token {
firstChar := chars[0]
lastChar := chars[len(chars)-1]
startLine := l.File.Line(firstChar.Pos())
endLine := l.File.Line(lastChar.Pos())
startPos := int(firstChar.Pos())
endPos := int(lastChar.Pos())
pos := position.NewPosition(
l.File.Line(firstChar.Pos()),
l.File.Line(lastChar.Pos()),
int(firstChar.Pos()),
int(lastChar.Pos()),
)
return NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments)
return NewToken(l.charsToBytes(chars), pos).SetComments(l.Comments)
}
func (l *Lexer) addComment(chars []lex.Char) {

4
scanner/scanner_test.go
View File

@ -28,10 +28,10 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
}
type lval struct {
Tkn scanner.Token
Tkn *scanner.Token
}
func (lv *lval) Token(t scanner.Token) {
func (lv *lval) Token(t *scanner.Token) {
lv.Tkn = t
}

26
scanner/token.go
View File

@ -2,35 +2,41 @@ package scanner
import (
"github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/position"
)
// Token value returned by lexer
type Token struct {
Value string
StartLine int
EndLine int
StartPos int
EndPos int
position *position.Position
comments []*comment.Comment
}
// NewToken Token constructor
// TODO: return pointer
func NewToken(value []byte, startLine int, endLine int, startPos int, endPos int) Token {
return Token{string(value), startLine, endLine, startPos, endPos, nil}
func NewToken(value []byte, pos *position.Position) *Token {
return &Token{
Value: string(value),
position: pos,
comments: nil,
}
}
func (t Token) String() string {
func (t *Token) String() string {
return string(t.Value)
}
// Position returns token position
func (t *Token) Position() *position.Position {
return t.position
}
// Comments returns attached comments
func (t Token) Comments() []*comment.Comment {
func (t *Token) Comments() []*comment.Comment {
return t.comments
}
// SetComments attach comments
func (t Token) SetComments(comments []*comment.Comment) Token {
func (t *Token) SetComments(comments []*comment.Comment) *Token {
t.comments = comments
return t
}

9
scanner/token_test.go
View File

@ -4,13 +4,16 @@ import (
"reflect"
"testing"
"github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/scanner"
)
func TestToken(t *testing.T) {
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
pos := position.NewPosition(1, 1, 0, 3)
tkn := scanner.NewToken([]byte(`foo`), pos)
c := []*comment.Comment{
comment.NewComment("test comment", nil),
@ -18,7 +21,7 @@ func TestToken(t *testing.T) {
tkn.SetComments(c)
if reflect.DeepEqual(tkn.Comments(), c) {
if !reflect.DeepEqual(tkn.Comments(), c) {
t.Errorf("comments are not equal\n")
}
@ -26,7 +29,7 @@ func TestToken(t *testing.T) {
t.Errorf("token value is not equal\n")
}
if tkn.StartLine != 1 || tkn.EndLine != 1 || tkn.StartPos != 0 || tkn.EndPos != 3 {
if tkn.Position() != pos {
t.Errorf("token position is not equal\n")
}
}