scanner.NewToken returns pointer, and scanner.Token saves position as position.Position

This commit is contained in:
z7zmey 2018-04-15 22:56:20 +03:00
parent bc15825663
commit 435dc5c706
14 changed files with 137 additions and 136 deletions

View File

@ -10,19 +10,14 @@ import (
// Error parsing error // Error parsing error
type Error struct { type Error struct {
Msg string Msg string
Pos position.Position Pos *position.Position
} }
// NewError creates and returns new Error // NewError creates and returns new Error
func NewError(msg string, t scanner.Token) *Error { func NewError(msg string, t *scanner.Token) *Error {
return &Error{ return &Error{
Msg: msg, Msg: msg,
Pos: position.Position{ Pos: t.Position(),
StartLine: t.StartLine,
EndLine: t.EndLine,
StartPos: t.StartPos,
EndPos: t.EndPos,
},
} }
} }

View File

@ -26,37 +26,22 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
} }
func TestConstructor(t *testing.T) { func TestConstructor(t *testing.T) {
token := scanner.Token{ pos := position.NewPosition(1, 2, 3, 4)
Value: "test", token := scanner.NewToken([]byte(`test`), pos)
StartLine: 1,
EndLine: 2,
StartPos: 3,
EndPos: 4,
}
actual := errors.NewError("message", token) actual := errors.NewError("message", token)
expected := &errors.Error{ expected := &errors.Error{
Msg: "message", Msg: "message",
Pos: position.Position{ Pos: pos,
StartLine: 1,
EndLine: 2,
StartPos: 3,
EndPos: 4,
},
} }
assertEqual(t, expected, actual) assertEqual(t, expected, actual)
} }
func TestPrint(t *testing.T) { func TestPrint(t *testing.T) {
token := scanner.Token{ pos := position.NewPosition(1, 2, 3, 4)
Value: "test", token := scanner.NewToken([]byte(`test`), pos)
StartLine: 1,
EndLine: 2,
StartPos: 3,
EndPos: 4,
}
Error := errors.NewError("message", token) Error := errors.NewError("message", token)

View File

@ -100,42 +100,37 @@ func (b *PositionBuilder) NewNodePosition(n node.Node) *position.Position {
} }
// NewTokenPosition returns new Position // NewTokenPosition returns new Position
func (b *PositionBuilder) NewTokenPosition(t scanner.Token) *position.Position { func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position {
return position.NewPosition( return t.Position()
t.StartLine,
t.EndLine,
t.StartPos,
t.EndPos,
)
} }
// NewTokensPosition returns new Position // NewTokensPosition returns new Position
func (b *PositionBuilder) NewTokensPosition(startToken scanner.Token, endToken scanner.Token) *position.Position { func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position {
return position.NewPosition( return position.NewPosition(
startToken.StartLine, startToken.Position().StartLine,
endToken.EndLine, endToken.Position().EndLine,
startToken.StartPos, startToken.Position().StartPos,
endToken.EndPos, endToken.Position().EndPos,
) )
} }
// NewTokenNodePosition returns new Position // NewTokenNodePosition returns new Position
func (b *PositionBuilder) NewTokenNodePosition(t scanner.Token, n node.Node) *position.Position { func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n node.Node) *position.Position {
return position.NewPosition( return position.NewPosition(
t.StartLine, t.Position().StartLine,
b.getNodeEndPos(n).endLine, b.getNodeEndPos(n).endLine,
t.StartPos, t.Position().StartPos,
b.getNodeEndPos(n).endPos, b.getNodeEndPos(n).endPos,
) )
} }
// NewNodeTokenPosition returns new Position // NewNodeTokenPosition returns new Position
func (b *PositionBuilder) NewNodeTokenPosition(n node.Node, t scanner.Token) *position.Position { func (b *PositionBuilder) NewNodeTokenPosition(n node.Node, t *scanner.Token) *position.Position {
return position.NewPosition( return position.NewPosition(
b.getNodeStartPos(n).startLine, b.getNodeStartPos(n).startLine,
t.EndLine, t.Position().EndLine,
b.getNodeStartPos(n).startPos, b.getNodeStartPos(n).startPos,
t.EndPos, t.Position().EndPos,
) )
} }
@ -150,21 +145,21 @@ func (b *PositionBuilder) NewNodesPosition(startNode node.Node, endNode node.Nod
} }
// NewNodeListTokenPosition returns new Position // NewNodeListTokenPosition returns new Position
func (b *PositionBuilder) NewNodeListTokenPosition(list []node.Node, t scanner.Token) *position.Position { func (b *PositionBuilder) NewNodeListTokenPosition(list []node.Node, t *scanner.Token) *position.Position {
return position.NewPosition( return position.NewPosition(
b.getListStartPos(list).startLine, b.getListStartPos(list).startLine,
t.EndLine, t.Position().EndLine,
b.getListStartPos(list).startPos, b.getListStartPos(list).startPos,
t.EndPos, t.Position().EndPos,
) )
} }
// NewTokenNodeListPosition returns new Position // NewTokenNodeListPosition returns new Position
func (b *PositionBuilder) NewTokenNodeListPosition(t scanner.Token, list []node.Node) *position.Position { func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []node.Node) *position.Position {
return position.NewPosition( return position.NewPosition(
t.StartLine, t.Position().StartLine,
b.getListEndPos(list).endLine, b.getListEndPos(list).endLine,
t.StartPos, t.Position().StartPos,
b.getListEndPos(list).endPos, b.getListEndPos(list).endPos,
) )
} }
@ -180,20 +175,20 @@ func (b *PositionBuilder) NewNodeNodeListPosition(n node.Node, list []node.Node)
} }
// NewOptionalListTokensPosition returns new Position // NewOptionalListTokensPosition returns new Position
func (b *PositionBuilder) NewOptionalListTokensPosition(list []node.Node, t scanner.Token, endToken scanner.Token) *position.Position { func (b *PositionBuilder) NewOptionalListTokensPosition(list []node.Node, t *scanner.Token, endToken *scanner.Token) *position.Position {
if list == nil { if list == nil {
return position.NewPosition( return position.NewPosition(
t.StartLine, t.Position().StartLine,
endToken.EndLine, endToken.Position().EndLine,
t.StartPos, t.Position().StartPos,
endToken.EndPos, endToken.Position().EndPos,
) )
} }
return position.NewPosition( return position.NewPosition(
b.getListStartPos(list).startLine, b.getListStartPos(list).startLine,
endToken.EndLine, endToken.Position().EndLine,
b.getListStartPos(list).startPos, b.getListStartPos(list).startPos,
endToken.EndPos, endToken.Position().EndPos,
) )
} }

View File

@ -13,7 +13,8 @@ import (
func TestNewTokenPosition(t *testing.T) { func TestNewTokenPosition(t *testing.T) {
builder := parser.PositionBuilder{} builder := parser.PositionBuilder{}
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) tpos := position.NewPosition(1, 1, 0, 3)
tkn := scanner.NewToken([]byte(`foo`), tpos)
pos := builder.NewTokenPosition(tkn) pos := builder.NewTokenPosition(tkn)
@ -25,8 +26,10 @@ func TestNewTokenPosition(t *testing.T) {
func TestNewTokensPosition(t *testing.T) { func TestNewTokensPosition(t *testing.T) {
builder := parser.PositionBuilder{} builder := parser.PositionBuilder{}
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) tpos1 := position.NewPosition(1, 1, 0, 3)
token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6) token1 := scanner.NewToken([]byte(`foo`), tpos1)
tpos2 := position.NewPosition(2, 2, 4, 6)
token2 := scanner.NewToken([]byte(`foo`), tpos2)
pos := builder.NewTokensPosition(token1, token2) pos := builder.NewTokensPosition(token1, token2)
@ -58,7 +61,8 @@ func TestNewNodePosition(t *testing.T) {
} }
func TestNewTokenNodePosition(t *testing.T) { func TestNewTokenNodePosition(t *testing.T) {
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) tpos := position.NewPosition(1, 1, 0, 3)
tkn := scanner.NewToken([]byte(`foo`), tpos)
n := node.NewIdentifier("test node") n := node.NewIdentifier("test node")
p := &parser.Positions{} p := &parser.Positions{}
@ -82,7 +86,8 @@ func TestNewTokenNodePosition(t *testing.T) {
func TestNewNodeTokenPosition(t *testing.T) { func TestNewNodeTokenPosition(t *testing.T) {
n := node.NewIdentifier("test node") n := node.NewIdentifier("test node")
tkn := scanner.NewToken([]byte(`foo`), 2, 2, 10, 12) tpos := position.NewPosition(2, 2, 10, 12)
tkn := scanner.NewToken([]byte(`foo`), tpos)
p := &parser.Positions{} p := &parser.Positions{}
p.AddPosition(n, &position.Position{ p.AddPosition(n, &position.Position{
@ -162,7 +167,8 @@ func TestNewNodesPosition(t *testing.T) {
func TestNewNodeListTokenPosition(t *testing.T) { func TestNewNodeListTokenPosition(t *testing.T) {
n1 := node.NewIdentifier("test node") n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node") n2 := node.NewIdentifier("test node")
tkn := scanner.NewToken([]byte(`foo`), 3, 3, 20, 22) tpos := position.NewPosition(3, 3, 20, 22)
tkn := scanner.NewToken([]byte(`foo`), tpos)
builder := parser.PositionBuilder{ builder := parser.PositionBuilder{
Positions: &parser.Positions{ Positions: &parser.Positions{
@ -189,7 +195,8 @@ func TestNewNodeListTokenPosition(t *testing.T) {
} }
func TestNewTokenNodeListPosition(t *testing.T) { func TestNewTokenNodeListPosition(t *testing.T) {
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 2) tpos := position.NewPosition(1, 1, 0, 2)
tkn := scanner.NewToken([]byte(`foo`), tpos)
n1 := node.NewIdentifier("test node") n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node") n2 := node.NewIdentifier("test node")
@ -255,8 +262,10 @@ func TestNewNodeNodeListPosition(t *testing.T) {
func TestNewOptionalListTokensPosition(t *testing.T) { func TestNewOptionalListTokensPosition(t *testing.T) {
builder := parser.PositionBuilder{} builder := parser.PositionBuilder{}
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) tpos1 := position.NewPosition(1, 1, 0, 3)
token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6) token1 := scanner.NewToken([]byte(`foo`), tpos1)
tpos2 := position.NewPosition(2, 2, 4, 6)
token2 := scanner.NewToken([]byte(`foo`), tpos2)
pos := builder.NewOptionalListTokensPosition(nil, token1, token2) pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
@ -293,8 +302,10 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
}, },
} }
token1 := scanner.NewToken([]byte(`foo`), 4, 4, 27, 29) tpos1 := position.NewPosition(4, 4, 27, 29)
token2 := scanner.NewToken([]byte(`foo`), 5, 5, 30, 32) token1 := scanner.NewToken([]byte(`foo`), tpos1)
tpos2 := position.NewPosition(5, 5, 30, 32)
token2 := scanner.NewToken([]byte(`foo`), tpos2)
pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2) pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2)
@ -335,7 +346,8 @@ func TestNilNodeListPos(t *testing.T) {
} }
func TestNilNodeListTokenPos(t *testing.T) { func TestNilNodeListTokenPos(t *testing.T) {
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) tpos := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken([]byte(`foo`), tpos)
builder := parser.PositionBuilder{} builder := parser.PositionBuilder{}
@ -368,7 +380,8 @@ func TestEmptyNodeListPos(t *testing.T) {
} }
func TestEmptyNodeListTokenPos(t *testing.T) { func TestEmptyNodeListTokenPos(t *testing.T) {
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) tpos := position.NewPosition(1, 1, 0, 3)
token1 := scanner.NewToken([]byte(`foo`), tpos)
builder := parser.PositionBuilder{} builder := parser.PositionBuilder{}

View File

@ -10,7 +10,7 @@ import (
"github.com/z7zmey/php-parser/scanner" "github.com/z7zmey/php-parser/scanner"
) )
func (lval *yySymType) Token(t scanner.Token) { func (lval *yySymType) Token(t *scanner.Token) {
lval.token = t lval.token = t
} }
@ -45,12 +45,12 @@ func NewParser(src io.Reader, path string) *Parser {
// Lex proxy to lexer Lex // Lex proxy to lexer Lex
func (l *Parser) Lex(lval *yySymType) int { func (l *Parser) Lex(lval *yySymType) int {
t := l.Lexer.Lex(lval) t := l.Lexer.Lex(lval)
l.lastToken = &lval.token l.lastToken = lval.token
return t return t
} }
func (l *Parser) Error(msg string) { func (l *Parser) Error(msg string) {
l.errors = append(l.errors, errors.NewError(msg, *l.lastToken)) l.errors = append(l.errors, errors.NewError(msg, l.lastToken))
} }
// Parse the php7 Parser entrypoint // Parse the php7 Parser entrypoint

View File

@ -23,7 +23,7 @@ import (
type yySymType struct { type yySymType struct {
yys int yys int
node node.Node node node.Node
token scanner.Token token *scanner.Token
boolWithToken boolWithToken boolWithToken boolWithToken
list []node.Node list []node.Node
foreachVariable foreachVariable foreachVariable foreachVariable
@ -345,6 +345,7 @@ const yyErrCode = 2
const yyInitialStackSize = 16 const yyInitialStackSize = 16
//line php5/php5.y:3825 //line php5/php5.y:3825
type foreachVariable struct { type foreachVariable struct {
node node.Node node node.Node
byRef bool byRef bool
@ -352,7 +353,7 @@ type foreachVariable struct {
type nodesWithEndToken struct { type nodesWithEndToken struct {
nodes []node.Node nodes []node.Node
endToken scanner.Token endToken *scanner.Token
} }
type boolWithToken struct { type boolWithToken struct {
@ -3132,7 +3133,7 @@ yydefault:
yyDollar = yyS[yypt-1 : yypt+1] yyDollar = yyS[yypt-1 : yypt+1]
//line php5/php5.y:910 //line php5/php5.y:910
{ {
yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token} yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
} }
case 91: case 91:
yyDollar = yyS[yypt-0 : yypt+1] yyDollar = yyS[yypt-0 : yypt+1]
@ -3144,7 +3145,7 @@ yydefault:
yyDollar = yyS[yypt-1 : yypt+1] yyDollar = yyS[yypt-1 : yypt+1]
//line php5/php5.y:917 //line php5/php5.y:917
{ {
yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token} yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
} }
case 93: case 93:
yyDollar = yyS[yypt-9 : yypt+1] yyDollar = yyS[yypt-9 : yypt+1]
@ -3557,10 +3558,10 @@ yydefault:
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewNodeTokenPosition(yyDollar[1].node, yyDollar[4].token)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewNodeTokenPosition(yyDollar[1].node, yyDollar[4].token))
yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node]) yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
} else if yyDollar[2].boolWithToken.value == true { } else if yyDollar[2].boolWithToken.value == true {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[2].boolWithToken.token, yyDollar[4].token)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[2].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments()) yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
} else if yyDollar[3].boolWithToken.value == true { } else if yyDollar[3].boolWithToken.value == true {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[3].boolWithToken.token, yyDollar[4].token)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[3].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments()) yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
} else { } else {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenPosition(yyDollar[4].token)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenPosition(yyDollar[4].token))
@ -3585,10 +3586,10 @@ yydefault:
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewNodesPosition(yyDollar[1].node, yyDollar[6].node)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewNodesPosition(yyDollar[1].node, yyDollar[6].node))
yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node]) yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
} else if yyDollar[2].boolWithToken.value == true { } else if yyDollar[2].boolWithToken.value == true {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[2].boolWithToken.token, yyDollar[6].node)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[2].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments()) yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
} else if yyDollar[3].boolWithToken.value == true { } else if yyDollar[3].boolWithToken.value == true {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[3].boolWithToken.token, yyDollar[6].node)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[3].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments()) yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
} else { } else {
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[4].token, yyDollar[6].node)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[4].token, yyDollar[6].node))

View File

@ -20,7 +20,7 @@ import (
%union{ %union{
node node.Node node node.Node
token scanner.Token token *scanner.Token
boolWithToken boolWithToken boolWithToken boolWithToken
list []node.Node list []node.Node
foreachVariable foreachVariable foreachVariable foreachVariable
@ -907,14 +907,14 @@ is_reference:
/* empty */ /* empty */
{ $$ = boolWithToken{false, nil} } { $$ = boolWithToken{false, nil} }
| '&' | '&'
{ $$ = boolWithToken{true, &$1} } { $$ = boolWithToken{true, $1} }
; ;
is_variadic: is_variadic:
/* empty */ /* empty */
{ $$ = boolWithToken{false, nil} } { $$ = boolWithToken{false, nil} }
| T_ELLIPSIS | T_ELLIPSIS
{ $$ = boolWithToken{true, &$1} } { $$ = boolWithToken{true, $1} }
; ;
unticked_function_declaration_statement: unticked_function_declaration_statement:
@ -1255,10 +1255,10 @@ parameter:
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewNodeTokenPosition($1, $4)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewNodeTokenPosition($1, $4))
yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1]) yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
} else if $2.value == true { } else if $2.value == true {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$2.token, $4)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($2.token, $4))
yylex.(*Parser).comments.AddComments($$, $2.token.Comments()) yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
} else if $3.value == true { } else if $3.value == true {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$3.token, $4)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($3.token, $4))
yylex.(*Parser).comments.AddComments($$, $3.token.Comments()) yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
} else { } else {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenPosition($4)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenPosition($4))
@ -1281,10 +1281,10 @@ parameter:
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewNodesPosition($1, $6)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewNodesPosition($1, $6))
yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1]) yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
} else if $2.value == true { } else if $2.value == true {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$2.token, $6)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($2.token, $6))
yylex.(*Parser).comments.AddComments($$, $2.token.Comments()) yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
} else if $3.value == true { } else if $3.value == true {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$3.token, $6)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($3.token, $6))
yylex.(*Parser).comments.AddComments($$, $3.token.Comments()) yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
} else { } else {
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($4, $6)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($4, $6))
@ -3831,7 +3831,7 @@ type foreachVariable struct {
type nodesWithEndToken struct { type nodesWithEndToken struct {
nodes []node.Node nodes []node.Node
endToken scanner.Token endToken *scanner.Token
} }
type boolWithToken struct { type boolWithToken struct {

View File

@ -10,7 +10,7 @@ import (
"github.com/z7zmey/php-parser/scanner" "github.com/z7zmey/php-parser/scanner"
) )
func (lval *yySymType) Token(t scanner.Token) { func (lval *yySymType) Token(t *scanner.Token) {
lval.token = t lval.token = t
} }
@ -45,12 +45,12 @@ func NewParser(src io.Reader, path string) *Parser {
// Lex proxy to lexer Lex // Lex proxy to lexer Lex
func (l *Parser) Lex(lval *yySymType) int { func (l *Parser) Lex(lval *yySymType) int {
t := l.Lexer.Lex(lval) t := l.Lexer.Lex(lval)
l.lastToken = &lval.token l.lastToken = lval.token
return t return t
} }
func (l *Parser) Error(msg string) { func (l *Parser) Error(msg string) {
l.errors = append(l.errors, errors.NewError(msg, *l.lastToken)) l.errors = append(l.errors, errors.NewError(msg, l.lastToken))
} }
// Parse the php7 Parser entrypoint // Parse the php7 Parser entrypoint

View File

@ -23,7 +23,7 @@ import (
type yySymType struct { type yySymType struct {
yys int yys int
node node.Node node node.Node
token scanner.Token token *scanner.Token
boolWithToken boolWithToken boolWithToken boolWithToken
list []node.Node list []node.Node
foreachVariable foreachVariable foreachVariable foreachVariable
@ -345,6 +345,7 @@ const yyErrCode = 2
const yyInitialStackSize = 16 const yyInitialStackSize = 16
//line php7/php7.y:2635 //line php7/php7.y:2635
type foreachVariable struct { type foreachVariable struct {
node node.Node node node.Node
byRef bool byRef bool
@ -352,7 +353,7 @@ type foreachVariable struct {
type nodesWithEndToken struct { type nodesWithEndToken struct {
nodes []node.Node nodes []node.Node
endToken scanner.Token endToken *scanner.Token
} }
type boolWithToken struct { type boolWithToken struct {
@ -3242,7 +3243,7 @@ yydefault:
yyDollar = yyS[yypt-1 : yypt+1] yyDollar = yyS[yypt-1 : yypt+1]
//line php7/php7.y:764 //line php7/php7.y:764
{ {
yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token} yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
} }
case 167: case 167:
yyDollar = yyS[yypt-0 : yypt+1] yyDollar = yyS[yypt-0 : yypt+1]
@ -3254,7 +3255,7 @@ yydefault:
yyDollar = yyS[yypt-1 : yypt+1] yyDollar = yyS[yypt-1 : yypt+1]
//line php7/php7.y:769 //line php7/php7.y:769
{ {
yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token} yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
} }
case 169: case 169:
yyDollar = yyS[yypt-9 : yypt+1] yyDollar = yyS[yypt-9 : yypt+1]
@ -3623,11 +3624,11 @@ yydefault:
yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node]) yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
} else if yyDollar[2].boolWithToken.value == true { } else if yyDollar[2].boolWithToken.value == true {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value) yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[2].boolWithToken.token, yyDollar[4].token)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[2].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments()) yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
} else if yyDollar[3].boolWithToken.value == true { } else if yyDollar[3].boolWithToken.value == true {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value) yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[3].boolWithToken.token, yyDollar[4].token)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[3].boolWithToken.token, yyDollar[4].token))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments()) yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
} else { } else {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value) yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
@ -3653,11 +3654,11 @@ yydefault:
yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node]) yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
} else if yyDollar[2].boolWithToken.value == true { } else if yyDollar[2].boolWithToken.value == true {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value) yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[2].boolWithToken.token, yyDollar[6].node)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[2].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments()) yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
} else if yyDollar[3].boolWithToken.value == true { } else if yyDollar[3].boolWithToken.value == true {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value) yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[3].boolWithToken.token, yyDollar[6].node)) yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[3].boolWithToken.token, yyDollar[6].node))
yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments()) yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
} else { } else {
yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value) yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
@ -4893,7 +4894,7 @@ yydefault:
yyDollar = yyS[yypt-1 : yypt+1] yyDollar = yyS[yypt-1 : yypt+1]
//line php7/php7.y:1963 //line php7/php7.y:1963
{ {
yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token} yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
} }
case 374: case 374:
yyDollar = yyS[yypt-0 : yypt+1] yyDollar = yyS[yypt-0 : yypt+1]

View File

@ -20,7 +20,7 @@ import (
%union{ %union{
node node.Node node node.Node
token scanner.Token token *scanner.Token
boolWithToken boolWithToken boolWithToken boolWithToken
list []node.Node list []node.Node
foreachVariable foreachVariable foreachVariable foreachVariable
@ -761,12 +761,12 @@ function_declaration_statement:
is_reference: is_reference:
/* empty */ { $$ = boolWithToken{false, nil} } /* empty */ { $$ = boolWithToken{false, nil} }
| '&' { $$ = boolWithToken{true, &$1} } | '&' { $$ = boolWithToken{true, $1} }
; ;
is_variadic: is_variadic:
/* empty */ { $$ = boolWithToken{false, nil} } /* empty */ { $$ = boolWithToken{false, nil} }
| T_ELLIPSIS { $$ = boolWithToken{true, &$1} } | T_ELLIPSIS { $$ = boolWithToken{true, $1} }
; ;
class_declaration_statement: class_declaration_statement:
@ -1044,11 +1044,11 @@ parameter:
yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1]) yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
} else if $2.value == true { } else if $2.value == true {
$$ = node.NewParameter($1, variable, nil, $2.value, $3.value) $$ = node.NewParameter($1, variable, nil, $2.value, $3.value)
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$2.token, $4)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($2.token, $4))
yylex.(*Parser).comments.AddComments($$, $2.token.Comments()) yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
} else if $3.value == true { } else if $3.value == true {
$$ = node.NewParameter($1, variable, nil, $2.value, $3.value) $$ = node.NewParameter($1, variable, nil, $2.value, $3.value)
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$3.token, $4)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($3.token, $4))
yylex.(*Parser).comments.AddComments($$, $3.token.Comments()) yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
} else { } else {
$$ = node.NewParameter($1, variable, nil, $2.value, $3.value) $$ = node.NewParameter($1, variable, nil, $2.value, $3.value)
@ -1072,11 +1072,11 @@ parameter:
yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1]) yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
} else if $2.value == true { } else if $2.value == true {
$$ = node.NewParameter($1, variable, $6, $2.value, $3.value) $$ = node.NewParameter($1, variable, $6, $2.value, $3.value)
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$2.token, $6)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($2.token, $6))
yylex.(*Parser).comments.AddComments($$, $2.token.Comments()) yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
} else if $3.value == true { } else if $3.value == true {
$$ = node.NewParameter($1, variable, $6, $2.value, $3.value) $$ = node.NewParameter($1, variable, $6, $2.value, $3.value)
yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$3.token, $6)) yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($3.token, $6))
yylex.(*Parser).comments.AddComments($$, $3.token.Comments()) yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
} else { } else {
$$ = node.NewParameter($1, variable, $6, $2.value, $3.value) $$ = node.NewParameter($1, variable, $6, $2.value, $3.value)
@ -1960,7 +1960,7 @@ backup_doc_comment:
returns_ref: returns_ref:
/* empty */ { $$ = boolWithToken{false, nil} } /* empty */ { $$ = boolWithToken{false, nil} }
| '&' { $$ = boolWithToken{true, &$1} } | '&' { $$ = boolWithToken{true, $1} }
; ;
lexical_vars: lexical_vars:
@ -2641,7 +2641,7 @@ type foreachVariable struct {
type nodesWithEndToken struct { type nodesWithEndToken struct {
nodes []node.Node nodes []node.Node
endToken scanner.Token endToken *scanner.Token
} }
type boolWithToken struct { type boolWithToken struct {

View File

@ -432,7 +432,7 @@ const T_POW = 57481
// Lval parsers yySymType must implement this interface // Lval parsers yySymType must implement this interface
type Lval interface { type Lval interface {
Token(tkn Token) Token(tkn *Token)
} }
// Lexer php lexer // Lexer php lexer
@ -510,16 +510,18 @@ func (l *Lexer) getCurrentState() int {
return l.StateStack[len(l.StateStack)-1] return l.StateStack[len(l.StateStack)-1]
} }
func (l *Lexer) newToken(chars []lex.Char) Token { func (l *Lexer) newToken(chars []lex.Char) *Token {
firstChar := chars[0] firstChar := chars[0]
lastChar := chars[len(chars)-1] lastChar := chars[len(chars)-1]
startLine := l.File.Line(firstChar.Pos()) pos := position.NewPosition(
endLine := l.File.Line(lastChar.Pos()) l.File.Line(firstChar.Pos()),
startPos := int(firstChar.Pos()) l.File.Line(lastChar.Pos()),
endPos := int(lastChar.Pos()) int(firstChar.Pos()),
int(lastChar.Pos()),
)
return NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments) return NewToken(l.charsToBytes(chars), pos).SetComments(l.Comments)
} }
func (l *Lexer) addComment(chars []lex.Char) { func (l *Lexer) addComment(chars []lex.Char) {

View File

@ -28,10 +28,10 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
} }
type lval struct { type lval struct {
Tkn scanner.Token Tkn *scanner.Token
} }
func (lv *lval) Token(t scanner.Token) { func (lv *lval) Token(t *scanner.Token) {
lv.Tkn = t lv.Tkn = t
} }

View File

@ -2,35 +2,41 @@ package scanner
import ( import (
"github.com/z7zmey/php-parser/comment" "github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/position"
) )
// Token value returned by lexer // Token value returned by lexer
type Token struct { type Token struct {
Value string Value string
StartLine int position *position.Position
EndLine int
StartPos int
EndPos int
comments []*comment.Comment comments []*comment.Comment
} }
// NewToken Token constructor // NewToken Token constructor
// TODO: return pointer func NewToken(value []byte, pos *position.Position) *Token {
func NewToken(value []byte, startLine int, endLine int, startPos int, endPos int) Token { return &Token{
return Token{string(value), startLine, endLine, startPos, endPos, nil} Value: string(value),
position: pos,
comments: nil,
}
} }
func (t Token) String() string { func (t *Token) String() string {
return string(t.Value) return string(t.Value)
} }
// Position returns token position
func (t *Token) Position() *position.Position {
return t.position
}
// Comments returns attached comments // Comments returns attached comments
func (t Token) Comments() []*comment.Comment { func (t *Token) Comments() []*comment.Comment {
return t.comments return t.comments
} }
// SetComments attach comments // SetComments attach comments
func (t Token) SetComments(comments []*comment.Comment) Token { func (t *Token) SetComments(comments []*comment.Comment) *Token {
t.comments = comments t.comments = comments
return t return t
} }

View File

@ -4,13 +4,16 @@ import (
"reflect" "reflect"
"testing" "testing"
"github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/comment" "github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/scanner" "github.com/z7zmey/php-parser/scanner"
) )
func TestToken(t *testing.T) { func TestToken(t *testing.T) {
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) pos := position.NewPosition(1, 1, 0, 3)
tkn := scanner.NewToken([]byte(`foo`), pos)
c := []*comment.Comment{ c := []*comment.Comment{
comment.NewComment("test comment", nil), comment.NewComment("test comment", nil),
@ -18,7 +21,7 @@ func TestToken(t *testing.T) {
tkn.SetComments(c) tkn.SetComments(c)
if reflect.DeepEqual(tkn.Comments(), c) { if !reflect.DeepEqual(tkn.Comments(), c) {
t.Errorf("comments are not equal\n") t.Errorf("comments are not equal\n")
} }
@ -26,7 +29,7 @@ func TestToken(t *testing.T) {
t.Errorf("token value is not equal\n") t.Errorf("token value is not equal\n")
} }
if tkn.StartLine != 1 || tkn.EndLine != 1 || tkn.StartPos != 0 || tkn.EndPos != 3 { if tkn.Position() != pos {
t.Errorf("token position is not equal\n") t.Errorf("token position is not equal\n")
} }
} }