scanner.NewToken returns pointer, and scanner.Token saves position as position.Position

errors/error.go

@@ -10,19 +10,14 @@ import (
 // Error parsing error
 type Error struct {
 	Msg string
-	Pos position.Position
+	Pos *position.Position
 }
 
 // NewError creates and returns new Error
-func NewError(msg string, t scanner.Token) *Error {
+func NewError(msg string, t *scanner.Token) *Error {
 	return &Error{
 		Msg: msg,
-		Pos: position.Position{
+		Pos: t.Position(),
-			StartLine: t.StartLine,
-			EndLine:   t.EndLine,
-			StartPos:  t.StartPos,
-			EndPos:    t.EndPos,
-		},
 	}
 }
 

errors/error_test.go

@@ -26,37 +26,22 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
 }
 
 func TestConstructor(t *testing.T) {
-	token := scanner.Token{
+	pos := position.NewPosition(1, 2, 3, 4)
-		Value:     "test",
+	token := scanner.NewToken([]byte(`test`), pos)
-		StartLine: 1,
-		EndLine:   2,
-		StartPos:  3,
-		EndPos:    4,
-	}
 
 	actual := errors.NewError("message", token)
 
 	expected := &errors.Error{
 		Msg: "message",
-		Pos: position.Position{
+		Pos: pos,
-			StartLine: 1,
-			EndLine:   2,
-			StartPos:  3,
-			EndPos:    4,
-		},
 	}
 
 	assertEqual(t, expected, actual)
 }
 
 func TestPrint(t *testing.T) {
-	token := scanner.Token{
+	pos := position.NewPosition(1, 2, 3, 4)
-		Value:     "test",
+	token := scanner.NewToken([]byte(`test`), pos)
-		StartLine: 1,
-		EndLine:   2,
-		StartPos:  3,
-		EndPos:    4,
-	}
 
 	Error := errors.NewError("message", token)
 

parser/position_builder.go

@@ -100,42 +100,37 @@ func (b *PositionBuilder) NewNodePosition(n node.Node) *position.Position {
 }
 
 // NewTokenPosition returns new Position
-func (b *PositionBuilder) NewTokenPosition(t scanner.Token) *position.Position {
+func (b *PositionBuilder) NewTokenPosition(t *scanner.Token) *position.Position {
-	return position.NewPosition(
+	return t.Position()
-		t.StartLine,
-		t.EndLine,
-		t.StartPos,
-		t.EndPos,
-	)
 }
 
 // NewTokensPosition returns new Position
-func (b *PositionBuilder) NewTokensPosition(startToken scanner.Token, endToken scanner.Token) *position.Position {
+func (b *PositionBuilder) NewTokensPosition(startToken *scanner.Token, endToken *scanner.Token) *position.Position {
 	return position.NewPosition(
-		startToken.StartLine,
+		startToken.Position().StartLine,
-		endToken.EndLine,
+		endToken.Position().EndLine,
-		startToken.StartPos,
+		startToken.Position().StartPos,
-		endToken.EndPos,
+		endToken.Position().EndPos,
 	)
 }
 
 // NewTokenNodePosition returns new Position
-func (b *PositionBuilder) NewTokenNodePosition(t scanner.Token, n node.Node) *position.Position {
+func (b *PositionBuilder) NewTokenNodePosition(t *scanner.Token, n node.Node) *position.Position {
 	return position.NewPosition(
-		t.StartLine,
+		t.Position().StartLine,
 		b.getNodeEndPos(n).endLine,
-		t.StartPos,
+		t.Position().StartPos,
 		b.getNodeEndPos(n).endPos,
 	)
 }
 
 // NewNodeTokenPosition returns new Position
-func (b *PositionBuilder) NewNodeTokenPosition(n node.Node, t scanner.Token) *position.Position {
+func (b *PositionBuilder) NewNodeTokenPosition(n node.Node, t *scanner.Token) *position.Position {
 	return position.NewPosition(
 		b.getNodeStartPos(n).startLine,
-		t.EndLine,
+		t.Position().EndLine,
 		b.getNodeStartPos(n).startPos,
-		t.EndPos,
+		t.Position().EndPos,
 	)
 }
 
@@ -150,21 +145,21 @@ func (b *PositionBuilder) NewNodesPosition(startNode node.Node, endNode node.Nod
 }
 
 // NewNodeListTokenPosition returns new Position
-func (b *PositionBuilder) NewNodeListTokenPosition(list []node.Node, t scanner.Token) *position.Position {
+func (b *PositionBuilder) NewNodeListTokenPosition(list []node.Node, t *scanner.Token) *position.Position {
 	return position.NewPosition(
 		b.getListStartPos(list).startLine,
-		t.EndLine,
+		t.Position().EndLine,
 		b.getListStartPos(list).startPos,
-		t.EndPos,
+		t.Position().EndPos,
 	)
 }
 
 // NewTokenNodeListPosition returns new Position
-func (b *PositionBuilder) NewTokenNodeListPosition(t scanner.Token, list []node.Node) *position.Position {
+func (b *PositionBuilder) NewTokenNodeListPosition(t *scanner.Token, list []node.Node) *position.Position {
 	return position.NewPosition(
-		t.StartLine,
+		t.Position().StartLine,
 		b.getListEndPos(list).endLine,
-		t.StartPos,
+		t.Position().StartPos,
 		b.getListEndPos(list).endPos,
 	)
 }
@@ -180,20 +175,20 @@ func (b *PositionBuilder) NewNodeNodeListPosition(n node.Node, list []node.Node)
 }
 
 // NewOptionalListTokensPosition returns new Position
-func (b *PositionBuilder) NewOptionalListTokensPosition(list []node.Node, t scanner.Token, endToken scanner.Token) *position.Position {
+func (b *PositionBuilder) NewOptionalListTokensPosition(list []node.Node, t *scanner.Token, endToken *scanner.Token) *position.Position {
 	if list == nil {
 		return position.NewPosition(
-			t.StartLine,
+			t.Position().StartLine,
-			endToken.EndLine,
+			endToken.Position().EndLine,
-			t.StartPos,
+			t.Position().StartPos,
-			endToken.EndPos,
+			endToken.Position().EndPos,
 		)
 	}
 
 	return position.NewPosition(
 		b.getListStartPos(list).startLine,
-		endToken.EndLine,
+		endToken.Position().EndLine,
 		b.getListStartPos(list).startPos,
-		endToken.EndPos,
+		endToken.Position().EndPos,
 	)
 }

parser/position_builder_test.go

@@ -13,7 +13,8 @@ import (
 func TestNewTokenPosition(t *testing.T) {
 	builder := parser.PositionBuilder{}
 
-	tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
+	tpos := position.NewPosition(1, 1, 0, 3)
+	tkn := scanner.NewToken([]byte(`foo`), tpos)
 
 	pos := builder.NewTokenPosition(tkn)
 
@@ -25,8 +26,10 @@ func TestNewTokenPosition(t *testing.T) {
 func TestNewTokensPosition(t *testing.T) {
 	builder := parser.PositionBuilder{}
 
-	token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
+	tpos1 := position.NewPosition(1, 1, 0, 3)
-	token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6)
+	token1 := scanner.NewToken([]byte(`foo`), tpos1)
+	tpos2 := position.NewPosition(2, 2, 4, 6)
+	token2 := scanner.NewToken([]byte(`foo`), tpos2)
 
 	pos := builder.NewTokensPosition(token1, token2)
 
@@ -58,7 +61,8 @@ func TestNewNodePosition(t *testing.T) {
 }
 
 func TestNewTokenNodePosition(t *testing.T) {
-	tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
+	tpos := position.NewPosition(1, 1, 0, 3)
+	tkn := scanner.NewToken([]byte(`foo`), tpos)
 	n := node.NewIdentifier("test node")
 
 	p := &parser.Positions{}
@@ -82,7 +86,8 @@ func TestNewTokenNodePosition(t *testing.T) {
 
 func TestNewNodeTokenPosition(t *testing.T) {
 	n := node.NewIdentifier("test node")
-	tkn := scanner.NewToken([]byte(`foo`), 2, 2, 10, 12)
+	tpos := position.NewPosition(2, 2, 10, 12)
+	tkn := scanner.NewToken([]byte(`foo`), tpos)
 
 	p := &parser.Positions{}
 	p.AddPosition(n, &position.Position{
@@ -162,7 +167,8 @@ func TestNewNodesPosition(t *testing.T) {
 func TestNewNodeListTokenPosition(t *testing.T) {
 	n1 := node.NewIdentifier("test node")
 	n2 := node.NewIdentifier("test node")
-	tkn := scanner.NewToken([]byte(`foo`), 3, 3, 20, 22)
+	tpos := position.NewPosition(3, 3, 20, 22)
+	tkn := scanner.NewToken([]byte(`foo`), tpos)
 
 	builder := parser.PositionBuilder{
 		Positions: &parser.Positions{
@@ -189,7 +195,8 @@ func TestNewNodeListTokenPosition(t *testing.T) {
 }
 
 func TestNewTokenNodeListPosition(t *testing.T) {
-	tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 2)
+	tpos := position.NewPosition(1, 1, 0, 2)
+	tkn := scanner.NewToken([]byte(`foo`), tpos)
 	n1 := node.NewIdentifier("test node")
 	n2 := node.NewIdentifier("test node")
 
@@ -255,8 +262,10 @@ func TestNewNodeNodeListPosition(t *testing.T) {
 func TestNewOptionalListTokensPosition(t *testing.T) {
 	builder := parser.PositionBuilder{}
 
-	token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
+	tpos1 := position.NewPosition(1, 1, 0, 3)
-	token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6)
+	token1 := scanner.NewToken([]byte(`foo`), tpos1)
+	tpos2 := position.NewPosition(2, 2, 4, 6)
+	token2 := scanner.NewToken([]byte(`foo`), tpos2)
 
 	pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
 
@@ -293,8 +302,10 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
 		},
 	}
 
-	token1 := scanner.NewToken([]byte(`foo`), 4, 4, 27, 29)
+	tpos1 := position.NewPosition(4, 4, 27, 29)
-	token2 := scanner.NewToken([]byte(`foo`), 5, 5, 30, 32)
+	token1 := scanner.NewToken([]byte(`foo`), tpos1)
+	tpos2 := position.NewPosition(5, 5, 30, 32)
+	token2 := scanner.NewToken([]byte(`foo`), tpos2)
 
 	pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2)
 
@@ -335,7 +346,8 @@ func TestNilNodeListPos(t *testing.T) {
 }
 
 func TestNilNodeListTokenPos(t *testing.T) {
-	token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
+	tpos := position.NewPosition(1, 1, 0, 3)
+	token1 := scanner.NewToken([]byte(`foo`), tpos)
 
 	builder := parser.PositionBuilder{}
 
@@ -368,7 +380,8 @@ func TestEmptyNodeListPos(t *testing.T) {
 }
 
 func TestEmptyNodeListTokenPos(t *testing.T) {
-	token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
+	tpos := position.NewPosition(1, 1, 0, 3)
+	token1 := scanner.NewToken([]byte(`foo`), tpos)
 
 	builder := parser.PositionBuilder{}
 

php5/parser.go

@@ -10,7 +10,7 @@ import (
 	"github.com/z7zmey/php-parser/scanner"
 )
 
-func (lval *yySymType) Token(t scanner.Token) {
+func (lval *yySymType) Token(t *scanner.Token) {
 	lval.token = t
 }
 
@@ -45,12 +45,12 @@ func NewParser(src io.Reader, path string) *Parser {
 // Lex proxy to lexer Lex
 func (l *Parser) Lex(lval *yySymType) int {
 	t := l.Lexer.Lex(lval)
-	l.lastToken = &lval.token
+	l.lastToken = lval.token
 	return t
 }
 
 func (l *Parser) Error(msg string) {
-	l.errors = append(l.errors, errors.NewError(msg, *l.lastToken))
+	l.errors = append(l.errors, errors.NewError(msg, l.lastToken))
 }
 
 // Parse the php7 Parser entrypoint

php5/php5.go

@@ -23,7 +23,7 @@ import (
 type yySymType struct {
 	yys                     int
 	node                    node.Node
-	token                   scanner.Token
+	token                   *scanner.Token
 	boolWithToken           boolWithToken
 	list                    []node.Node
 	foreachVariable         foreachVariable
@@ -345,6 +345,7 @@ const yyErrCode = 2
 const yyInitialStackSize = 16
 
 //line php5/php5.y:3825
+
 type foreachVariable struct {
 	node  node.Node
 	byRef bool
@@ -352,7 +353,7 @@ type foreachVariable struct {
 
 type nodesWithEndToken struct {
 	nodes    []node.Node
-	endToken scanner.Token
+	endToken *scanner.Token
 }
 
 type boolWithToken struct {
@@ -3132,7 +3133,7 @@ yydefault:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		//line php5/php5.y:910
 		{
-			yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token}
+			yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
 		}
 	case 91:
 		yyDollar = yyS[yypt-0 : yypt+1]
@@ -3144,7 +3145,7 @@ yydefault:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		//line php5/php5.y:917
 		{
-			yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token}
+			yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
 		}
 	case 93:
 		yyDollar = yyS[yypt-9 : yypt+1]
@@ -3557,10 +3558,10 @@ yydefault:
 				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewNodeTokenPosition(yyDollar[1].node, yyDollar[4].token))
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
 			} else if yyDollar[2].boolWithToken.value == true {
-				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[2].boolWithToken.token, yyDollar[4].token))
+				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[2].boolWithToken.token, yyDollar[4].token))
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
 			} else if yyDollar[3].boolWithToken.value == true {
-				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[3].boolWithToken.token, yyDollar[4].token))
+				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[3].boolWithToken.token, yyDollar[4].token))
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
 			} else {
 				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenPosition(yyDollar[4].token))
@@ -3585,10 +3586,10 @@ yydefault:
 				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewNodesPosition(yyDollar[1].node, yyDollar[6].node))
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
 			} else if yyDollar[2].boolWithToken.value == true {
-				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[2].boolWithToken.token, yyDollar[6].node))
+				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[2].boolWithToken.token, yyDollar[6].node))
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
 			} else if yyDollar[3].boolWithToken.value == true {
-				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[3].boolWithToken.token, yyDollar[6].node))
+				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[3].boolWithToken.token, yyDollar[6].node))
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
 			} else {
 				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[4].token, yyDollar[6].node))

php5/php5.y

@@ -20,7 +20,7 @@ import (
 
 %union{
     node node.Node
-    token scanner.Token
+    token *scanner.Token
     boolWithToken boolWithToken
     list []node.Node
     foreachVariable foreachVariable
@@ -907,14 +907,14 @@ is_reference:
         /* empty */
             { $$ = boolWithToken{false, nil} }
     |   '&'
-            { $$ = boolWithToken{true, &$1} }
+            { $$ = boolWithToken{true, $1} }
 ;
 
 is_variadic:
         /* empty */
             { $$ = boolWithToken{false, nil} }
     |   T_ELLIPSIS
-            { $$ = boolWithToken{true, &$1} }
+            { $$ = boolWithToken{true, $1} }
 ;
 
 unticked_function_declaration_statement:
@@ -1255,10 +1255,10 @@ parameter:
                     yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewNodeTokenPosition($1, $4))
                     yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
                 } else if $2.value == true {
-                    yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$2.token, $4))
+                    yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($2.token, $4))
                     yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
                 } else if $3.value == true {
-                    yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$3.token, $4))
+                    yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($3.token, $4))
                     yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
                 } else {
                     yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenPosition($4))
@@ -1281,10 +1281,10 @@ parameter:
                     yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewNodesPosition($1, $6))
                     yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
                 } else if $2.value == true {
-                    yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$2.token, $6))
+                    yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($2.token, $6))
                     yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
                 } else if $3.value == true {
-                    yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$3.token, $6))
+                    yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($3.token, $6))
                     yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
                 } else {
                     yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($4, $6))
@@ -3831,7 +3831,7 @@ type foreachVariable struct {
 
 type nodesWithEndToken struct {
 	nodes    []node.Node
-	endToken scanner.Token
+	endToken *scanner.Token
 }
 
 type boolWithToken struct {

php7/parser.go

@@ -10,7 +10,7 @@ import (
 	"github.com/z7zmey/php-parser/scanner"
 )
 
-func (lval *yySymType) Token(t scanner.Token) {
+func (lval *yySymType) Token(t *scanner.Token) {
 	lval.token = t
 }
 
@@ -45,12 +45,12 @@ func NewParser(src io.Reader, path string) *Parser {
 // Lex proxy to lexer Lex
 func (l *Parser) Lex(lval *yySymType) int {
 	t := l.Lexer.Lex(lval)
-	l.lastToken = &lval.token
+	l.lastToken = lval.token
 	return t
 }
 
 func (l *Parser) Error(msg string) {
-	l.errors = append(l.errors, errors.NewError(msg, *l.lastToken))
+	l.errors = append(l.errors, errors.NewError(msg, l.lastToken))
 }
 
 // Parse the php7 Parser entrypoint

php7/php7.go

@@ -23,7 +23,7 @@ import (
 type yySymType struct {
 	yys               int
 	node              node.Node
-	token             scanner.Token
+	token             *scanner.Token
 	boolWithToken     boolWithToken
 	list              []node.Node
 	foreachVariable   foreachVariable
@@ -345,6 +345,7 @@ const yyErrCode = 2
 const yyInitialStackSize = 16
 
 //line php7/php7.y:2635
+
 type foreachVariable struct {
 	node  node.Node
 	byRef bool
@@ -352,7 +353,7 @@ type foreachVariable struct {
 
 type nodesWithEndToken struct {
 	nodes    []node.Node
-	endToken scanner.Token
+	endToken *scanner.Token
 }
 
 type boolWithToken struct {
@@ -3242,7 +3243,7 @@ yydefault:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		//line php7/php7.y:764
 		{
-			yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token}
+			yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
 		}
 	case 167:
 		yyDollar = yyS[yypt-0 : yypt+1]
@@ -3254,7 +3255,7 @@ yydefault:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		//line php7/php7.y:769
 		{
-			yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token}
+			yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
 		}
 	case 169:
 		yyDollar = yyS[yypt-9 : yypt+1]
@@ -3623,11 +3624,11 @@ yydefault:
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
 			} else if yyDollar[2].boolWithToken.value == true {
 				yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
-				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[2].boolWithToken.token, yyDollar[4].token))
+				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[2].boolWithToken.token, yyDollar[4].token))
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
 			} else if yyDollar[3].boolWithToken.value == true {
 				yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
-				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(*yyDollar[3].boolWithToken.token, yyDollar[4].token))
+				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokensPosition(yyDollar[3].boolWithToken.token, yyDollar[4].token))
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
 			} else {
 				yyVAL.node = node.NewParameter(yyDollar[1].node, variable, nil, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
@@ -3653,11 +3654,11 @@ yydefault:
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yylex.(*Parser).comments[yyDollar[1].node])
 			} else if yyDollar[2].boolWithToken.value == true {
 				yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
-				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[2].boolWithToken.token, yyDollar[6].node))
+				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[2].boolWithToken.token, yyDollar[6].node))
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[2].boolWithToken.token.Comments())
 			} else if yyDollar[3].boolWithToken.value == true {
 				yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
-				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*yyDollar[3].boolWithToken.token, yyDollar[6].node))
+				yylex.(*Parser).positions.AddPosition(yyVAL.node, yylex.(*Parser).positionBuilder.NewTokenNodePosition(yyDollar[3].boolWithToken.token, yyDollar[6].node))
 				yylex.(*Parser).comments.AddComments(yyVAL.node, yyDollar[3].boolWithToken.token.Comments())
 			} else {
 				yyVAL.node = node.NewParameter(yyDollar[1].node, variable, yyDollar[6].node, yyDollar[2].boolWithToken.value, yyDollar[3].boolWithToken.value)
@@ -4893,7 +4894,7 @@ yydefault:
 		yyDollar = yyS[yypt-1 : yypt+1]
 		//line php7/php7.y:1963
 		{
-			yyVAL.boolWithToken = boolWithToken{true, &yyDollar[1].token}
+			yyVAL.boolWithToken = boolWithToken{true, yyDollar[1].token}
 		}
 	case 374:
 		yyDollar = yyS[yypt-0 : yypt+1]

php7/php7.y

@@ -20,7 +20,7 @@ import (
 
 %union{
     node node.Node
-    token scanner.Token
+    token *scanner.Token
     boolWithToken boolWithToken
     list []node.Node
     foreachVariable foreachVariable
@@ -761,12 +761,12 @@ function_declaration_statement:
 
 is_reference:
         /* empty */                                     { $$ = boolWithToken{false, nil} }
-    |   '&'                                             { $$ = boolWithToken{true, &$1} }
+    |   '&'                                             { $$ = boolWithToken{true, $1} }
 ;
 
 is_variadic:
         /* empty */                                     { $$ = boolWithToken{false, nil} }
-    |   T_ELLIPSIS                                      { $$ = boolWithToken{true, &$1} }
+    |   T_ELLIPSIS                                      { $$ = boolWithToken{true, $1} }
 ;
 
 class_declaration_statement:
@@ -1044,11 +1044,11 @@ parameter:
                 yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
             } else if $2.value == true {
                 $$ = node.NewParameter($1, variable, nil, $2.value, $3.value)
-                yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$2.token, $4))
+                yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($2.token, $4))
                 yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
             } else if $3.value == true {
                 $$ = node.NewParameter($1, variable, nil, $2.value, $3.value)
-                yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition(*$3.token, $4))
+                yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokensPosition($3.token, $4))
                 yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
             } else {
                 $$ = node.NewParameter($1, variable, nil, $2.value, $3.value)
@@ -1072,11 +1072,11 @@ parameter:
                 yylex.(*Parser).comments.AddComments($$, yylex.(*Parser).comments[$1])
             } else if $2.value == true {
                 $$ = node.NewParameter($1, variable, $6, $2.value, $3.value)
-                yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$2.token, $6))
+                yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($2.token, $6))
                 yylex.(*Parser).comments.AddComments($$, $2.token.Comments())
             } else if $3.value == true {
                 $$ = node.NewParameter($1, variable, $6, $2.value, $3.value)
-                yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition(*$3.token, $6))
+                yylex.(*Parser).positions.AddPosition($$, yylex.(*Parser).positionBuilder.NewTokenNodePosition($3.token, $6))
                 yylex.(*Parser).comments.AddComments($$, $3.token.Comments())
             } else {
                 $$ = node.NewParameter($1, variable, $6, $2.value, $3.value)
@@ -1960,7 +1960,7 @@ backup_doc_comment:
 
 returns_ref:
         /* empty */                                     { $$ = boolWithToken{false, nil} }
-    |   '&'                                             { $$ = boolWithToken{true, &$1} }
+    |   '&'                                             { $$ = boolWithToken{true, $1} }
 ;
 
 lexical_vars:
@@ -2641,7 +2641,7 @@ type foreachVariable struct {
 
 type nodesWithEndToken struct {
 	nodes    []node.Node
-	endToken scanner.Token
+	endToken *scanner.Token
 }
 
 type boolWithToken struct {

scanner/lexer.go

@@ -432,7 +432,7 @@ const T_POW = 57481
 
 // Lval parsers yySymType must implement this interface
 type Lval interface {
-	Token(tkn Token)
+	Token(tkn *Token)
 }
 
 // Lexer php lexer
@@ -510,16 +510,18 @@ func (l *Lexer) getCurrentState() int {
 	return l.StateStack[len(l.StateStack)-1]
 }
 
-func (l *Lexer) newToken(chars []lex.Char) Token {
+func (l *Lexer) newToken(chars []lex.Char) *Token {
 	firstChar := chars[0]
 	lastChar := chars[len(chars)-1]
 
-	startLine := l.File.Line(firstChar.Pos())
+	pos := position.NewPosition(
-	endLine := l.File.Line(lastChar.Pos())
+		l.File.Line(firstChar.Pos()),
-	startPos := int(firstChar.Pos())
+		l.File.Line(lastChar.Pos()),
-	endPos := int(lastChar.Pos())
+		int(firstChar.Pos()),
+		int(lastChar.Pos()),
+	)
 
-	return NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments)
+	return NewToken(l.charsToBytes(chars), pos).SetComments(l.Comments)
 }
 
 func (l *Lexer) addComment(chars []lex.Char) {

scanner/scanner_test.go

@@ -28,10 +28,10 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
 }
 
 type lval struct {
-	Tkn scanner.Token
+	Tkn *scanner.Token
 }
 
-func (lv *lval) Token(t scanner.Token) {
+func (lv *lval) Token(t *scanner.Token) {
 	lv.Tkn = t
 }
 

scanner/token.go

@@ -2,35 +2,41 @@ package scanner
 
 import (
 	"github.com/z7zmey/php-parser/comment"
+	"github.com/z7zmey/php-parser/position"
 )
 
 // Token value returned by lexer
 type Token struct {
 	Value    string
-	StartLine int
+	position *position.Position
-	EndLine   int
-	StartPos  int
-	EndPos    int
 	comments []*comment.Comment
 }
 
 // NewToken Token constructor
-// TODO: return pointer
+func NewToken(value []byte, pos *position.Position) *Token {
-func NewToken(value []byte, startLine int, endLine int, startPos int, endPos int) Token {
+	return &Token{
-	return Token{string(value), startLine, endLine, startPos, endPos, nil}
+		Value:    string(value),
+		position: pos,
+		comments: nil,
+	}
 }
 
-func (t Token) String() string {
+func (t *Token) String() string {
 	return string(t.Value)
 }
 
+// Position returns token position
+func (t *Token) Position() *position.Position {
+	return t.position
+}
+
 // Comments returns attached comments
-func (t Token) Comments() []*comment.Comment {
+func (t *Token) Comments() []*comment.Comment {
 	return t.comments
 }
 
 // SetComments attach comments
-func (t Token) SetComments(comments []*comment.Comment) Token {
+func (t *Token) SetComments(comments []*comment.Comment) *Token {
 	t.comments = comments
 	return t
 }

scanner/token_test.go

@@ -4,13 +4,16 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/z7zmey/php-parser/position"
+
 	"github.com/z7zmey/php-parser/comment"
 
 	"github.com/z7zmey/php-parser/scanner"
 )
 
 func TestToken(t *testing.T) {
-	tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
+	pos := position.NewPosition(1, 1, 0, 3)
+	tkn := scanner.NewToken([]byte(`foo`), pos)
 
 	c := []*comment.Comment{
 		comment.NewComment("test comment", nil),
@@ -18,7 +21,7 @@ func TestToken(t *testing.T) {
 
 	tkn.SetComments(c)
 
-	if reflect.DeepEqual(tkn.Comments(), c) {
+	if !reflect.DeepEqual(tkn.Comments(), c) {
 		t.Errorf("comments are not equal\n")
 	}
 
@@ -26,7 +29,7 @@ func TestToken(t *testing.T) {
 		t.Errorf("token value is not equal\n")
 	}
 
-	if tkn.StartLine != 1 || tkn.EndLine != 1 || tkn.StartPos != 0 || tkn.EndPos != 3 {
+	if tkn.Position() != pos {
 		t.Errorf("token position is not equal\n")
 	}
 }