scanner.NewToken returns pointer, and scanner.Token saves position as position.Position

scanner/lexer.go

@@ -432,7 +432,7 @@ const T_POW = 57481
 
 // Lval parsers yySymType must implement this interface
 type Lval interface {
-	Token(tkn Token)
+	Token(tkn *Token)
 }
 
 // Lexer php lexer
@@ -510,16 +510,18 @@ func (l *Lexer) getCurrentState() int {
 	return l.StateStack[len(l.StateStack)-1]
 }
 
-func (l *Lexer) newToken(chars []lex.Char) Token {
+func (l *Lexer) newToken(chars []lex.Char) *Token {
 	firstChar := chars[0]
 	lastChar := chars[len(chars)-1]
 
-	startLine := l.File.Line(firstChar.Pos())
-	endLine := l.File.Line(lastChar.Pos())
-	startPos := int(firstChar.Pos())
-	endPos := int(lastChar.Pos())
+	pos := position.NewPosition(
+		l.File.Line(firstChar.Pos()),
+		l.File.Line(lastChar.Pos()),
+		int(firstChar.Pos()),
+		int(lastChar.Pos()),
+	)
 
-	return NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments)
+	return NewToken(l.charsToBytes(chars), pos).SetComments(l.Comments)
 }
 
 func (l *Lexer) addComment(chars []lex.Char) {

scanner/scanner_test.go

@@ -28,10 +28,10 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
 }
 
 type lval struct {
-	Tkn scanner.Token
+	Tkn *scanner.Token
 }
 
-func (lv *lval) Token(t scanner.Token) {
+func (lv *lval) Token(t *scanner.Token) {
 	lv.Tkn = t
 }
 

scanner/token.go

@@ -2,35 +2,41 @@ package scanner
 
 import (
 	"github.com/z7zmey/php-parser/comment"
+	"github.com/z7zmey/php-parser/position"
 )
 
 // Token value returned by lexer
 type Token struct {
-	Value     string
-	StartLine int
-	EndLine   int
-	StartPos  int
-	EndPos    int
-	comments  []*comment.Comment
+	Value    string
+	position *position.Position
+	comments []*comment.Comment
 }
 
 // NewToken Token constructor
-// TODO: return pointer
-func NewToken(value []byte, startLine int, endLine int, startPos int, endPos int) Token {
-	return Token{string(value), startLine, endLine, startPos, endPos, nil}
+func NewToken(value []byte, pos *position.Position) *Token {
+	return &Token{
+		Value:    string(value),
+		position: pos,
+		comments: nil,
+	}
 }
 
-func (t Token) String() string {
+func (t *Token) String() string {
 	return string(t.Value)
 }
 
+// Position returns token position
+func (t *Token) Position() *position.Position {
+	return t.position
+}
+
 // Comments returns attached comments
-func (t Token) Comments() []*comment.Comment {
+func (t *Token) Comments() []*comment.Comment {
 	return t.comments
 }
 
 // SetComments attach comments
-func (t Token) SetComments(comments []*comment.Comment) Token {
+func (t *Token) SetComments(comments []*comment.Comment) *Token {
 	t.comments = comments
 	return t
 }

scanner/token_test.go

@@ -4,13 +4,16 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/z7zmey/php-parser/position"
+
 	"github.com/z7zmey/php-parser/comment"
 
 	"github.com/z7zmey/php-parser/scanner"
 )
 
 func TestToken(t *testing.T) {
-	tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
+	pos := position.NewPosition(1, 1, 0, 3)
+	tkn := scanner.NewToken([]byte(`foo`), pos)
 
 	c := []*comment.Comment{
 		comment.NewComment("test comment", nil),
@@ -18,7 +21,7 @@ func TestToken(t *testing.T) {
 
 	tkn.SetComments(c)
 
-	if reflect.DeepEqual(tkn.Comments(), c) {
+	if !reflect.DeepEqual(tkn.Comments(), c) {
 		t.Errorf("comments are not equal\n")
 	}
 
@@ -26,7 +29,7 @@ func TestToken(t *testing.T) {
 		t.Errorf("token value is not equal\n")
 	}
 
-	if tkn.StartLine != 1 || tkn.EndLine != 1 || tkn.StartPos != 0 || tkn.EndPos != 3 {
+	if tkn.Position() != pos {
 		t.Errorf("token position is not equal\n")
 	}
 }