diff --git a/errors/error.go b/errors/error.go index 06ac931..0ac2023 100644 --- a/errors/error.go +++ b/errors/error.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/z7zmey/php-parser/position" - "github.com/z7zmey/php-parser/token" + "github.com/z7zmey/php-parser/scanner" ) // Error parsing error @@ -14,7 +14,7 @@ type Error struct { } // NewError creates and returns new Error -func NewError(msg string, t token.Token) *Error { +func NewError(msg string, t scanner.Token) *Error { return &Error{ Msg: msg, Pos: position.Position{ diff --git a/errors/error_test.go b/errors/error_test.go index 2892658..f5b1a47 100644 --- a/errors/error_test.go +++ b/errors/error_test.go @@ -7,7 +7,7 @@ import ( "github.com/z7zmey/php-parser/position" "github.com/z7zmey/php-parser/errors" - "github.com/z7zmey/php-parser/token" + "github.com/z7zmey/php-parser/scanner" "github.com/kylelemons/godebug/pretty" ) @@ -26,7 +26,7 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) { } func TestConstructor(t *testing.T) { - token := token.Token{ + token := scanner.Token{ Value: "test", StartLine: 1, EndLine: 2, @@ -50,7 +50,7 @@ func TestConstructor(t *testing.T) { } func TestPrint(t *testing.T) { - token := token.Token{ + token := scanner.Token{ Value: "test", StartLine: 1, EndLine: 2, diff --git a/php5/parser.go b/php5/parser.go index ffb9cc6..5df3681 100644 --- a/php5/parser.go +++ b/php5/parser.go @@ -8,10 +8,9 @@ import ( "github.com/z7zmey/php-parser/node" "github.com/z7zmey/php-parser/position" "github.com/z7zmey/php-parser/scanner" - "github.com/z7zmey/php-parser/token" ) -func (lval *yySymType) Token(t token.Token) { +func (lval *yySymType) Token(t scanner.Token) { lval.token = t } @@ -19,7 +18,7 @@ func (lval *yySymType) Token(t token.Token) { type Parser struct { *scanner.Lexer path string - lastToken *token.Token + lastToken *scanner.Token positionBuilder *position.Builder errors []*errors.Error rootNode node.Node diff --git a/php5/php5.go b/php5/php5.go index c248a3f..523c6f8 100644 --- a/php5/php5.go +++ b/php5/php5.go @@ -16,14 +16,14 @@ import ( "github.com/z7zmey/php-parser/node/name" "github.com/z7zmey/php-parser/node/scalar" "github.com/z7zmey/php-parser/node/stmt" - "github.com/z7zmey/php-parser/token" + "github.com/z7zmey/php-parser/scanner" ) //line php5/php5.y:21 type yySymType struct { yys int node node.Node - token token.Token + token scanner.Token boolWithToken boolWithToken list []node.Node foreachVariable foreachVariable @@ -345,7 +345,6 @@ const yyErrCode = 2 const yyInitialStackSize = 16 //line php5/php5.y:3825 - type foreachVariable struct { node node.Node byRef bool @@ -353,12 +352,12 @@ type foreachVariable struct { type nodesWithEndToken struct { nodes []node.Node - endToken token.Token + endToken scanner.Token } type boolWithToken struct { value bool - token *token.Token + token *scanner.Token } type simpleIndirectReference struct { diff --git a/php5/php5.y b/php5/php5.y index f79ac9a..4a81af3 100644 --- a/php5/php5.y +++ b/php5/php5.y @@ -5,7 +5,7 @@ import ( "strings" "strconv" - "github.com/z7zmey/php-parser/token" + "github.com/z7zmey/php-parser/scanner" "github.com/z7zmey/php-parser/node" "github.com/z7zmey/php-parser/node/scalar" "github.com/z7zmey/php-parser/node/name" @@ -20,7 +20,7 @@ import ( %union{ node node.Node - token token.Token + token scanner.Token boolWithToken boolWithToken list []node.Node foreachVariable foreachVariable @@ -3831,12 +3831,12 @@ type foreachVariable struct { type nodesWithEndToken struct { nodes []node.Node - endToken token.Token + endToken scanner.Token } type boolWithToken struct { value bool - token *token.Token + token *scanner.Token } type simpleIndirectReference struct { diff --git a/php7/parser.go b/php7/parser.go index a8d3bd2..355189b 100644 --- a/php7/parser.go +++ b/php7/parser.go @@ -8,10 +8,9 @@ import ( "github.com/z7zmey/php-parser/node" "github.com/z7zmey/php-parser/position" "github.com/z7zmey/php-parser/scanner" - "github.com/z7zmey/php-parser/token" ) -func (lval *yySymType) Token(t token.Token) { +func (lval *yySymType) Token(t scanner.Token) { lval.token = t } @@ -19,7 +18,7 @@ func (lval *yySymType) Token(t token.Token) { type Parser struct { *scanner.Lexer path string - lastToken *token.Token + lastToken *scanner.Token positionBuilder *position.Builder errors []*errors.Error rootNode node.Node diff --git a/php7/php7.go b/php7/php7.go index 891ad31..ee9c5e7 100644 --- a/php7/php7.go +++ b/php7/php7.go @@ -16,14 +16,14 @@ import ( "github.com/z7zmey/php-parser/node/name" "github.com/z7zmey/php-parser/node/scalar" "github.com/z7zmey/php-parser/node/stmt" - "github.com/z7zmey/php-parser/token" + "github.com/z7zmey/php-parser/scanner" ) //line php7/php7.y:21 type yySymType struct { yys int node node.Node - token token.Token + token scanner.Token boolWithToken boolWithToken list []node.Node foreachVariable foreachVariable @@ -345,7 +345,6 @@ const yyErrCode = 2 const yyInitialStackSize = 16 //line php7/php7.y:2635 - type foreachVariable struct { node node.Node byRef bool @@ -353,12 +352,12 @@ type foreachVariable struct { type nodesWithEndToken struct { nodes []node.Node - endToken token.Token + endToken scanner.Token } type boolWithToken struct { value bool - token *token.Token + token *scanner.Token } type altSyntaxNode struct { diff --git a/php7/php7.y b/php7/php7.y index 4b7b0f9..478ec97 100644 --- a/php7/php7.y +++ b/php7/php7.y @@ -5,7 +5,7 @@ import ( "strings" "strconv" - "github.com/z7zmey/php-parser/token" + "github.com/z7zmey/php-parser/scanner" "github.com/z7zmey/php-parser/node" "github.com/z7zmey/php-parser/node/scalar" "github.com/z7zmey/php-parser/node/name" @@ -20,7 +20,7 @@ import ( %union{ node node.Node - token token.Token + token scanner.Token boolWithToken boolWithToken list []node.Node foreachVariable foreachVariable @@ -2641,12 +2641,12 @@ type foreachVariable struct { type nodesWithEndToken struct { nodes []node.Node - endToken token.Token + endToken scanner.Token } type boolWithToken struct { value bool - token *token.Token + token *scanner.Token } type altSyntaxNode struct { diff --git a/position/builder.go b/position/builder.go index 2c2f990..1a98c2f 100644 --- a/position/builder.go +++ b/position/builder.go @@ -2,7 +2,7 @@ package position import ( "github.com/z7zmey/php-parser/node" - "github.com/z7zmey/php-parser/token" + "github.com/z7zmey/php-parser/scanner" ) // Builder provide functions to constuct positions @@ -99,7 +99,7 @@ func (b *Builder) NewNodePosition(n node.Node) *Position { } // NewTokenPosition returns new Position -func (b *Builder) NewTokenPosition(t token.Token) *Position { +func (b *Builder) NewTokenPosition(t scanner.Token) *Position { return &Position{ t.StartLine, t.EndLine, @@ -109,7 +109,7 @@ func (b *Builder) NewTokenPosition(t token.Token) *Position { } // NewTokensPosition returns new Position -func (b *Builder) NewTokensPosition(startToken token.Token, endToken token.Token) *Position { +func (b *Builder) NewTokensPosition(startToken scanner.Token, endToken scanner.Token) *Position { return &Position{ startToken.StartLine, endToken.EndLine, @@ -119,7 +119,7 @@ func (b *Builder) NewTokensPosition(startToken token.Token, endToken token.Token } // NewTokenNodePosition returns new Position -func (b *Builder) NewTokenNodePosition(t token.Token, n node.Node) *Position { +func (b *Builder) NewTokenNodePosition(t scanner.Token, n node.Node) *Position { return &Position{ t.StartLine, b.getNodeEndPos(n).endLine, @@ -129,7 +129,7 @@ func (b *Builder) NewTokenNodePosition(t token.Token, n node.Node) *Position { } // NewNodeTokenPosition returns new Position -func (b *Builder) NewNodeTokenPosition(n node.Node, t token.Token) *Position { +func (b *Builder) NewNodeTokenPosition(n node.Node, t scanner.Token) *Position { return &Position{ b.getNodeStartPos(n).startLine, t.EndLine, @@ -149,7 +149,7 @@ func (b *Builder) NewNodesPosition(startNode node.Node, endNode node.Node) *Posi } // NewNodeListTokenPosition returns new Position -func (b *Builder) NewNodeListTokenPosition(list []node.Node, t token.Token) *Position { +func (b *Builder) NewNodeListTokenPosition(list []node.Node, t scanner.Token) *Position { return &Position{ b.getListStartPos(list).startLine, t.EndLine, @@ -159,7 +159,7 @@ func (b *Builder) NewNodeListTokenPosition(list []node.Node, t token.Token) *Pos } // NewTokenNodeListPosition returns new Position -func (b *Builder) NewTokenNodeListPosition(t token.Token, list []node.Node) *Position { +func (b *Builder) NewTokenNodeListPosition(t scanner.Token, list []node.Node) *Position { return &Position{ t.StartLine, b.getListEndPos(list).endLine, @@ -179,7 +179,7 @@ func (b *Builder) NewNodeNodeListPosition(n node.Node, list []node.Node) *Positi } // NewOptionalListTokensPosition returns new Position -func (b *Builder) NewOptionalListTokensPosition(list []node.Node, t token.Token, endToken token.Token) *Position { +func (b *Builder) NewOptionalListTokensPosition(list []node.Node, t scanner.Token, endToken scanner.Token) *Position { if list == nil { return &Position{ t.StartLine, diff --git a/position/position_test.go b/position/position_test.go index 7cbc878..01f1985 100644 --- a/position/position_test.go +++ b/position/position_test.go @@ -6,13 +6,13 @@ import ( "github.com/z7zmey/php-parser/node" "github.com/z7zmey/php-parser/position" - "github.com/z7zmey/php-parser/token" + "github.com/z7zmey/php-parser/scanner" ) func TestNewTokenPosition(t *testing.T) { builder := position.Builder{} - tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) + tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) pos := builder.NewTokenPosition(tkn) @@ -24,8 +24,8 @@ func TestNewTokenPosition(t *testing.T) { func TestNewTokensPosition(t *testing.T) { builder := position.Builder{} - token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) - token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6) + token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) + token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6) pos := builder.NewTokensPosition(token1, token2) @@ -57,7 +57,7 @@ func TestNewNodePosition(t *testing.T) { } func TestNewTokenNodePosition(t *testing.T) { - tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) + tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) n := node.NewIdentifier("test node") p := &position.Positions{} @@ -81,7 +81,7 @@ func TestNewTokenNodePosition(t *testing.T) { func TestNewNodeTokenPosition(t *testing.T) { n := node.NewIdentifier("test node") - tkn := token.NewToken([]byte(`foo`), 2, 2, 10, 12) + tkn := scanner.NewToken([]byte(`foo`), 2, 2, 10, 12) p := &position.Positions{} p.AddPosition(n, &position.Position{ @@ -161,7 +161,7 @@ func TestNewNodesPosition(t *testing.T) { func TestNewNodeListTokenPosition(t *testing.T) { n1 := node.NewIdentifier("test node") n2 := node.NewIdentifier("test node") - tkn := token.NewToken([]byte(`foo`), 3, 3, 20, 22) + tkn := scanner.NewToken([]byte(`foo`), 3, 3, 20, 22) builder := position.Builder{ Positions: &position.Positions{ @@ -188,7 +188,7 @@ func TestNewNodeListTokenPosition(t *testing.T) { } func TestNewTokenNodeListPosition(t *testing.T) { - tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 2) + tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 2) n1 := node.NewIdentifier("test node") n2 := node.NewIdentifier("test node") @@ -254,8 +254,8 @@ func TestNewNodeNodeListPosition(t *testing.T) { func TestNewOptionalListTokensPosition(t *testing.T) { builder := position.Builder{} - token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) - token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6) + token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) + token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6) pos := builder.NewOptionalListTokensPosition(nil, token1, token2) @@ -292,8 +292,8 @@ func TestNewOptionalListTokensPosition2(t *testing.T) { }, } - token1 := token.NewToken([]byte(`foo`), 4, 4, 27, 29) - token2 := token.NewToken([]byte(`foo`), 5, 5, 30, 32) + token1 := scanner.NewToken([]byte(`foo`), 4, 4, 27, 29) + token2 := scanner.NewToken([]byte(`foo`), 5, 5, 30, 32) pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2) @@ -334,7 +334,7 @@ func TestNilNodeListPos(t *testing.T) { } func TestNilNodeListTokenPos(t *testing.T) { - token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) + token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) builder := position.Builder{} @@ -367,7 +367,7 @@ func TestEmptyNodeListPos(t *testing.T) { } func TestEmptyNodeListTokenPos(t *testing.T) { - token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) + token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) builder := position.Builder{} diff --git a/scanner/lexer.go b/scanner/lexer.go index 0ddf176..d3b3b03 100644 --- a/scanner/lexer.go +++ b/scanner/lexer.go @@ -10,7 +10,6 @@ import ( "github.com/cznic/golex/lex" "github.com/z7zmey/php-parser/comment" - t "github.com/z7zmey/php-parser/token" ) // Allocate Character classes anywhere in [0x80, 0xFF]. @@ -431,7 +430,7 @@ const T_POW = 57481 // Lval parsers yySymType must implement this interface type Lval interface { - Token(tkn t.Token) + Token(tkn Token) } // Lexer php lexer @@ -509,7 +508,7 @@ func (l *Lexer) getCurrentState() int { return l.StateStack[len(l.StateStack)-1] } -func (l *Lexer) newToken(chars []lex.Char) t.Token { +func (l *Lexer) newToken(chars []lex.Char) Token { firstChar := chars[0] lastChar := chars[len(chars)-1] @@ -518,7 +517,7 @@ func (l *Lexer) newToken(chars []lex.Char) t.Token { startPos := int(firstChar.Pos()) endPos := int(lastChar.Pos()) - return t.NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments) + return NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments) } func (l *Lexer) addComment(c comment.Comment) { diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 12b1bbd..6beed3c 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -8,7 +8,6 @@ import ( "github.com/z7zmey/php-parser/comment" "github.com/z7zmey/php-parser/scanner" - "github.com/z7zmey/php-parser/token" "github.com/kylelemons/godebug/pretty" ) @@ -27,10 +26,10 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) { } type lval struct { - Tkn token.Token + Tkn scanner.Token } -func (lv *lval) Token(t token.Token) { +func (lv *lval) Token(t scanner.Token) { lv.Tkn = t } diff --git a/token/token.go b/scanner/token.go similarity index 97% rename from token/token.go rename to scanner/token.go index 0da6189..1154d6f 100644 --- a/token/token.go +++ b/scanner/token.go @@ -1,4 +1,4 @@ -package token +package scanner import ( "github.com/z7zmey/php-parser/comment" diff --git a/token/token_test.go b/scanner/token_test.go similarity index 81% rename from token/token_test.go rename to scanner/token_test.go index 5f61365..84173ca 100644 --- a/token/token_test.go +++ b/scanner/token_test.go @@ -1,4 +1,4 @@ -package token_test +package scanner_test import ( "reflect" @@ -6,11 +6,11 @@ import ( "github.com/z7zmey/php-parser/comment" - "github.com/z7zmey/php-parser/token" + "github.com/z7zmey/php-parser/scanner" ) func TestToken(t *testing.T) { - tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) + tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3) c := []comment.Comment{ comment.NewPlainComment("test comment"),