maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge token package into scanner package

Browse Source
This commit is contained in:
z7zmey 2018-04-15 14:47:40 +03:00
parent 983c721e83
commit e65ace8984
14 changed files with 56 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
errors/error.go
View File

@ -4,7 +4,7 @@ import (
"fmt" "fmt"
"github.com/z7zmey/php-parser/position" "github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/token" "github.com/z7zmey/php-parser/scanner"
) )
// Error parsing error // Error parsing error
@ -14,7 +14,7 @@ type Error struct {
} }
// NewError creates and returns new Error // NewError creates and returns new Error
func NewError(msg string, t token.Token) *Error { func NewError(msg string, t scanner.Token) *Error {
return &Error{ return &Error{
Msg: msg, Msg: msg,
Pos: position.Position{ Pos: position.Position{

6
errors/error_test.go
View File

@ -7,7 +7,7 @@ import (
"github.com/z7zmey/php-parser/position" "github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/errors" "github.com/z7zmey/php-parser/errors"
"github.com/z7zmey/php-parser/token" "github.com/z7zmey/php-parser/scanner"
"github.com/kylelemons/godebug/pretty" "github.com/kylelemons/godebug/pretty"
) )
@ -26,7 +26,7 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
} }
func TestConstructor(t *testing.T) { func TestConstructor(t *testing.T) {
token := token.Token{ token := scanner.Token{
Value: "test", Value: "test",
StartLine: 1, StartLine: 1,
EndLine: 2, EndLine: 2,
@ -50,7 +50,7 @@ func TestConstructor(t *testing.T) {
} }
func TestPrint(t *testing.T) { func TestPrint(t *testing.T) {
token := token.Token{ token := scanner.Token{
Value: "test", Value: "test",
StartLine: 1, StartLine: 1,
EndLine: 2, EndLine: 2,

5
php5/parser.go
View File

@ -8,10 +8,9 @@ import (
"github.com/z7zmey/php-parser/node" "github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/position" "github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/scanner" "github.com/z7zmey/php-parser/scanner"
"github.com/z7zmey/php-parser/token"
) )
func (lval *yySymType) Token(t token.Token) { func (lval *yySymType) Token(t scanner.Token) {
lval.token = t lval.token = t
} }
@ -19,7 +18,7 @@ func (lval *yySymType) Token(t token.Token) {
type Parser struct { type Parser struct {
*scanner.Lexer *scanner.Lexer
path string path string
lastToken *token.Token lastToken *scanner.Token
positionBuilder *position.Builder positionBuilder *position.Builder
errors []*errors.Error errors []*errors.Error
rootNode node.Node rootNode node.Node

9
php5/php5.go
View File

@ -16,14 +16,14 @@ import (
"github.com/z7zmey/php-parser/node/name" "github.com/z7zmey/php-parser/node/name"
"github.com/z7zmey/php-parser/node/scalar" "github.com/z7zmey/php-parser/node/scalar"
"github.com/z7zmey/php-parser/node/stmt" "github.com/z7zmey/php-parser/node/stmt"
"github.com/z7zmey/php-parser/token" "github.com/z7zmey/php-parser/scanner"
) )
//line php5/php5.y:21 //line php5/php5.y:21
type yySymType struct { type yySymType struct {
yys int yys int
node node.Node node node.Node
token token.Token token scanner.Token
boolWithToken boolWithToken boolWithToken boolWithToken
list []node.Node list []node.Node
foreachVariable foreachVariable foreachVariable foreachVariable
@ -345,7 +345,6 @@ const yyErrCode = 2
const yyInitialStackSize = 16 const yyInitialStackSize = 16
//line php5/php5.y:3825 //line php5/php5.y:3825
type foreachVariable struct { type foreachVariable struct {
node node.Node node node.Node
byRef bool byRef bool
@ -353,12 +352,12 @@ type foreachVariable struct {
type nodesWithEndToken struct { type nodesWithEndToken struct {
nodes []node.Node nodes []node.Node
endToken token.Token endToken scanner.Token
} }
type boolWithToken struct { type boolWithToken struct {
value bool value bool
token *token.Token token *scanner.Token
} }
type simpleIndirectReference struct { type simpleIndirectReference struct {

8
php5/php5.y
View File

@ -5,7 +5,7 @@ import (
"strings" "strings"
"strconv" "strconv"
"github.com/z7zmey/php-parser/token" "github.com/z7zmey/php-parser/scanner"
"github.com/z7zmey/php-parser/node" "github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/node/scalar" "github.com/z7zmey/php-parser/node/scalar"
"github.com/z7zmey/php-parser/node/name" "github.com/z7zmey/php-parser/node/name"
@ -20,7 +20,7 @@ import (
%union{ %union{
node node.Node node node.Node
token token.Token token scanner.Token
boolWithToken boolWithToken boolWithToken boolWithToken
list []node.Node list []node.Node
foreachVariable foreachVariable foreachVariable foreachVariable
@ -3831,12 +3831,12 @@ type foreachVariable struct {
type nodesWithEndToken struct { type nodesWithEndToken struct {
nodes []node.Node nodes []node.Node
endToken token.Token endToken scanner.Token
} }
type boolWithToken struct { type boolWithToken struct {
value bool value bool
token *token.Token token *scanner.Token
} }
type simpleIndirectReference struct { type simpleIndirectReference struct {

5
php7/parser.go
View File

@ -8,10 +8,9 @@ import (
"github.com/z7zmey/php-parser/node" "github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/position" "github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/scanner" "github.com/z7zmey/php-parser/scanner"
"github.com/z7zmey/php-parser/token"
) )
func (lval *yySymType) Token(t token.Token) { func (lval *yySymType) Token(t scanner.Token) {
lval.token = t lval.token = t
} }
@ -19,7 +18,7 @@ func (lval *yySymType) Token(t token.Token) {
type Parser struct { type Parser struct {
*scanner.Lexer *scanner.Lexer
path string path string
lastToken *token.Token lastToken *scanner.Token
positionBuilder *position.Builder positionBuilder *position.Builder
errors []*errors.Error errors []*errors.Error
rootNode node.Node rootNode node.Node

9
php7/php7.go
View File

@ -16,14 +16,14 @@ import (
"github.com/z7zmey/php-parser/node/name" "github.com/z7zmey/php-parser/node/name"
"github.com/z7zmey/php-parser/node/scalar" "github.com/z7zmey/php-parser/node/scalar"
"github.com/z7zmey/php-parser/node/stmt" "github.com/z7zmey/php-parser/node/stmt"
"github.com/z7zmey/php-parser/token" "github.com/z7zmey/php-parser/scanner"
) )
//line php7/php7.y:21 //line php7/php7.y:21
type yySymType struct { type yySymType struct {
yys int yys int
node node.Node node node.Node
token token.Token token scanner.Token
boolWithToken boolWithToken boolWithToken boolWithToken
list []node.Node list []node.Node
foreachVariable foreachVariable foreachVariable foreachVariable
@ -345,7 +345,6 @@ const yyErrCode = 2
const yyInitialStackSize = 16 const yyInitialStackSize = 16
//line php7/php7.y:2635 //line php7/php7.y:2635
type foreachVariable struct { type foreachVariable struct {
node node.Node node node.Node
byRef bool byRef bool
@ -353,12 +352,12 @@ type foreachVariable struct {
type nodesWithEndToken struct { type nodesWithEndToken struct {
nodes []node.Node nodes []node.Node
endToken token.Token endToken scanner.Token
} }
type boolWithToken struct { type boolWithToken struct {
value bool value bool
token *token.Token token *scanner.Token
} }
type altSyntaxNode struct { type altSyntaxNode struct {

8
php7/php7.y
View File

@ -5,7 +5,7 @@ import (
"strings" "strings"
"strconv" "strconv"
"github.com/z7zmey/php-parser/token" "github.com/z7zmey/php-parser/scanner"
"github.com/z7zmey/php-parser/node" "github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/node/scalar" "github.com/z7zmey/php-parser/node/scalar"
"github.com/z7zmey/php-parser/node/name" "github.com/z7zmey/php-parser/node/name"
@ -20,7 +20,7 @@ import (
%union{ %union{
node node.Node node node.Node
token token.Token token scanner.Token
boolWithToken boolWithToken boolWithToken boolWithToken
list []node.Node list []node.Node
foreachVariable foreachVariable foreachVariable foreachVariable
@ -2641,12 +2641,12 @@ type foreachVariable struct {
type nodesWithEndToken struct { type nodesWithEndToken struct {
nodes []node.Node nodes []node.Node
endToken token.Token endToken scanner.Token
} }
type boolWithToken struct { type boolWithToken struct {
value bool value bool
token *token.Token token *scanner.Token
} }
type altSyntaxNode struct { type altSyntaxNode struct {

16
position/builder.go
View File

@ -2,7 +2,7 @@ package position
import ( import (
"github.com/z7zmey/php-parser/node" "github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/token" "github.com/z7zmey/php-parser/scanner"
) )
// Builder provide functions to constuct positions // Builder provide functions to constuct positions
@ -99,7 +99,7 @@ func (b *Builder) NewNodePosition(n node.Node) *Position {
} }
// NewTokenPosition returns new Position // NewTokenPosition returns new Position
func (b *Builder) NewTokenPosition(t token.Token) *Position { func (b *Builder) NewTokenPosition(t scanner.Token) *Position {
return &Position{ return &Position{
t.StartLine, t.StartLine,
t.EndLine, t.EndLine,
@ -109,7 +109,7 @@ func (b *Builder) NewTokenPosition(t token.Token) *Position {
} }
// NewTokensPosition returns new Position // NewTokensPosition returns new Position
func (b *Builder) NewTokensPosition(startToken token.Token, endToken token.Token) *Position { func (b *Builder) NewTokensPosition(startToken scanner.Token, endToken scanner.Token) *Position {
return &Position{ return &Position{
startToken.StartLine, startToken.StartLine,
endToken.EndLine, endToken.EndLine,
@ -119,7 +119,7 @@ func (b *Builder) NewTokensPosition(startToken token.Token, endToken token.Token
} }
// NewTokenNodePosition returns new Position // NewTokenNodePosition returns new Position
func (b *Builder) NewTokenNodePosition(t token.Token, n node.Node) *Position { func (b *Builder) NewTokenNodePosition(t scanner.Token, n node.Node) *Position {
return &Position{ return &Position{
t.StartLine, t.StartLine,
b.getNodeEndPos(n).endLine, b.getNodeEndPos(n).endLine,
@ -129,7 +129,7 @@ func (b *Builder) NewTokenNodePosition(t token.Token, n node.Node) *Position {
} }
// NewNodeTokenPosition returns new Position // NewNodeTokenPosition returns new Position
func (b *Builder) NewNodeTokenPosition(n node.Node, t token.Token) *Position { func (b *Builder) NewNodeTokenPosition(n node.Node, t scanner.Token) *Position {
return &Position{ return &Position{
b.getNodeStartPos(n).startLine, b.getNodeStartPos(n).startLine,
t.EndLine, t.EndLine,
@ -149,7 +149,7 @@ func (b *Builder) NewNodesPosition(startNode node.Node, endNode node.Node) *Posi
} }
// NewNodeListTokenPosition returns new Position // NewNodeListTokenPosition returns new Position
func (b *Builder) NewNodeListTokenPosition(list []node.Node, t token.Token) *Position { func (b *Builder) NewNodeListTokenPosition(list []node.Node, t scanner.Token) *Position {
return &Position{ return &Position{
b.getListStartPos(list).startLine, b.getListStartPos(list).startLine,
t.EndLine, t.EndLine,
@ -159,7 +159,7 @@ func (b *Builder) NewNodeListTokenPosition(list []node.Node, t token.Token) *Pos
} }
// NewTokenNodeListPosition returns new Position // NewTokenNodeListPosition returns new Position
func (b *Builder) NewTokenNodeListPosition(t token.Token, list []node.Node) *Position { func (b *Builder) NewTokenNodeListPosition(t scanner.Token, list []node.Node) *Position {
return &Position{ return &Position{
t.StartLine, t.StartLine,
b.getListEndPos(list).endLine, b.getListEndPos(list).endLine,
@ -179,7 +179,7 @@ func (b *Builder) NewNodeNodeListPosition(n node.Node, list []node.Node) *Positi
} }
// NewOptionalListTokensPosition returns new Position // NewOptionalListTokensPosition returns new Position
func (b *Builder) NewOptionalListTokensPosition(list []node.Node, t token.Token, endToken token.Token) *Position { func (b *Builder) NewOptionalListTokensPosition(list []node.Node, t scanner.Token, endToken scanner.Token) *Position {
if list == nil { if list == nil {
return &Position{ return &Position{
t.StartLine, t.StartLine,

28
position/position_test.go
View File

@ -6,13 +6,13 @@ import (
"github.com/z7zmey/php-parser/node" "github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/position" "github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/token" "github.com/z7zmey/php-parser/scanner"
) )
func TestNewTokenPosition(t *testing.T) { func TestNewTokenPosition(t *testing.T) {
builder := position.Builder{} builder := position.Builder{}
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
pos := builder.NewTokenPosition(tkn) pos := builder.NewTokenPosition(tkn)
@ -24,8 +24,8 @@ func TestNewTokenPosition(t *testing.T) {
func TestNewTokensPosition(t *testing.T) { func TestNewTokensPosition(t *testing.T) {
builder := position.Builder{} builder := position.Builder{}
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6) token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6)
pos := builder.NewTokensPosition(token1, token2) pos := builder.NewTokensPosition(token1, token2)
@ -57,7 +57,7 @@ func TestNewNodePosition(t *testing.T) {
} }
func TestNewTokenNodePosition(t *testing.T) { func TestNewTokenNodePosition(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
n := node.NewIdentifier("test node") n := node.NewIdentifier("test node")
p := &position.Positions{} p := &position.Positions{}
@ -81,7 +81,7 @@ func TestNewTokenNodePosition(t *testing.T) {
func TestNewNodeTokenPosition(t *testing.T) { func TestNewNodeTokenPosition(t *testing.T) {
n := node.NewIdentifier("test node") n := node.NewIdentifier("test node")
tkn := token.NewToken([]byte(`foo`), 2, 2, 10, 12) tkn := scanner.NewToken([]byte(`foo`), 2, 2, 10, 12)
p := &position.Positions{} p := &position.Positions{}
p.AddPosition(n, &position.Position{ p.AddPosition(n, &position.Position{
@ -161,7 +161,7 @@ func TestNewNodesPosition(t *testing.T) {
func TestNewNodeListTokenPosition(t *testing.T) { func TestNewNodeListTokenPosition(t *testing.T) {
n1 := node.NewIdentifier("test node") n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node") n2 := node.NewIdentifier("test node")
tkn := token.NewToken([]byte(`foo`), 3, 3, 20, 22) tkn := scanner.NewToken([]byte(`foo`), 3, 3, 20, 22)
builder := position.Builder{ builder := position.Builder{
Positions: &position.Positions{ Positions: &position.Positions{
@ -188,7 +188,7 @@ func TestNewNodeListTokenPosition(t *testing.T) {
} }
func TestNewTokenNodeListPosition(t *testing.T) { func TestNewTokenNodeListPosition(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 2) tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 2)
n1 := node.NewIdentifier("test node") n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node") n2 := node.NewIdentifier("test node")
@ -254,8 +254,8 @@ func TestNewNodeNodeListPosition(t *testing.T) {
func TestNewOptionalListTokensPosition(t *testing.T) { func TestNewOptionalListTokensPosition(t *testing.T) {
builder := position.Builder{} builder := position.Builder{}
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6) token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6)
pos := builder.NewOptionalListTokensPosition(nil, token1, token2) pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
@ -292,8 +292,8 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
}, },
} }
token1 := token.NewToken([]byte(`foo`), 4, 4, 27, 29) token1 := scanner.NewToken([]byte(`foo`), 4, 4, 27, 29)
token2 := token.NewToken([]byte(`foo`), 5, 5, 30, 32) token2 := scanner.NewToken([]byte(`foo`), 5, 5, 30, 32)
pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2) pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2)
@ -334,7 +334,7 @@ func TestNilNodeListPos(t *testing.T) {
} }
func TestNilNodeListTokenPos(t *testing.T) { func TestNilNodeListTokenPos(t *testing.T) {
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
builder := position.Builder{} builder := position.Builder{}
@ -367,7 +367,7 @@ func TestEmptyNodeListPos(t *testing.T) {
} }
func TestEmptyNodeListTokenPos(t *testing.T) { func TestEmptyNodeListTokenPos(t *testing.T) {
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3) token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
builder := position.Builder{} builder := position.Builder{}

7
scanner/lexer.go
View File

@ -10,7 +10,6 @@ import (
"github.com/cznic/golex/lex" "github.com/cznic/golex/lex"
"github.com/z7zmey/php-parser/comment" "github.com/z7zmey/php-parser/comment"
t "github.com/z7zmey/php-parser/token"
) )
// Allocate Character classes anywhere in [0x80, 0xFF]. // Allocate Character classes anywhere in [0x80, 0xFF].
@ -431,7 +430,7 @@ const T_POW = 57481
// Lval parsers yySymType must implement this interface // Lval parsers yySymType must implement this interface
type Lval interface { type Lval interface {
Token(tkn t.Token) Token(tkn Token)
} }
// Lexer php lexer // Lexer php lexer
@ -509,7 +508,7 @@ func (l *Lexer) getCurrentState() int {
return l.StateStack[len(l.StateStack)-1] return l.StateStack[len(l.StateStack)-1]
} }
func (l *Lexer) newToken(chars []lex.Char) t.Token { func (l *Lexer) newToken(chars []lex.Char) Token {
firstChar := chars[0] firstChar := chars[0]
lastChar := chars[len(chars)-1] lastChar := chars[len(chars)-1]
@ -518,7 +517,7 @@ func (l *Lexer) newToken(chars []lex.Char) t.Token {
startPos := int(firstChar.Pos()) startPos := int(firstChar.Pos())
endPos := int(lastChar.Pos()) endPos := int(lastChar.Pos())
return t.NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments) return NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments)
} }
func (l *Lexer) addComment(c comment.Comment) { func (l *Lexer) addComment(c comment.Comment) {

5
scanner/scanner_test.go
View File

@ -8,7 +8,6 @@ import (
"github.com/z7zmey/php-parser/comment" "github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/scanner" "github.com/z7zmey/php-parser/scanner"
"github.com/z7zmey/php-parser/token"
"github.com/kylelemons/godebug/pretty" "github.com/kylelemons/godebug/pretty"
) )
@ -27,10 +26,10 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
} }
type lval struct { type lval struct {
Tkn token.Token Tkn scanner.Token
} }
func (lv *lval) Token(t token.Token) { func (lv *lval) Token(t scanner.Token) {
lv.Tkn = t lv.Tkn = t
} }

2
token/token.go → scanner/token.go
View File

@ -1,4 +1,4 @@
package token package scanner
import ( import (
"github.com/z7zmey/php-parser/comment" "github.com/z7zmey/php-parser/comment"

6
token/token_test.go → scanner/token_test.go
View File

@ -1,4 +1,4 @@
package token_test package scanner_test
import ( import (
"reflect" "reflect"
@ -6,11 +6,11 @@ import (
"github.com/z7zmey/php-parser/comment" "github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/token" "github.com/z7zmey/php-parser/scanner"
) )
func TestToken(t *testing.T) { func TestToken(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3) tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
c := []comment.Comment{ c := []comment.Comment{
comment.NewPlainComment("test comment"), comment.NewPlainComment("test comment"),