maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge token package into scanner package

Browse Source
This commit is contained in:
z7zmey 2018-04-15 14:47:40 +03:00
parent 983c721e83
commit e65ace8984
14 changed files with 56 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
errors/error.go
View File

@ -4,7 +4,7 @@ import (
"fmt"
"github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/token"
"github.com/z7zmey/php-parser/scanner"
)
// Error parsing error
@ -14,7 +14,7 @@ type Error struct {
}
// NewError creates and returns new Error
func NewError(msg string, t token.Token) *Error {
func NewError(msg string, t scanner.Token) *Error {
return &Error{
Msg: msg,
Pos: position.Position{

6
errors/error_test.go
View File

@ -7,7 +7,7 @@ import (
"github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/errors"
"github.com/z7zmey/php-parser/token"
"github.com/z7zmey/php-parser/scanner"
"github.com/kylelemons/godebug/pretty"
)
@ -26,7 +26,7 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
}
func TestConstructor(t *testing.T) {
token := token.Token{
token := scanner.Token{
Value: "test",
StartLine: 1,
EndLine: 2,
@ -50,7 +50,7 @@ func TestConstructor(t *testing.T) {
}
func TestPrint(t *testing.T) {
token := token.Token{
token := scanner.Token{
Value: "test",
StartLine: 1,
EndLine: 2,

5
php5/parser.go
View File

@ -8,10 +8,9 @@ import (
"github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/scanner"
"github.com/z7zmey/php-parser/token"
)
func (lval *yySymType) Token(t token.Token) {
func (lval *yySymType) Token(t scanner.Token) {
lval.token = t
}
@ -19,7 +18,7 @@ func (lval *yySymType) Token(t token.Token) {
type Parser struct {
*scanner.Lexer
path string
lastToken *token.Token
lastToken *scanner.Token
positionBuilder *position.Builder
errors []*errors.Error
rootNode node.Node

9
php5/php5.go
View File

@ -16,14 +16,14 @@ import (
"github.com/z7zmey/php-parser/node/name"
"github.com/z7zmey/php-parser/node/scalar"
"github.com/z7zmey/php-parser/node/stmt"
"github.com/z7zmey/php-parser/token"
"github.com/z7zmey/php-parser/scanner"
)
//line php5/php5.y:21
type yySymType struct {
yys int
node node.Node
token token.Token
token scanner.Token
boolWithToken boolWithToken
list []node.Node
foreachVariable foreachVariable
@ -345,7 +345,6 @@ const yyErrCode = 2
const yyInitialStackSize = 16
//line php5/php5.y:3825
type foreachVariable struct {
node node.Node
byRef bool
@ -353,12 +352,12 @@ type foreachVariable struct {
type nodesWithEndToken struct {
nodes []node.Node
endToken token.Token
endToken scanner.Token
}
type boolWithToken struct {
value bool
token *token.Token
token *scanner.Token
}
type simpleIndirectReference struct {

8
php5/php5.y
View File

@ -5,7 +5,7 @@ import (
"strings"
"strconv"
"github.com/z7zmey/php-parser/token"
"github.com/z7zmey/php-parser/scanner"
"github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/node/scalar"
"github.com/z7zmey/php-parser/node/name"
@ -20,7 +20,7 @@ import (
%union{
node node.Node
token token.Token
token scanner.Token
boolWithToken boolWithToken
list []node.Node
foreachVariable foreachVariable
@ -3831,12 +3831,12 @@ type foreachVariable struct {
type nodesWithEndToken struct {
nodes []node.Node
endToken token.Token
endToken scanner.Token
}
type boolWithToken struct {
value bool
token *token.Token
token *scanner.Token
}
type simpleIndirectReference struct {

5
php7/parser.go
View File

@ -8,10 +8,9 @@ import (
"github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/scanner"
"github.com/z7zmey/php-parser/token"
)
func (lval *yySymType) Token(t token.Token) {
func (lval *yySymType) Token(t scanner.Token) {
lval.token = t
}
@ -19,7 +18,7 @@ func (lval *yySymType) Token(t token.Token) {
type Parser struct {
*scanner.Lexer
path string
lastToken *token.Token
lastToken *scanner.Token
positionBuilder *position.Builder
errors []*errors.Error
rootNode node.Node

9
php7/php7.go
View File

@ -16,14 +16,14 @@ import (
"github.com/z7zmey/php-parser/node/name"
"github.com/z7zmey/php-parser/node/scalar"
"github.com/z7zmey/php-parser/node/stmt"
"github.com/z7zmey/php-parser/token"
"github.com/z7zmey/php-parser/scanner"
)
//line php7/php7.y:21
type yySymType struct {
yys int
node node.Node
token token.Token
token scanner.Token
boolWithToken boolWithToken
list []node.Node
foreachVariable foreachVariable
@ -345,7 +345,6 @@ const yyErrCode = 2
const yyInitialStackSize = 16
//line php7/php7.y:2635
type foreachVariable struct {
node node.Node
byRef bool
@ -353,12 +352,12 @@ type foreachVariable struct {
type nodesWithEndToken struct {
nodes []node.Node
endToken token.Token
endToken scanner.Token
}
type boolWithToken struct {
value bool
token *token.Token
token *scanner.Token
}
type altSyntaxNode struct {

8
php7/php7.y
View File

@ -5,7 +5,7 @@ import (
"strings"
"strconv"
"github.com/z7zmey/php-parser/token"
"github.com/z7zmey/php-parser/scanner"
"github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/node/scalar"
"github.com/z7zmey/php-parser/node/name"
@ -20,7 +20,7 @@ import (
%union{
node node.Node
token token.Token
token scanner.Token
boolWithToken boolWithToken
list []node.Node
foreachVariable foreachVariable
@ -2641,12 +2641,12 @@ type foreachVariable struct {
type nodesWithEndToken struct {
nodes []node.Node
endToken token.Token
endToken scanner.Token
}
type boolWithToken struct {
value bool
token *token.Token
token *scanner.Token
}
type altSyntaxNode struct {

16
position/builder.go
View File

@ -2,7 +2,7 @@ package position
import (
"github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/token"
"github.com/z7zmey/php-parser/scanner"
)
// Builder provide functions to constuct positions
@ -99,7 +99,7 @@ func (b *Builder) NewNodePosition(n node.Node) *Position {
}
// NewTokenPosition returns new Position
func (b *Builder) NewTokenPosition(t token.Token) *Position {
func (b *Builder) NewTokenPosition(t scanner.Token) *Position {
return &Position{
t.StartLine,
t.EndLine,
@ -109,7 +109,7 @@ func (b *Builder) NewTokenPosition(t token.Token) *Position {
}
// NewTokensPosition returns new Position
func (b *Builder) NewTokensPosition(startToken token.Token, endToken token.Token) *Position {
func (b *Builder) NewTokensPosition(startToken scanner.Token, endToken scanner.Token) *Position {
return &Position{
startToken.StartLine,
endToken.EndLine,
@ -119,7 +119,7 @@ func (b *Builder) NewTokensPosition(startToken token.Token, endToken token.Token
}
// NewTokenNodePosition returns new Position
func (b *Builder) NewTokenNodePosition(t token.Token, n node.Node) *Position {
func (b *Builder) NewTokenNodePosition(t scanner.Token, n node.Node) *Position {
return &Position{
t.StartLine,
b.getNodeEndPos(n).endLine,
@ -129,7 +129,7 @@ func (b *Builder) NewTokenNodePosition(t token.Token, n node.Node) *Position {
}
// NewNodeTokenPosition returns new Position
func (b *Builder) NewNodeTokenPosition(n node.Node, t token.Token) *Position {
func (b *Builder) NewNodeTokenPosition(n node.Node, t scanner.Token) *Position {
return &Position{
b.getNodeStartPos(n).startLine,
t.EndLine,
@ -149,7 +149,7 @@ func (b *Builder) NewNodesPosition(startNode node.Node, endNode node.Node) *Posi
}
// NewNodeListTokenPosition returns new Position
func (b *Builder) NewNodeListTokenPosition(list []node.Node, t token.Token) *Position {
func (b *Builder) NewNodeListTokenPosition(list []node.Node, t scanner.Token) *Position {
return &Position{
b.getListStartPos(list).startLine,
t.EndLine,
@ -159,7 +159,7 @@ func (b *Builder) NewNodeListTokenPosition(list []node.Node, t token.Token) *Pos
}
// NewTokenNodeListPosition returns new Position
func (b *Builder) NewTokenNodeListPosition(t token.Token, list []node.Node) *Position {
func (b *Builder) NewTokenNodeListPosition(t scanner.Token, list []node.Node) *Position {
return &Position{
t.StartLine,
b.getListEndPos(list).endLine,
@ -179,7 +179,7 @@ func (b *Builder) NewNodeNodeListPosition(n node.Node, list []node.Node) *Positi
}
// NewOptionalListTokensPosition returns new Position
func (b *Builder) NewOptionalListTokensPosition(list []node.Node, t token.Token, endToken token.Token) *Position {
func (b *Builder) NewOptionalListTokensPosition(list []node.Node, t scanner.Token, endToken scanner.Token) *Position {
if list == nil {
return &Position{
t.StartLine,

28
position/position_test.go
View File

@ -6,13 +6,13 @@ import (
"github.com/z7zmey/php-parser/node"
"github.com/z7zmey/php-parser/position"
"github.com/z7zmey/php-parser/token"
"github.com/z7zmey/php-parser/scanner"
)
func TestNewTokenPosition(t *testing.T) {
builder := position.Builder{}
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
pos := builder.NewTokenPosition(tkn)
@ -24,8 +24,8 @@ func TestNewTokenPosition(t *testing.T) {
func TestNewTokensPosition(t *testing.T) {
builder := position.Builder{}
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6)
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6)
pos := builder.NewTokensPosition(token1, token2)
@ -57,7 +57,7 @@ func TestNewNodePosition(t *testing.T) {
}
func TestNewTokenNodePosition(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
n := node.NewIdentifier("test node")
p := &position.Positions{}
@ -81,7 +81,7 @@ func TestNewTokenNodePosition(t *testing.T) {
func TestNewNodeTokenPosition(t *testing.T) {
n := node.NewIdentifier("test node")
tkn := token.NewToken([]byte(`foo`), 2, 2, 10, 12)
tkn := scanner.NewToken([]byte(`foo`), 2, 2, 10, 12)
p := &position.Positions{}
p.AddPosition(n, &position.Position{
@ -161,7 +161,7 @@ func TestNewNodesPosition(t *testing.T) {
func TestNewNodeListTokenPosition(t *testing.T) {
n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node")
tkn := token.NewToken([]byte(`foo`), 3, 3, 20, 22)
tkn := scanner.NewToken([]byte(`foo`), 3, 3, 20, 22)
builder := position.Builder{
Positions: &position.Positions{
@ -188,7 +188,7 @@ func TestNewNodeListTokenPosition(t *testing.T) {
}
func TestNewTokenNodeListPosition(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 2)
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 2)
n1 := node.NewIdentifier("test node")
n2 := node.NewIdentifier("test node")
@ -254,8 +254,8 @@ func TestNewNodeNodeListPosition(t *testing.T) {
func TestNewOptionalListTokensPosition(t *testing.T) {
builder := position.Builder{}
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
token2 := token.NewToken([]byte(`foo`), 2, 2, 4, 6)
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
token2 := scanner.NewToken([]byte(`foo`), 2, 2, 4, 6)
pos := builder.NewOptionalListTokensPosition(nil, token1, token2)
@ -292,8 +292,8 @@ func TestNewOptionalListTokensPosition2(t *testing.T) {
},
}
token1 := token.NewToken([]byte(`foo`), 4, 4, 27, 29)
token2 := token.NewToken([]byte(`foo`), 5, 5, 30, 32)
token1 := scanner.NewToken([]byte(`foo`), 4, 4, 27, 29)
token2 := scanner.NewToken([]byte(`foo`), 5, 5, 30, 32)
pos := builder.NewOptionalListTokensPosition([]node.Node{n2, n3}, token1, token2)
@ -334,7 +334,7 @@ func TestNilNodeListPos(t *testing.T) {
}
func TestNilNodeListTokenPos(t *testing.T) {
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
builder := position.Builder{}
@ -367,7 +367,7 @@ func TestEmptyNodeListPos(t *testing.T) {
}
func TestEmptyNodeListTokenPos(t *testing.T) {
token1 := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
token1 := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
builder := position.Builder{}

7
scanner/lexer.go
View File

@ -10,7 +10,6 @@ import (
"github.com/cznic/golex/lex"
"github.com/z7zmey/php-parser/comment"
t "github.com/z7zmey/php-parser/token"
)
// Allocate Character classes anywhere in [0x80, 0xFF].
@ -431,7 +430,7 @@ const T_POW = 57481
// Lval parsers yySymType must implement this interface
type Lval interface {
Token(tkn t.Token)
Token(tkn Token)
}
// Lexer php lexer
@ -509,7 +508,7 @@ func (l *Lexer) getCurrentState() int {
return l.StateStack[len(l.StateStack)-1]
}
func (l *Lexer) newToken(chars []lex.Char) t.Token {
func (l *Lexer) newToken(chars []lex.Char) Token {
firstChar := chars[0]
lastChar := chars[len(chars)-1]
@ -518,7 +517,7 @@ func (l *Lexer) newToken(chars []lex.Char) t.Token {
startPos := int(firstChar.Pos())
endPos := int(lastChar.Pos())
return t.NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments)
return NewToken(l.charsToBytes(chars), startLine, endLine, startPos, endPos).SetComments(l.Comments)
}
func (l *Lexer) addComment(c comment.Comment) {

5
scanner/scanner_test.go
View File

@ -8,7 +8,6 @@ import (
"github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/scanner"
"github.com/z7zmey/php-parser/token"
"github.com/kylelemons/godebug/pretty"
)
@ -27,10 +26,10 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
}
type lval struct {
Tkn token.Token
Tkn scanner.Token
}
func (lv *lval) Token(t token.Token) {
func (lv *lval) Token(t scanner.Token) {
lv.Tkn = t
}

2
token/token.go → scanner/token.go
View File

@ -1,4 +1,4 @@
package token
package scanner
import (
"github.com/z7zmey/php-parser/comment"

6
token/token_test.go → scanner/token_test.go
View File

@ -1,4 +1,4 @@
package token_test
package scanner_test
import (
"reflect"
@ -6,11 +6,11 @@ import (
"github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/token"
"github.com/z7zmey/php-parser/scanner"
)
func TestToken(t *testing.T) {
tkn := token.NewToken([]byte(`foo`), 1, 1, 0, 3)
tkn := scanner.NewToken([]byte(`foo`), 1, 1, 0, 3)
c := []comment.Comment{
comment.NewPlainComment("test comment"),