maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[refactoring] move errors from scanner to parser

Browse Source
This commit is contained in:
Vadym Slizov 2020-05-18 20:07:17 +03:00
parent 4f79d47b1e
commit 291dc7e884
6 changed files with 105 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
internal/php5/parser.go
View File

@ -21,19 +21,21 @@ type Parser struct {
currentToken *scanner.Token
positionBuilder *positionbuilder.PositionBuilder
rootNode ast.Vertex
errors []*errors.Error
}
// NewParser creates and returns new Parser
func NewParser(src []byte, v string) *Parser {
lexer := scanner.NewLexer(src)
parser := &Parser{}
lexer := scanner.NewLexer(src, func(e *errors.Error) {
parser.errors = append(parser.errors, e)
})
lexer.PHPVersion = v
return &Parser{
lexer,
nil,
nil,
nil,
}
parser.Lexer = lexer
return parser
}
// Lex proxy to scanner Lex
@ -49,7 +51,12 @@ func (l *Parser) Lex(lval *yySymType) int {
func (l *Parser) Error(msg string) {
var pos = l.currentToken.Position
l.Lexer.AddError(errors.NewError(msg, &pos))
l.errors = append(l.errors, errors.NewError(msg, &pos))
}
// GetErrors returns errors list
func (l *Parser) GetErrors() []*errors.Error {
return l.errors
}
func (l *Parser) WithTokens() {
@ -59,7 +66,7 @@ func (l *Parser) WithTokens() {
// Parse the php7 Parser entrypoint
func (l *Parser) Parse() int {
// init
l.Lexer.SetErrors(nil)
l.errors = nil
l.rootNode = nil
l.positionBuilder = &positionbuilder.PositionBuilder{}
@ -73,11 +80,6 @@ func (l *Parser) GetRootNode() ast.Vertex {
return l.rootNode
}
// GetErrors returns errors list
func (l *Parser) GetErrors() []*errors.Error {
return l.Lexer.GetErrors()
}
// helpers
func lastNode(nn []ast.Vertex) ast.Vertex {

30
internal/php7/parser.go
View File

@ -20,19 +20,21 @@ type Parser struct {
currentToken *scanner.Token
positionBuilder *positionbuilder.PositionBuilder
rootNode ast.Vertex
errors []*errors.Error
}
// NewParser creates and returns new Parser
func NewParser(src []byte, v string) *Parser {
lexer := scanner.NewLexer(src)
parser := &Parser{}
lexer := scanner.NewLexer(src, func(e *errors.Error) {
parser.errors = append(parser.errors, e)
})
lexer.PHPVersion = v
return &Parser{
lexer,
nil,
nil,
nil,
}
parser.Lexer = lexer
return parser
}
func (l *Parser) Lex(lval *yySymType) int {
@ -47,7 +49,12 @@ func (l *Parser) Lex(lval *yySymType) int {
func (l *Parser) Error(msg string) {
var pos = l.currentToken.Position
l.Lexer.AddError(errors.NewError(msg, &pos))
l.errors = append(l.errors, errors.NewError(msg, &pos))
}
// GetErrors returns errors list
func (l *Parser) GetErrors() []*errors.Error {
return l.errors
}
func (l *Parser) WithTokens() {
@ -57,7 +64,7 @@ func (l *Parser) WithTokens() {
// Parse the php7 Parser entrypoint
func (l *Parser) Parse() int {
// init
l.Lexer.SetErrors(nil)
l.errors = nil
l.rootNode = nil
l.positionBuilder = &positionbuilder.PositionBuilder{}
@ -71,11 +78,6 @@ func (l *Parser) GetRootNode() ast.Vertex {
return l.rootNode
}
// GetErrors returns errors list
func (l *Parser) GetErrors() []*errors.Error {
return l.Lexer.GetErrors()
}
// helpers
func lastNode(nn []ast.Vertex) ast.Vertex {

43
internal/scanner/lexer.go
View File

@ -13,15 +13,14 @@ import (
type Scanner interface {
Lex() *Token
ReturnTokenToPool(t *Token)
GetErrors() []*errors.Error
GetWithHiddenTokens() bool
SetWithHiddenTokens(bool)
AddError(e *errors.Error)
SetErrors(e []*errors.Error)
}
type Lexer struct {
data []byte
errHandlerFunc func(*errors.Error)
p, pe, cs int
ts, te, act int
stack []int
@ -31,17 +30,29 @@ type Lexer struct {
TokenPool *TokenPool
HiddenTokens []token.Token
WithHiddenTokens bool
Errors []*errors.Error
NewLines NewLines
PHPVersion string
}
func (l *Lexer) ReturnTokenToPool(t *Token) {
l.TokenPool.Put(t)
func NewLexer(data []byte, errHandlerFunc func(*errors.Error)) *Lexer {
lex := &Lexer{
data: data,
errHandlerFunc: errHandlerFunc,
pe: len(data),
stack: make([]int, 0),
TokenPool: &TokenPool{},
NewLines: NewLines{make([]int, 0, 128)},
}
func (l *Lexer) GetErrors() []*errors.Error {
return l.Errors
initLexer(lex)
return lex
}
func (l *Lexer) ReturnTokenToPool(t *Token) {
l.TokenPool.Put(t)
}
func (l *Lexer) GetWithHiddenTokens() bool {
@ -52,14 +63,6 @@ func (l *Lexer) SetWithHiddenTokens(b bool) {
l.WithHiddenTokens = b
}
func (l *Lexer) AddError(e *errors.Error) {
l.Errors = append(l.Errors, e)
}
func (l *Lexer) SetErrors(e []*errors.Error) {
l.Errors = e
}
func (lex *Lexer) setTokenPosition(token *Token) {
token.Position.StartLine = lex.NewLines.GetLine(lex.ts)
token.Position.EndLine = lex.NewLines.GetLine(lex.te - 1)
@ -230,7 +233,11 @@ func (lex *Lexer) ungetCnt(n int) {
lex.te = lex.te - n
}
func (lex *Lexer) Error(msg string) {
func (lex *Lexer) error(msg string) {
if lex.errHandlerFunc == nil {
return
}
pos := position.NewPosition(
lex.NewLines.GetLine(lex.ts),
lex.NewLines.GetLine(lex.te-1),
@ -238,7 +245,7 @@ func (lex *Lexer) Error(msg string) {
lex.te,
)
lex.Errors = append(lex.Errors, errors.NewError(msg, pos))
lex.errHandlerFunc(errors.NewError(msg, pos))
}
func isValidVarNameStart(r byte) bool {

BIN
internal/scanner/scanner.go
View File

Binary file not shown.

15
internal/scanner/scanner.rl
View File

@ -14,17 +14,8 @@ import (
variable pe lex.pe;
}%%
func NewLexer(data []byte) *Lexer {
lex := &Lexer{
data: data,
pe: len(data),
stack: make([]int, 0),
TokenPool: &TokenPool{},
NewLines: NewLines{make([]int, 0, 128)},
}
func initLexer(lex *Lexer) {
%% write init;
return lex
}
func (lex *Lexer) Lex() *Token {
@ -382,7 +373,7 @@ func (lex *Lexer) Lex() *Token {
any_line => {
c := lex.data[lex.p]
lex.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c));
lex.error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c));
};
*|;
@ -473,7 +464,7 @@ func (lex *Lexer) Lex() *Token {
']' > (svi, 2) => {lex.setTokenPosition(token); tok = TokenID(int(']')); lex.ret(2); goto _out;};
any_line => {
c := lex.data[lex.p]
lex.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c));
lex.error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c));
};
*|;

75
internal/scanner/scanner_test.go
View File

@ -1,6 +1,8 @@
package scanner
import (
"github.com/z7zmey/php-parser/pkg/errors"
"github.com/z7zmey/php-parser/pkg/position"
"testing"
"github.com/z7zmey/php-parser/pkg/token"
@ -351,7 +353,7 @@ func TestTokens(t *testing.T) {
T_UNSET_CAST.String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
actual := []string{}
@ -379,7 +381,7 @@ func TestShebang(t *testing.T) {
"\n",
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
actual := []string{}
@ -399,7 +401,7 @@ func TestShebangHtml(t *testing.T) {
0.1
`
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
tkn := lexer.Lex()
@ -449,7 +451,7 @@ func TestNumberTokens(t *testing.T) {
T_DNUMBER.String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
actual := []string{}
@ -506,7 +508,7 @@ func TestConstantStrings(t *testing.T) {
T_CONSTANT_ENCAPSED_STRING.String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
actual := []string{}
@ -553,7 +555,7 @@ func TestSingleQuoteStringTokens(t *testing.T) {
T_CONSTANT_ENCAPSED_STRING.String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
actual := []string{}
for {
@ -640,7 +642,7 @@ func TestTeplateStringTokens(t *testing.T) {
TokenID(int('"')).String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
actual := []string{}
@ -725,7 +727,7 @@ func TestBackquoteStringTokens(t *testing.T) {
TokenID(int('`')).String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
actual := []string{}
@ -819,7 +821,7 @@ CAT;
TokenID(int(';')).String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
actual := []string{}
@ -892,7 +894,7 @@ CAT
T_END_HEREDOC.String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
actual := []string{}
@ -931,7 +933,7 @@ CAT;
TokenID(int(';')).String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
actual := []string{}
@ -962,7 +964,7 @@ func TestHereDocTokens73(t *testing.T) {
T_VARIABLE.String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
actual := []string{}
@ -992,7 +994,7 @@ CAT;`
TokenID(int(';')).String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.PHPVersion = "7.2"
lexer.WithHiddenTokens = true
actual := []string{}
@ -1025,7 +1027,7 @@ func TestInlineHtmlNopTokens(t *testing.T) {
T_INLINE_HTML.String(),
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
actual := []string{}
@ -1060,7 +1062,7 @@ func TestStringTokensAfterVariable(t *testing.T) {
"\"",
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
actual := []string{}
actualTokens := []string{}
@ -1093,7 +1095,7 @@ func TestSlashAfterVariable(t *testing.T) {
"3",
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
actual := []string{}
actualTokens := []string{}
@ -1130,7 +1132,7 @@ func TestCommentEnd(t *testing.T) {
},
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer.Lex()
@ -1159,7 +1161,7 @@ func TestCommentNewLine(t *testing.T) {
},
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
tkn := lexer.Lex()
@ -1188,7 +1190,7 @@ func TestCommentNewLine1(t *testing.T) {
},
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
tkn := lexer.Lex()
@ -1217,7 +1219,7 @@ func TestCommentNewLine2(t *testing.T) {
},
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
tkn := lexer.Lex()
@ -1247,7 +1249,7 @@ func TestCommentWithPhpEndTag(t *testing.T) {
},
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
tkn := lexer.Lex()
@ -1277,7 +1279,7 @@ func TestInlineComment(t *testing.T) {
},
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
tkn := lexer.Lex()
@ -1307,7 +1309,7 @@ func TestInlineComment2(t *testing.T) {
},
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer.Lex()
@ -1341,7 +1343,7 @@ func TestEmptyInlineComment(t *testing.T) {
},
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
lexer.Lex()
@ -1371,7 +1373,7 @@ func TestEmptyInlineComment2(t *testing.T) {
},
}
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
tkn := lexer.Lex()
@ -1385,7 +1387,7 @@ func TestMethodCallTokens(t *testing.T) {
src := `<?php
$a -> bar ( '' ) ;`
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
expected := []token.Token{
@ -1467,7 +1469,7 @@ func TestYieldFromTokens(t *testing.T) {
src := `<?php
yield from $a`
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
lexer.WithHiddenTokens = true
expected := []token.Token{
@ -1498,7 +1500,7 @@ func TestYieldFromTokens(t *testing.T) {
func TestVarNameByteChars(t *testing.T) {
src := "<?php $\x80 $\xff"
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
tkn := lexer.Lex()
assert.Equal(t, "$\x80", string(tkn.Value))
@ -1510,7 +1512,7 @@ func TestVarNameByteChars(t *testing.T) {
func TestStringVarNameByteChars(t *testing.T) {
src := "<?php \"$\x80 $\xff\""
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
tkn := lexer.Lex()
assert.Equal(t, "\"", string(tkn.Value))
@ -1531,7 +1533,12 @@ func TestStringVarNameByteChars(t *testing.T) {
func TestIgnoreControllCharacters(t *testing.T) {
src := "<?php \004 echo $b;"
lexer := NewLexer([]byte(src))
var actualErr *errors.Error
errHandler := func(e *errors.Error) {
actualErr = e
}
lexer := NewLexer([]byte(src), errHandler)
expected := "echo"
tkn := lexer.Lex()
@ -1542,12 +1549,18 @@ func TestIgnoreControllCharacters(t *testing.T) {
tkn = lexer.Lex()
actual = string(tkn.Value)
assert.DeepEqual(t, expected, actual)
expectedErr := &errors.Error{
Msg: "WARNING: Unexpected character in input: '\x04' (ASCII=4)",
Pos: &position.Position{StartLine: 1, EndLine: 1, StartPos: 6, EndPos: 7},
}
assert.DeepEqual(t, expectedErr, actualErr)
}
func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) {
src := "<?php \"$a[test\004]\";"
lexer := NewLexer([]byte(src))
lexer := NewLexer([]byte(src), nil)
expected := "\""
tkn := lexer.Lex()