#67: skip unexpected character in input

errors/error.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/z7zmey/php-parser/position"
-	"github.com/z7zmey/php-parser/scanner"
 )
 
 // Error parsing error
@@ -14,13 +13,18 @@ type Error struct {
 }
 
 // NewError creates and returns new Error
-func NewError(msg string, t *scanner.Token) *Error {
+func NewError(msg string, p *position.Position) *Error {
 	return &Error{
 		Msg: msg,
-		Pos: t.Position,
+		Pos: p,
 	}
 }
 
 func (e *Error) String() string {
-	return fmt.Sprintf("%s at line %d", e.Msg, e.Pos.StartLine)
+	atLine := ""
+	if e.Pos != nil {
+		atLine = fmt.Sprintf(" at line %d", e.Pos.StartLine)
+	}
+
+	return fmt.Sprintf("%s%s", e.Msg, atLine)
 }

errors/error_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/z7zmey/php-parser/position"
 
 	"github.com/z7zmey/php-parser/errors"
-	"github.com/z7zmey/php-parser/scanner"
 
 	"github.com/kylelemons/godebug/pretty"
 )
@@ -27,12 +26,8 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
 
 func TestConstructor(t *testing.T) {
 	pos := position.NewPosition(1, 2, 3, 4)
-	token := &scanner.Token{
-		Value:    `test`,
-		Position: pos,
-	}
 
-	actual := errors.NewError("message", token)
+	actual := errors.NewError("message", pos)
 
 	expected := &errors.Error{
 		Msg: "message",
@@ -44,12 +39,8 @@ func TestConstructor(t *testing.T) {
 
 func TestPrint(t *testing.T) {
 	pos := position.NewPosition(1, 2, 3, 4)
-	token := &scanner.Token{
-		Value:    `test`,
-		Position: pos,
-	}
 
-	Error := errors.NewError("message", token)
+	Error := errors.NewError("message", pos)
 
 	actual := Error.String()
 
@@ -57,3 +48,13 @@ func TestPrint(t *testing.T) {
 
 	assertEqual(t, expected, actual)
 }
+
+func TestPrintWithotPos(t *testing.T) {
+	Error := errors.NewError("message", nil)
+
+	actual := Error.String()
+
+	expected := "message"
+
+	assertEqual(t, expected, actual)
+}

php5/parser.go

@@ -20,7 +20,6 @@ type Parser struct {
 	path            string
 	currentToken    *scanner.Token
 	positionBuilder *parser.PositionBuilder
-	errors          []*errors.Error
 	rootNode        node.Node
 	comments        parser.Comments
 	positions       parser.Positions
@@ -38,7 +37,6 @@ func NewParser(src io.Reader, path string) *Parser {
 		nil,
 		nil,
 		nil,
-		nil,
 	}
 }
 
@@ -50,13 +48,13 @@ func (l *Parser) Lex(lval *yySymType) int {
 }
 
 func (l *Parser) Error(msg string) {
-	l.errors = append(l.errors, errors.NewError(msg, l.currentToken))
+	l.Lexer.Errors = append(l.Lexer.Errors, errors.NewError(msg, l.currentToken.Position))
 }
 
 // Parse the php7 Parser entrypoint
 func (l *Parser) Parse() int {
 	// init
-	l.errors = nil
+	l.Lexer.Errors = nil
 	l.rootNode = nil
 	l.comments = parser.Comments{}
 	l.positions = parser.Positions{}
@@ -92,7 +90,7 @@ func (l *Parser) GetRootNode() node.Node {
 
 // GetErrors returns errors list
 func (l *Parser) GetErrors() []*errors.Error {
-	return l.errors
+	return l.Lexer.Errors
 }
 
 // GetComments returns comments list

php5/php5.go

@@ -1,3 +1,5 @@
+// Code generated by goyacc -o php5/php5.go php5/php5.y. DO NOT EDIT.
+
 //line php5/php5.y:2
 package php5
 

php5/php5_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/kylelemons/godebug/pretty"
+	"github.com/z7zmey/php-parser/errors"
 	"github.com/z7zmey/php-parser/node/expr"
 	"github.com/z7zmey/php-parser/node/expr/assign"
 	"github.com/z7zmey/php-parser/node/expr/binary"
@@ -13,6 +14,7 @@ import (
 	"github.com/z7zmey/php-parser/node/name"
 	"github.com/z7zmey/php-parser/node/scalar"
 	"github.com/z7zmey/php-parser/php5"
+	"github.com/z7zmey/php-parser/position"
 
 	"github.com/z7zmey/php-parser/node"
 	"github.com/z7zmey/php-parser/node/stmt"
@@ -3750,3 +3752,23 @@ CAD;
 	actual := php5parser.GetRootNode()
 	assertEqual(t, expected, actual)
 }
+
+func TestPhp5ControlCharsErrors(t *testing.T) {
+	src := "<?php \004 echo $b; \"$a[\005test]\";"
+
+	expected := []*errors.Error{
+		{
+			Msg: "WARNING: Unexpected character in input: '\004' (ASCII=4)",
+			Pos: &position.Position{1, 1, 7, 7},
+		},
+		{
+			Msg: "WARNING: Unexpected character in input: '\005' (ASCII=5)",
+			Pos: &position.Position{1, 1, 22, 22},
+		},
+	}
+
+	php5parser := php5.NewParser(bytes.NewBufferString(src), "test.php")
+	php5parser.Parse()
+	actual := php5parser.GetErrors()
+	assertEqual(t, expected, actual)
+}

php7/parser.go

@@ -19,7 +19,6 @@ type Parser struct {
 	path            string
 	currentToken    *scanner.Token
 	positionBuilder *parser.PositionBuilder
-	errors          []*errors.Error
 	rootNode        node.Node
 	comments        parser.Comments
 	positions       parser.Positions
@@ -37,7 +36,6 @@ func NewParser(src io.Reader, path string) *Parser {
 		nil,
 		nil,
 		nil,
-		nil,
 	}
 }
 
@@ -49,13 +47,13 @@ func (l *Parser) Lex(lval *yySymType) int {
 }
 
 func (l *Parser) Error(msg string) {
-	l.errors = append(l.errors, errors.NewError(msg, l.currentToken))
+	l.Lexer.Errors = append(l.Lexer.Errors, errors.NewError(msg, l.currentToken.Position))
 }
 
 // Parse the php7 Parser entrypoint
 func (l *Parser) Parse() int {
 	// init
-	l.errors = nil
+	l.Lexer.Errors = nil
 	l.rootNode = nil
 	l.comments = parser.Comments{}
 	l.positions = parser.Positions{}
@@ -81,7 +79,7 @@ func (l *Parser) GetRootNode() node.Node {
 
 // GetErrors returns errors list
 func (l *Parser) GetErrors() []*errors.Error {
-	return l.errors
+	return l.Lexer.Errors
 }
 
 // GetComments returns comments list

php7/php7.go

@@ -1,3 +1,5 @@
+// Code generated by goyacc -o php7/php7.go php7/php7.y. DO NOT EDIT.
+
 //line php7/php7.y:2
 package php7
 

php7/php7_test.go

@@ -6,12 +6,14 @@ import (
 	"testing"
 
 	"github.com/kylelemons/godebug/pretty"
+	"github.com/z7zmey/php-parser/errors"
 	"github.com/z7zmey/php-parser/node/expr"
 	"github.com/z7zmey/php-parser/node/expr/assign"
 	"github.com/z7zmey/php-parser/node/expr/binary"
 	"github.com/z7zmey/php-parser/node/expr/cast"
 	"github.com/z7zmey/php-parser/node/name"
 	"github.com/z7zmey/php-parser/node/scalar"
+	"github.com/z7zmey/php-parser/position"
 
 	"github.com/z7zmey/php-parser/node"
 	"github.com/z7zmey/php-parser/node/stmt"
@@ -3336,3 +3338,23 @@ CAD;
 	actual := php7parser.GetRootNode()
 	assertEqual(t, expected, actual)
 }
+
+func TestPhp7ControlCharsErrors(t *testing.T) {
+	src := "<?php \004 echo $b; \"$a[\005test]\";"
+
+	expected := []*errors.Error{
+		{
+			Msg: "WARNING: Unexpected character in input: '\004' (ASCII=4)",
+			Pos: &position.Position{1, 1, 7, 7},
+		},
+		{
+			Msg: "WARNING: Unexpected character in input: '\005' (ASCII=5)",
+			Pos: &position.Position{1, 1, 22, 22},
+		},
+	}
+
+	php7parser := php7.NewParser(bytes.NewBufferString(src), "test.php")
+	php7parser.Parse()
+	actual := php7parser.GetErrors()
+	assertEqual(t, expected, actual)
+}

scanner/lexer.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"unicode"
 
+	"github.com/z7zmey/php-parser/errors"
 	"github.com/z7zmey/php-parser/position"
 
 	"github.com/cznic/golex/lex"
@@ -446,6 +447,7 @@ type Lexer struct {
 	tokenBytesBuf *bytes.Buffer
 	TokenPool     sync.Pool
 	PositionPool  sync.Pool
+	Errors        []*errors.Error
 }
 
 // Rune2Class returns the rune integer id
@@ -492,6 +494,21 @@ func NewLexer(src io.Reader, fName string) *Lexer {
 	}
 }
 
+func (l *Lexer) Error(msg string) {
+	chars := l.Token()
+	firstChar := chars[0]
+	lastChar := chars[len(chars)-1]
+
+	pos := position.NewPosition(
+		l.File.Line(firstChar.Pos()),
+		l.File.Line(lastChar.Pos()),
+		int(firstChar.Pos()),
+		int(lastChar.Pos()),
+	)
+
+	l.Errors = append(l.Errors, errors.NewError(msg, pos))
+}
+
 func (l *Lexer) ungetChars(n int) []lex.Char {
 	l.Unget(l.Lookahead())
 

scanner/scanner.l

@@ -54,6 +54,7 @@ EXPONENT_DNUM   (({LNUM}|{DNUM})[eE][+-]?{LNUM})
 VAR_NAME        [a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*
 OPERATORS       [;:,.\[\]()|\/\^&\+-*=%!~$<>?@]
 NEW_LINE        (\r|\n|\r\n)
+ANY_CHAR        .
 
 %%
     c = l.Rule0()
@@ -648,13 +649,15 @@ NEW_LINE        (\r|\n|\r\n)
 <STRING_VAR_INDEX>\]                        l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
 <STRING_VAR_INDEX>[ \n\r\t\\'#]             l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return T_ENCAPSED_AND_WHITESPACE
 <STRING_VAR_INDEX>{OPERATORS}               lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
-<STRING_VAR_INDEX>.                         lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
+<STRING_VAR_INDEX>{ANY_CHAR}                l.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", l.TokenBytes(nil)[0], l.TokenBytes(nil)[0]));l.Abort();
 
 <STRING_VAR_NAME>{VAR_NAME}[\[\}]           l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return T_STRING_VARNAME
 <STRING_VAR_NAME>.                          l.ungetChars(1);l.popState();l.pushState(PHP)
 
 <HALT_COMPILER>.|[ \t\n\r]                  // do nothing
 
+<PHP>{ANY_CHAR}                             l.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", l.TokenBytes(nil)[0], l.TokenBytes(nil)[0]));l.Abort();
+
 %%
     if c, ok := l.Abort(); ok { return int(c) }
     goto yyAction

scanner/scanner_test.go

@@ -1077,3 +1077,52 @@ func TestEmptyInlineComment2(t *testing.T) {
 
 	assertEqual(t, expected, actual)
 }
+
+func TestIgnoreControllCharacters(t *testing.T) {
+	src := "<?php \004 echo $b;"
+
+	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
+	lv := &lval{}
+
+	expected := "echo"
+	lexer.Lex(lv)
+	actual := lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "$b"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+}
+
+func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) {
+	src := "<?php \"$a[test\004]\";"
+
+	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
+	lv := &lval{}
+
+	expected := "\""
+	lexer.Lex(lv)
+	actual := lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "$a"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "["
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "test"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "]"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+}