Merge branch 'master' into dev

scanner/lexer.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"unicode"
 
+	"github.com/z7zmey/php-parser/errors"
 	"github.com/z7zmey/php-parser/position"
 
 	"github.com/cznic/golex/lex"
@@ -38,6 +39,7 @@ type Lexer struct {
 	TokenPool     *TokenPool
 	WithMeta      bool
 	lastToken     *Token
+	Errors        []*errors.Error
 }
 
 // Rune2Class returns the rune integer id
@@ -79,6 +81,21 @@ func NewLexer(src io.Reader, fName string) *Lexer {
 	}
 }
 
+func (l *Lexer) Error(msg string) {
+	chars := l.Token()
+	firstChar := chars[0]
+	lastChar := chars[len(chars)-1]
+
+	pos := position.NewPosition(
+		l.File.Line(firstChar.Pos()),
+		l.File.Line(lastChar.Pos()),
+		int(firstChar.Pos()),
+		int(lastChar.Pos()),
+	)
+
+	l.Errors = append(l.Errors, errors.NewError(msg, pos))
+}
+
 func (l *Lexer) ungetChars(n int) []lex.Char {
 	l.Unget(l.Lookahead())
 

scanner/scanner.l

@@ -55,6 +55,7 @@ EXPONENT_DNUM   (({LNUM}|{DNUM})[eE][+-]?{LNUM})
 VAR_NAME        [a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*
 OPERATORS       [;:,.\[\]()|\/\^&\+-*=%!~$<>?@]
 NEW_LINE        (\r|\n|\r\n)
+ANY_CHAR        .
 
 %%
     c = l.Rule0()
@@ -668,13 +669,15 @@ NEW_LINE        (\r|\n|\r\n)
 <STRING_VAR_INDEX>\]                        l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
 <STRING_VAR_INDEX>[ \n\r\t\\'#]             l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return int(T_ENCAPSED_AND_WHITESPACE)
 <STRING_VAR_INDEX>{OPERATORS}               lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
-<STRING_VAR_INDEX>.                         lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
+<STRING_VAR_INDEX>{ANY_CHAR}                l.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", l.TokenBytes(nil)[0], l.TokenBytes(nil)[0]));l.Abort();
 
 <STRING_VAR_NAME>{VAR_NAME}[\[\}]           l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_STRING_VARNAME)
 <STRING_VAR_NAME>.                          l.ungetChars(1);l.popState();l.pushState(PHP)
 
 <HALT_COMPILER>.|[ \t\n\r]                  l.addMeta(meta.TokenType, l.Token())
 
+<PHP>{ANY_CHAR}                             l.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", l.TokenBytes(nil)[0], l.TokenBytes(nil)[0]));l.Abort();
+
 %%
     if _, ok := l.Abort(); ok {
         // always return same $end token

scanner/scanner_test.go

@@ -1388,3 +1388,52 @@ func TestYieldFromTokens(t *testing.T) {
 	actual = lv.Tkn.Meta
 	assertEqual(t, expected, actual)
 }
+
+func TestIgnoreControllCharacters(t *testing.T) {
+	src := "<?php \004 echo $b;"
+
+	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
+	lv := &lval{}
+
+	expected := "echo"
+	lexer.Lex(lv)
+	actual := lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "$b"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+}
+
+func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) {
+	src := "<?php \"$a[test\004]\";"
+
+	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
+	lv := &lval{}
+
+	expected := "\""
+	lexer.Lex(lv)
+	actual := lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "$a"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "["
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "test"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "]"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+}