Merge branch 'master' into dev

.travis.yml

@@ -0,0 +1,16 @@
+language: go
+branches:
+  only:
+    - master
+before_script:
+  - go get -u golang.org/x/tools/cmd/goyacc
+  - go get -u github.com/kylelemons/godebug/pretty
+  - go get -u github.com/cznic/golex/lex
+  - go get -u github.com/yookoala/realpath
+  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+  - chmod +x ./cc-test-reporter
+  - ./cc-test-reporter before-build
+script:
+  - go test -coverprofile c.out ./...
+after_script:
+  - ./cc-test-reporter after-build --coverage-input-type gocov --exit-code $TRAVIS_TEST_RESULT

README.md

@@ -10,10 +10,11 @@ PHP Parser written in Go
 
 <img src="./parser.jpg" alt="PHP Parser written in Go" width="980"/>
 
-[![Go Report Card](https://goreportcard.com/badge/github.com/z7zmey/php-parser)](https://goreportcard.com/report/github.com/z7zmey/php-parser)
-[![Exago](https://api.exago.io:443/badge/tests/github.com/z7zmey/php-parser)](https://exago.io/project/github.com/z7zmey/php-parser)
-[![Exago](https://api.exago.io:443/badge/cov/github.com/z7zmey/php-parser)](https://exago.io/project/github.com/z7zmey/php-parser)
 [![GoDoc](https://godoc.org/github.com/z7zmey/php-parser?status.svg)](https://godoc.org/github.com/z7zmey/php-parser)
+[![Build Status](https://travis-ci.org/z7zmey/php-parser.svg?branch=master)](https://travis-ci.org/z7zmey/php-parser)
+[![Go Report Card](https://goreportcard.com/badge/github.com/z7zmey/php-parser)](https://goreportcard.com/report/github.com/z7zmey/php-parser)
+[![Maintainability](https://api.codeclimate.com/v1/badges/950783b2e739db26e0ed/maintainability)](https://codeclimate.com/github/z7zmey/php-parser/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/950783b2e739db26e0ed/test_coverage)](https://codeclimate.com/github/z7zmey/php-parser/test_coverage)
 
 This project uses [goyacc](https://godoc.org/golang.org/x/tools/cmd/goyacc) and [golex](https://github.com/cznic/golex) libraries to parse PHP sources into [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree). It can be used to write static analysis, refactoring, metrics, code style formatting tools.
 

errors/error.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/z7zmey/php-parser/position"
-	"github.com/z7zmey/php-parser/scanner"
 )
 
 // Error parsing error
@@ -14,18 +13,18 @@ type Error struct {
 }
 
 // NewError creates and returns new Error
-func NewError(msg string, t *scanner.Token) *Error {
+func NewError(msg string, p *position.Position) *Error {
 	return &Error{
 		Msg: msg,
-		Pos: &position.Position{
+		Pos: p,
-			StartLine: t.StartLine,
-			EndLine:   t.EndLine,
-			StartPos:  t.StartPos,
-			EndPos:    t.EndPos,
-		},
 	}
 }
 
 func (e *Error) String() string {
-	return fmt.Sprintf("%s at line %d", e.Msg, e.Pos.StartLine)
+	atLine := ""
+	if e.Pos != nil {
+		atLine = fmt.Sprintf(" at line %d", e.Pos.StartLine)
+	}
+
+	return fmt.Sprintf("%s%s", e.Msg, atLine)
 }

errors/error_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/z7zmey/php-parser/position"
 
 	"github.com/z7zmey/php-parser/errors"
-	"github.com/z7zmey/php-parser/scanner"
 
 	"github.com/kylelemons/godebug/pretty"
 )
@@ -27,15 +26,8 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
 
 func TestConstructor(t *testing.T) {
 	pos := position.NewPosition(1, 2, 3, 4)
-	token := &scanner.Token{
-		Value:     `test`,
-		StartLine: 1,
-		EndLine:   2,
-		StartPos:  3,
-		EndPos:    4,
-	}
 
-	actual := errors.NewError("message", token)
+	actual := errors.NewError("message", pos)
 
 	expected := &errors.Error{
 		Msg: "message",
@@ -46,15 +38,9 @@ func TestConstructor(t *testing.T) {
 }
 
 func TestPrint(t *testing.T) {
-	token := &scanner.Token{
+	pos := position.NewPosition(1, 2, 3, 4)
-		Value:     `test`,
-		StartLine: 1,
-		EndLine:   2,
-		StartPos:  3,
-		EndPos:    4,
-	}
 
-	Error := errors.NewError("message", token)
+	Error := errors.NewError("message", pos)
 
 	actual := Error.String()
 
@@ -62,3 +48,13 @@ func TestPrint(t *testing.T) {
 
 	assertEqual(t, expected, actual)
 }
+
+func TestPrintWithotPos(t *testing.T) {
+	Error := errors.NewError("message", nil)
+
+	actual := Error.String()
+
+	expected := "message"
+
+	assertEqual(t, expected, actual)
+}

php5/parser.go

@@ -7,6 +7,7 @@ import (
 	"github.com/z7zmey/php-parser/meta"
 	"github.com/z7zmey/php-parser/node"
 	"github.com/z7zmey/php-parser/parser"
+	"github.com/z7zmey/php-parser/position"
 	"github.com/z7zmey/php-parser/scanner"
 )
 
@@ -20,7 +21,6 @@ type Parser struct {
 	path            string
 	currentToken    *scanner.Token
 	positionBuilder *parser.PositionBuilder
-	errors          []*errors.Error
 	rootNode        node.Node
 }
 
@@ -34,7 +34,6 @@ func NewParser(src io.Reader, path string) *Parser {
 		nil,
 		nil,
 		nil,
-		nil,
 	}
 }
 
@@ -46,7 +45,14 @@ func (l *Parser) Lex(lval *yySymType) int {
 }
 
 func (l *Parser) Error(msg string) {
-	l.errors = append(l.errors, errors.NewError(msg, l.currentToken))
+	pos := &position.Position{
+		StartLine: l.currentToken.StartLine,
+		EndLine:   l.currentToken.EndLine,
+		StartPos:  l.currentToken.StartPos,
+		EndPos:    l.currentToken.EndPos,
+	}
+
+	l.Lexer.Errors = append(l.Lexer.Errors, errors.NewError(msg, pos))
 }
 
 func (l *Parser) WithMeta() {
@@ -56,7 +62,7 @@ func (l *Parser) WithMeta() {
 // Parse the php7 Parser entrypoint
 func (l *Parser) Parse() int {
 	// init
-	l.errors = nil
+	l.Lexer.Errors = nil
 	l.rootNode = nil
 	l.positionBuilder = &parser.PositionBuilder{}
 
@@ -87,7 +93,7 @@ func (l *Parser) GetRootNode() node.Node {
 
 // GetErrors returns errors list
 func (l *Parser) GetErrors() []*errors.Error {
-	return l.errors
+	return l.Lexer.Errors
 }
 
 // helpers

php5/php5.go

@@ -1,3 +1,5 @@
+// Code generated by goyacc -o php5/php5.go php5/php5.y. DO NOT EDIT.
+
 //line php5/php5.y:2
 package php5
 
@@ -347,7 +349,6 @@ const yyErrCode = 2
 const yyInitialStackSize = 16
 
 //line php5/php5.y:6993
-
 type simpleIndirectReference struct {
 	all  []*expr.Variable
 	last *expr.Variable

php5/php5_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/kylelemons/godebug/pretty"
+	"github.com/z7zmey/php-parser/errors"
 	"github.com/z7zmey/php-parser/node/expr"
 	"github.com/z7zmey/php-parser/node/expr/assign"
 	"github.com/z7zmey/php-parser/node/expr/binary"
@@ -18708,3 +18709,23 @@ CAD;
 	actual := php5parser.GetRootNode()
 	assertEqual(t, expected, actual)
 }
+
+func TestPhp5ControlCharsErrors(t *testing.T) {
+	src := "<?php \004 echo $b; \"$a[\005test]\";"
+
+	expected := []*errors.Error{
+		{
+			Msg: "WARNING: Unexpected character in input: '\004' (ASCII=4)",
+			Pos: &position.Position{1, 1, 7, 7},
+		},
+		{
+			Msg: "WARNING: Unexpected character in input: '\005' (ASCII=5)",
+			Pos: &position.Position{1, 1, 22, 22},
+		},
+	}
+
+	php5parser := php5.NewParser(bytes.NewBufferString(src), "test.php")
+	php5parser.Parse()
+	actual := php5parser.GetErrors()
+	assertEqual(t, expected, actual)
+}

php7/parser.go

@@ -3,6 +3,8 @@ package php7
 import (
 	"io"
 
+	"github.com/z7zmey/php-parser/position"
+
 	"github.com/z7zmey/php-parser/meta"
 
 	"github.com/z7zmey/php-parser/errors"
@@ -21,7 +23,6 @@ type Parser struct {
 	path            string
 	currentToken    *scanner.Token
 	positionBuilder *parser.PositionBuilder
-	errors          []*errors.Error
 	rootNode        node.Node
 }
 
@@ -35,7 +36,6 @@ func NewParser(src io.Reader, path string) *Parser {
 		nil,
 		nil,
 		nil,
-		nil,
 	}
 }
 
@@ -47,7 +47,14 @@ func (l *Parser) Lex(lval *yySymType) int {
 }
 
 func (l *Parser) Error(msg string) {
-	l.errors = append(l.errors, errors.NewError(msg, l.currentToken))
+	pos := &position.Position{
+		StartLine: l.currentToken.StartLine,
+		EndLine:   l.currentToken.EndLine,
+		StartPos:  l.currentToken.StartPos,
+		EndPos:    l.currentToken.EndPos,
+	}
+
+	l.Lexer.Errors = append(l.Lexer.Errors, errors.NewError(msg, pos))
 }
 
 func (l *Parser) WithMeta() {
@@ -57,7 +64,7 @@ func (l *Parser) WithMeta() {
 // Parse the php7 Parser entrypoint
 func (l *Parser) Parse() int {
 	// init
-	l.errors = nil
+	l.Lexer.Errors = nil
 	l.rootNode = nil
 	l.positionBuilder = &parser.PositionBuilder{}
 
@@ -78,7 +85,7 @@ func (l *Parser) GetRootNode() node.Node {
 
 // GetErrors returns errors list
 func (l *Parser) GetErrors() []*errors.Error {
-	return l.errors
+	return l.Lexer.Errors
 }
 
 // helpers

php7/php7.go

@@ -1,3 +1,5 @@
+// Code generated by goyacc -o php7/php7.go php7/php7.y. DO NOT EDIT.
+
 //line php7/php7.y:2
 package php7
 

php7/php7_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/kylelemons/godebug/pretty"
+	"github.com/z7zmey/php-parser/errors"
 	"github.com/z7zmey/php-parser/node"
 	"github.com/z7zmey/php-parser/node/expr"
 	"github.com/z7zmey/php-parser/node/expr/assign"
@@ -16432,3 +16433,23 @@ CAD;
 	actual := php7parser.GetRootNode()
 	assertEqual(t, expected, actual)
 }
+
+func TestPhp7ControlCharsErrors(t *testing.T) {
+	src := "<?php \004 echo $b; \"$a[\005test]\";"
+
+	expected := []*errors.Error{
+		{
+			Msg: "WARNING: Unexpected character in input: '\004' (ASCII=4)",
+			Pos: &position.Position{1, 1, 7, 7},
+		},
+		{
+			Msg: "WARNING: Unexpected character in input: '\005' (ASCII=5)",
+			Pos: &position.Position{1, 1, 22, 22},
+		},
+	}
+
+	php7parser := php7.NewParser(bytes.NewBufferString(src), "test.php")
+	php7parser.Parse()
+	actual := php7parser.GetErrors()
+	assertEqual(t, expected, actual)
+}

scanner/lexer.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"unicode"
 
+	"github.com/z7zmey/php-parser/errors"
 	"github.com/z7zmey/php-parser/position"
 
 	"github.com/cznic/golex/lex"
@@ -38,6 +39,7 @@ type Lexer struct {
 	TokenPool     *TokenPool
 	WithMeta      bool
 	lastToken     *Token
+	Errors        []*errors.Error
 }
 
 // Rune2Class returns the rune integer id
@@ -79,6 +81,21 @@ func NewLexer(src io.Reader, fName string) *Lexer {
 	}
 }
 
+func (l *Lexer) Error(msg string) {
+	chars := l.Token()
+	firstChar := chars[0]
+	lastChar := chars[len(chars)-1]
+
+	pos := position.NewPosition(
+		l.File.Line(firstChar.Pos()),
+		l.File.Line(lastChar.Pos()),
+		int(firstChar.Pos()),
+		int(lastChar.Pos()),
+	)
+
+	l.Errors = append(l.Errors, errors.NewError(msg, pos))
+}
+
 func (l *Lexer) ungetChars(n int) []lex.Char {
 	l.Unget(l.Lookahead())
 

scanner/scanner.l

@@ -55,6 +55,7 @@ EXPONENT_DNUM   (({LNUM}|{DNUM})[eE][+-]?{LNUM})
 VAR_NAME        [a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*
 OPERATORS       [;:,.\[\]()|\/\^&\+-*=%!~$<>?@]
 NEW_LINE        (\r|\n|\r\n)
+ANY_CHAR        .
 
 %%
     c = l.Rule0()
@@ -668,13 +669,15 @@ NEW_LINE        (\r|\n|\r\n)
 <STRING_VAR_INDEX>\]                        l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
 <STRING_VAR_INDEX>[ \n\r\t\\'#]             l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return int(T_ENCAPSED_AND_WHITESPACE)
 <STRING_VAR_INDEX>{OPERATORS}               lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
-<STRING_VAR_INDEX>.                         lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
+<STRING_VAR_INDEX>{ANY_CHAR}                l.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", l.TokenBytes(nil)[0], l.TokenBytes(nil)[0]));l.Abort();
 
 <STRING_VAR_NAME>{VAR_NAME}[\[\}]           l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_STRING_VARNAME)
 <STRING_VAR_NAME>.                          l.ungetChars(1);l.popState();l.pushState(PHP)
 
 <HALT_COMPILER>.|[ \t\n\r]                  l.addMeta(meta.TokenType, l.Token())
 
+<PHP>{ANY_CHAR}                             l.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", l.TokenBytes(nil)[0], l.TokenBytes(nil)[0]));l.Abort();
+
 %%
     if _, ok := l.Abort(); ok {
         // always return same $end token

scanner/scanner_test.go

@@ -1388,3 +1388,52 @@ func TestYieldFromTokens(t *testing.T) {
 	actual = lv.Tkn.Meta
 	assertEqual(t, expected, actual)
 }
+
+func TestIgnoreControllCharacters(t *testing.T) {
+	src := "<?php \004 echo $b;"
+
+	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
+	lv := &lval{}
+
+	expected := "echo"
+	lexer.Lex(lv)
+	actual := lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "$b"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+}
+
+func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) {
+	src := "<?php \"$a[test\004]\";"
+
+	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
+	lv := &lval{}
+
+	expected := "\""
+	lexer.Lex(lv)
+	actual := lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "$a"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "["
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "test"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+
+	expected = "]"
+	lexer.Lex(lv)
+	actual = lv.Tkn.Value
+	assertEqual(t, expected, actual)
+}