[#96] handle lexer errors

scanner/lexer.go

@@ -4,6 +4,7 @@ package scanner
 import (
 	"bufio"
 	"bytes"
+	"go/token"
 	t "go/token"
 	"io"
 	"unicode"
@@ -62,23 +63,31 @@ func Rune2Class(r rune) int {
 	return classOther
 }
 
-// NewLexer the Lexer constructor
+func (l *Lexer) lexErrorFunc(p token.Pos, msg string) {
-func NewLexer(src io.Reader, fName string) *Lexer {
+	pos := position.NewPosition(
-	file := t.NewFileSet().AddFile(fName, -1, 1<<31-3)
+		l.File.Line(p),
-	lx, err := lex.New(file, bufio.NewReader(src), lex.RuneClass(Rune2Class))
+		l.File.Line(p),
-	if err != nil {
+		int(p),
-		panic(err)
+		int(p),
+	)
+	l.Errors = append(l.Errors, errors.NewError(msg, pos))
 }
 
-	return &Lexer{
+// NewLexer the Lexer constructor
-		Lexer:         lx,
+func NewLexer(src io.Reader, fName string) *Lexer {
+	lexer := &Lexer{
 		StateStack:    []int{0},
-		PhpDocComment: "",
-		FreeFloating:  nil,
-		heredocLabel:  "",
 		tokenBytesBuf: &bytes.Buffer{},
 		TokenPool:     &TokenPool{},
 	}
+
+	file := t.NewFileSet().AddFile(fName, -1, 1<<31-3)
+	lx, err := lex.New(file, bufio.NewReader(src), lex.RuneClass(Rune2Class), lex.ErrorFunc(lexer.lexErrorFunc))
+	if err != nil {
+		panic(err)
+	}
+	lexer.Lexer = lx
+	return lexer
 }
 
 func (l *Lexer) Error(msg string) {

scanner/scanner_test.go

@@ -1420,3 +1420,15 @@ func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) {
 	actual = lv.Tkn.Value
 	assert.DeepEqual(t, expected, actual)
 }
+
+func TestBomInMiddleOfFile(t *testing.T) {
+	src := "<?php \xEF\xBB\xBF $a;"
+
+	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
+	lv := &lval{}
+
+	lexer.Lex(lv)
+
+	assert.Assert(t, len(lexer.Errors) > 0)
+	assert.Assert(t, lexer.Errors[0].String() == "unicode (UTF-8) BOM in middle of file at line 1")
+}