[#96] handle lexer errors

This commit is contained in:
z7zmey 2019-08-20 21:51:45 +03:00
parent 5110c36dca
commit c97ca17c6a
2 changed files with 32 additions and 11 deletions

View File

@ -4,6 +4,7 @@ package scanner
import ( import (
"bufio" "bufio"
"bytes" "bytes"
"go/token"
t "go/token" t "go/token"
"io" "io"
"unicode" "unicode"
@ -62,23 +63,31 @@ func Rune2Class(r rune) int {
return classOther return classOther
} }
// NewLexer the Lexer constructor func (l *Lexer) lexErrorFunc(p token.Pos, msg string) {
func NewLexer(src io.Reader, fName string) *Lexer { pos := position.NewPosition(
file := t.NewFileSet().AddFile(fName, -1, 1<<31-3) l.File.Line(p),
lx, err := lex.New(file, bufio.NewReader(src), lex.RuneClass(Rune2Class)) l.File.Line(p),
if err != nil { int(p),
panic(err) int(p),
)
l.Errors = append(l.Errors, errors.NewError(msg, pos))
} }
return &Lexer{ // NewLexer the Lexer constructor
Lexer: lx, func NewLexer(src io.Reader, fName string) *Lexer {
lexer := &Lexer{
StateStack: []int{0}, StateStack: []int{0},
PhpDocComment: "",
FreeFloating: nil,
heredocLabel: "",
tokenBytesBuf: &bytes.Buffer{}, tokenBytesBuf: &bytes.Buffer{},
TokenPool: &TokenPool{}, TokenPool: &TokenPool{},
} }
file := t.NewFileSet().AddFile(fName, -1, 1<<31-3)
lx, err := lex.New(file, bufio.NewReader(src), lex.RuneClass(Rune2Class), lex.ErrorFunc(lexer.lexErrorFunc))
if err != nil {
panic(err)
}
lexer.Lexer = lx
return lexer
} }
func (l *Lexer) Error(msg string) { func (l *Lexer) Error(msg string) {

View File

@ -1420,3 +1420,15 @@ func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) {
actual = lv.Tkn.Value actual = lv.Tkn.Value
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
} }
func TestBomInMiddleOfFile(t *testing.T) {
src := "<?php \xEF\xBB\xBF $a;"
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lv := &lval{}
lexer.Lex(lv)
assert.Assert(t, len(lexer.Errors) > 0)
assert.Assert(t, lexer.Errors[0].String() == "unicode (UTF-8) BOM in middle of file at line 1")
}