distinguish unknown unicode characters from EOF

This commit is contained in:
Imuli 2018-07-14 08:03:49 -04:00
parent 6d554c0468
commit 3ba2d2a91b

View File

@ -462,8 +462,10 @@ func Rune2Class(r rune) int {
if unicode.IsGraphic(r) {
return classUnicodeGraphic
}
// return classOther
return -1
if r == lex.RuneEOF {
return int(r)
}
return classOther
}
// NewLexer the Lexer constructor