Merge branch 'ganlvtech-dev' into dev

scanner/scanner.rl

@@ -58,8 +58,9 @@ func (lex *Lexer) Lex(lval Lval) int {
         bnum = '0b' [01]+;
 
         exponent_dnum = (lnum | dnum) ('e'|'E') ('+'|'-')? lnum;
-        varname       = /[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/;
-        varname_first = /[a-zA-Z_\x7f-\xff]/;
+        varname_first = [a-zA-Z_] | (0x0080..0x00FF);
+        varname_second = varname_first | [0-9];
+        varname       = varname_first (varname_second)*;
         heredoc_label = varname >heredoc_lbl_start %heredoc_lbl_end;
         operators     = ';'|':'|','|'.'|'['|']'|'('|')'|'|'|'/'|'^'|'&'|'+'|'-'|'*'|'='|'%'|'!'|'~'|'$'|'<'|'>'|'?'|'@';
         

scanner/scanner_test.go

@@ -1375,6 +1375,19 @@ func TestYieldFromTokens(t *testing.T) {
 	assert.DeepEqual(t, expected, actual)
 }
 
+func TestVarNameByteChars(t *testing.T) {
+	src := "<?php $\x80 $\xff"
+
+	lexer := NewLexer([]byte(src))
+	lv := &lval{}
+
+	lexer.Lex(lv)
+	assert.Equal(t, "$\x80", lv.Tkn.Value)
+
+	lexer.Lex(lv)
+	assert.Equal(t, "$\xff", lv.Tkn.Value)
+}
+
 func TestIgnoreControllCharacters(t *testing.T) {
 	src := "<?php \004 echo $b;"