php8.1: added new octal numbers syntax (#10)

Also fixed a bug where `0X...` and `0B...` were not recognized as valid numbers.

internal/php8/scanner.rl

@@ -58,8 +58,9 @@ func (lex *Lexer) Lex() *token.Token {
 
         lnum = [0-9]+('_'[0-9]+)*;
         dnum = (lnum?"." lnum)|(lnum"."lnum?);
-        hnum = '0x'[0-9a-fA-F]+('_'[0-9a-fA-F]+)*;
+        hnum = '0x'i[0-9a-fA-F]+('_'[0-9a-fA-F]+)*;
-        bnum = '0b'[01]+('_'[01]+)*;
+        bnum = '0b'i[01]+('_'[01]+)*;
+        onum = '0o'i[0-7]+('_'[0-7]+)*;
 
         exponent_dnum = (lnum | dnum) ('e'|'E') ('+'|'-')? lnum;
         varname_first = [a-zA-Z_] | (0x0080..0x00FF);
@@ -164,7 +165,7 @@ func (lex *Lexer) Lex() *token.Token {
 
             (dnum | exponent_dnum)          => {lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;};
             bnum => {
-                s := strings.Replace(string(lex.data[lex.ts+2:lex.te]), "_", "", -1)
+                s := strings.ReplaceAll(string(lex.data[lex.ts+2:lex.te]), "_", "")
                 _, err := strconv.ParseInt(s, 2, 0)
 
                 if err == nil {
@@ -179,7 +180,7 @@ func (lex *Lexer) Lex() *token.Token {
                     base = 8
                 }
 
-                s := strings.Replace(string(lex.data[lex.ts:lex.te]), "_", "", -1)
+                s := strings.ReplaceAll(string(lex.data[lex.ts:lex.te]), "_", "")
                 _, err := strconv.ParseInt(s, base, 0)
 
                 if err == nil {
@@ -189,7 +190,7 @@ func (lex *Lexer) Lex() *token.Token {
                 lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;
             };
             hnum => {
-                s := strings.Replace(string(lex.data[lex.ts+2:lex.te]), "_", "", -1)
+                s := strings.ReplaceAll(string(lex.data[lex.ts+2:lex.te]), "_", "")
                 _, err := strconv.ParseInt(s, 16, 0)
 
                 if err == nil {
@@ -198,6 +199,16 @@ func (lex *Lexer) Lex() *token.Token {
 
                 lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;
             };
+            onum => {
+                s := strings.ReplaceAll(string(lex.data[lex.ts+2:lex.te]), "_", "")
+                _, err := strconv.ParseInt(s, 8, 0)
+
+                if err == nil {
+                    lex.setTokenPosition(tkn); tok = token.T_LNUMBER; fbreak;
+                }
+
+                lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;
+            };
 
             'namespace'i ('\\' varname)+      => {lex.setTokenPosition(tkn); tok = token.T_NAME_RELATIVE; fbreak;};
             varname ('\\' varname)+           => {lex.setTokenPosition(tkn); tok = token.T_NAME_QUALIFIED; fbreak;};
@@ -468,12 +479,12 @@ func (lex *Lexer) Lex() *token.Token {
         *|;
 
         string_var_index := |*
-            lnum | hnum | bnum       => {lex.setTokenPosition(tkn); tok = token.T_NUM_STRING; fbreak;};
+            lnum | hnum | bnum | onum => {lex.setTokenPosition(tkn); tok = token.T_NUM_STRING; fbreak;};
-            '$' varname              => {lex.setTokenPosition(tkn); tok = token.T_VARIABLE; fbreak;};
+            '$' varname               => {lex.setTokenPosition(tkn); tok = token.T_VARIABLE; fbreak;};
-            varname                  => {lex.setTokenPosition(tkn); tok = token.T_STRING; fbreak;};
+            varname                   => {lex.setTokenPosition(tkn); tok = token.T_STRING; fbreak;};
-            whitespace_line | [\\'#] => {lex.setTokenPosition(tkn); tok = token.T_ENCAPSED_AND_WHITESPACE; lex.ret(2); goto _out;};
+            whitespace_line | [\\'#]  => {lex.setTokenPosition(tkn); tok = token.T_ENCAPSED_AND_WHITESPACE; lex.ret(2); goto _out;};
-            operators > (svi, 1)     => {lex.setTokenPosition(tkn); tok = token.ID(int(lex.data[lex.ts])); fbreak;};
+            operators > (svi, 1)      => {lex.setTokenPosition(tkn); tok = token.ID(int(lex.data[lex.ts])); fbreak;};
-            ']'       > (svi, 2)     => {lex.setTokenPosition(tkn); tok = token.ID(int(']')); lex.ret(2); goto _out;};
+            ']'       > (svi, 2)      => {lex.setTokenPosition(tkn); tok = token.ID(int(']')); lex.ret(2); goto _out;};
             any_line => {
                 c := lex.data[lex.p]
                 lex.error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c));

internal/php8/scanner_php8_1_test.go

@@ -31,3 +31,127 @@ func TestReadonlyTokens(t *testing.T) {
 	}
 	suite.Run()
 }
+
+func TestNumberTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStructTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = `<?php
+0x10;
+0X10;
+0b10;
+0B10;
+0o10;
+0O10;
+`
+	suite.Expected = []*token.Token{
+		{
+			ID:    token.T_LNUMBER,
+			Value: []byte("0x10"),
+		},
+		{
+			ID:    ';',
+			Value: []byte(";"),
+		},
+		{
+			ID:    token.T_LNUMBER,
+			Value: []byte("0X10"),
+		},
+		{
+			ID:    ';',
+			Value: []byte(";"),
+		},
+		{
+			ID:    token.T_LNUMBER,
+			Value: []byte("0b10"),
+		},
+		{
+			ID:    ';',
+			Value: []byte(";"),
+		},
+		{
+			ID:    token.T_LNUMBER,
+			Value: []byte("0B10"),
+		},
+		{
+			ID:    ';',
+			Value: []byte(";"),
+		},
+		{
+			ID:    token.T_LNUMBER,
+			Value: []byte("0o10"),
+		},
+		{
+			ID:    ';',
+			Value: []byte(";"),
+		},
+		{
+			ID:    token.T_LNUMBER,
+			Value: []byte("0O10"),
+		},
+		{
+			ID:    ';',
+			Value: []byte(";"),
+		},
+	}
+	suite.Run()
+}
+
+func TestNumberStringTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStructTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = `<?php
+"$a[0o10]"
+"$a[0O10]"
+`
+	suite.Expected = []*token.Token{
+		{
+			ID:    '"',
+			Value: []byte("\""),
+		},
+		{
+			ID:    token.T_VARIABLE,
+			Value: []byte("$a"),
+		},
+		{
+			ID:    '[',
+			Value: []byte("["),
+		},
+		{
+			ID:    token.T_NUM_STRING,
+			Value: []byte("0o10"),
+		},
+		{
+			ID:    ']',
+			Value: []byte("]"),
+		},
+		{
+			ID:    '"',
+			Value: []byte("\""),
+		},
+		{
+			ID:    '"',
+			Value: []byte("\""),
+		},
+		{
+			ID:    token.T_VARIABLE,
+			Value: []byte("$a"),
+		},
+		{
+			ID:    '[',
+			Value: []byte("["),
+		},
+		{
+			ID:    token.T_NUM_STRING,
+			Value: []byte("0O10"),
+		},
+		{
+			ID:    ']',
+			Value: []byte("]"),
+		},
+		{
+			ID:    '"',
+			Value: []byte("\""),
+		},
+	}
+	suite.Run()
+}

pkg/visitor/printer/printer_php8_1_test.go

@@ -29,3 +29,14 @@ func TestNeverTypePHP81(t *testing.T) {
 function f(): never {}
 `)
 }
+
+func TestNumbersPHP81(t *testing.T) {
+	tester.NewParserPrintTestSuite(t).UsePHP8().Run(`<?php
+echo 0x10;
+echo 0X10;
+echo 0b10;
+echo 0B10;
+echo 0o10;
+echo 0O10;
+`)
+}