maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

php8.1: added new octal numbers syntax (#10)

Browse Source 
Also fixed a bug where `0X...` and `0B...` were not recognized as valid numbers.
This commit is contained in:
Makhnev Petr 2021-07-31 18:37:01 +03:00 committed by GitHub
parent 8df80651e0
commit 44bbff6073
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 10595 additions and 10291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20718
internal/php8/scanner.go
View File

File diff suppressed because it is too large Load Diff

33
internal/php8/scanner.rl
View File

@ -58,8 +58,9 @@ func (lex *Lexer) Lex() *token.Token {
lnum = [0-9]+('_'[0-9]+)*;
dnum = (lnum?"." lnum)|(lnum"."lnum?);
hnum = '0x'[0-9a-fA-F]+('_'[0-9a-fA-F]+)*;
bnum = '0b'[01]+('_'[01]+)*;
hnum = '0x'i[0-9a-fA-F]+('_'[0-9a-fA-F]+)*;
bnum = '0b'i[01]+('_'[01]+)*;
onum = '0o'i[0-7]+('_'[0-7]+)*;
exponent_dnum = (lnum | dnum) ('e'|'E') ('+'|'-')? lnum;
varname_first = [a-zA-Z_] | (0x0080..0x00FF);
@ -164,7 +165,7 @@ func (lex *Lexer) Lex() *token.Token {
(dnum | exponent_dnum) => {lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;};
bnum => {
s := strings.Replace(string(lex.data[lex.ts+2:lex.te]), "_", "", -1)
s := strings.ReplaceAll(string(lex.data[lex.ts+2:lex.te]), "_", "")
_, err := strconv.ParseInt(s, 2, 0)
if err == nil {
@ -179,7 +180,7 @@ func (lex *Lexer) Lex() *token.Token {
base = 8
}
s := strings.Replace(string(lex.data[lex.ts:lex.te]), "_", "", -1)
s := strings.ReplaceAll(string(lex.data[lex.ts:lex.te]), "_", "")
_, err := strconv.ParseInt(s, base, 0)
if err == nil {
@ -189,7 +190,7 @@ func (lex *Lexer) Lex() *token.Token {
lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;
};
hnum => {
s := strings.Replace(string(lex.data[lex.ts+2:lex.te]), "_", "", -1)
s := strings.ReplaceAll(string(lex.data[lex.ts+2:lex.te]), "_", "")
_, err := strconv.ParseInt(s, 16, 0)
if err == nil {
@ -198,6 +199,16 @@ func (lex *Lexer) Lex() *token.Token {
lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;
};
onum => {
s := strings.ReplaceAll(string(lex.data[lex.ts+2:lex.te]), "_", "")
_, err := strconv.ParseInt(s, 8, 0)
if err == nil {
lex.setTokenPosition(tkn); tok = token.T_LNUMBER; fbreak;
}
lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;
};
'namespace'i ('\\' varname)+ => {lex.setTokenPosition(tkn); tok = token.T_NAME_RELATIVE; fbreak;};
varname ('\\' varname)+ => {lex.setTokenPosition(tkn); tok = token.T_NAME_QUALIFIED; fbreak;};
@ -468,12 +479,12 @@ func (lex *Lexer) Lex() *token.Token {
*|;
string_var_index := |*
lnum | hnum | bnum => {lex.setTokenPosition(tkn); tok = token.T_NUM_STRING; fbreak;};
'$' varname => {lex.setTokenPosition(tkn); tok = token.T_VARIABLE; fbreak;};
varname => {lex.setTokenPosition(tkn); tok = token.T_STRING; fbreak;};
whitespace_line | [\\'#] => {lex.setTokenPosition(tkn); tok = token.T_ENCAPSED_AND_WHITESPACE; lex.ret(2); goto _out;};
operators > (svi, 1) => {lex.setTokenPosition(tkn); tok = token.ID(int(lex.data[lex.ts])); fbreak;};
']' > (svi, 2) => {lex.setTokenPosition(tkn); tok = token.ID(int(']')); lex.ret(2); goto _out;};
lnum | hnum | bnum | onum => {lex.setTokenPosition(tkn); tok = token.T_NUM_STRING; fbreak;};
'$' varname => {lex.setTokenPosition(tkn); tok = token.T_VARIABLE; fbreak;};
varname => {lex.setTokenPosition(tkn); tok = token.T_STRING; fbreak;};
whitespace_line | [\\'#] => {lex.setTokenPosition(tkn); tok = token.T_ENCAPSED_AND_WHITESPACE; lex.ret(2); goto _out;};
operators > (svi, 1) => {lex.setTokenPosition(tkn); tok = token.ID(int(lex.data[lex.ts])); fbreak;};
']' > (svi, 2) => {lex.setTokenPosition(tkn); tok = token.ID(int(']')); lex.ret(2); goto _out;};
any_line => {
c := lex.data[lex.p]
lex.error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c));

124
internal/php8/scanner_php8_1_test.go
View File

@ -31,3 +31,127 @@ func TestReadonlyTokens(t *testing.T) {
}
suite.Run()
}
func TestNumberTokens(t *testing.T) {
suite := tester.NewLexerTokenStructTestSuite(t)
suite.UsePHP8()
suite.Code = `<?php
0x10;
0X10;
0b10;
0B10;
0o10;
0O10;
`
suite.Expected = []*token.Token{
{
ID: token.T_LNUMBER,
Value: []byte("0x10"),
},
{
ID: ';',
Value: []byte(";"),
},
{
ID: token.T_LNUMBER,
Value: []byte("0X10"),
},
{
ID: ';',
Value: []byte(";"),
},
{
ID: token.T_LNUMBER,
Value: []byte("0b10"),
},
{
ID: ';',
Value: []byte(";"),
},
{
ID: token.T_LNUMBER,
Value: []byte("0B10"),
},
{
ID: ';',
Value: []byte(";"),
},
{
ID: token.T_LNUMBER,
Value: []byte("0o10"),
},
{
ID: ';',
Value: []byte(";"),
},
{
ID: token.T_LNUMBER,
Value: []byte("0O10"),
},
{
ID: ';',
Value: []byte(";"),
},
}
suite.Run()
}
func TestNumberStringTokens(t *testing.T) {
suite := tester.NewLexerTokenStructTestSuite(t)
suite.UsePHP8()
suite.Code = `<?php
"$a[0o10]"
"$a[0O10]"
`
suite.Expected = []*token.Token{
{
ID: '"',
Value: []byte("\""),
},
{
ID: token.T_VARIABLE,
Value: []byte("$a"),
},
{
ID: '[',
Value: []byte("["),
},
{
ID: token.T_NUM_STRING,
Value: []byte("0o10"),
},
{
ID: ']',
Value: []byte("]"),
},
{
ID: '"',
Value: []byte("\""),
},
{
ID: '"',
Value: []byte("\""),
},
{
ID: token.T_VARIABLE,
Value: []byte("$a"),
},
{
ID: '[',
Value: []byte("["),
},
{
ID: token.T_NUM_STRING,
Value: []byte("0O10"),
},
{
ID: ']',
Value: []byte("]"),
},
{
ID: '"',
Value: []byte("\""),
},
}
suite.Run()
}

11
pkg/visitor/printer/printer_php8_1_test.go
View File

@ -29,3 +29,14 @@ func TestNeverTypePHP81(t *testing.T) {
function f(): never {}
`)
}
func TestNumbersPHP81(t *testing.T) {
tester.NewParserPrintTestSuite(t).UsePHP8().Run(`<?php
echo 0x10;
echo 0X10;
echo 0b10;
echo 0B10;
echo 0o10;
echo 0O10;
`)
}