maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[#82] PHP 7.4: numeric literal separator

Browse Source
This commit is contained in:
z7zmey 2019-12-26 23:54:44 +02:00
parent ec6be0d9bd
commit 0e55cb3b25
3 changed files with 8243 additions and 7976 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16071
scanner/scanner.go
View File

File diff suppressed because it is too large Load Diff

44
scanner/scanner.rl
View File

@ -2,6 +2,8 @@ package scanner
import ( import (
"fmt" "fmt"
"strconv"
"strings"
"github.com/z7zmey/php-parser/freefloating" "github.com/z7zmey/php-parser/freefloating"
) )
@ -66,10 +68,10 @@ func (lex *Lexer) Lex(lval Lval) int {
whitespace = [\t\v\f ]; whitespace = [\t\v\f ];
whitespace_line = [\t\v\f ] | newline; whitespace_line = [\t\v\f ] | newline;
lnum = [0-9]+; lnum = [0-9]+('_'[0-9]+)*;
dnum = ( [0-9]* "." [0-9]+ ) | ( [0-9]+ "." [0-9]* ); dnum = (lnum?"." lnum)|(lnum"."lnum?);
hnum = '0x' [0-9a-fA-F]+; hnum = '0x'[0-9a-fA-F]+('_'[0-9a-fA-F]+)*;
bnum = '0b' [01]+; bnum = '0b'[01]+('_'[01]+)*;
exponent_dnum = (lnum | dnum) ('e'|'E') ('+'|'-')? lnum; exponent_dnum = (lnum | dnum) ('e'|'E') ('+'|'-')? lnum;
varname_first = [a-zA-Z_] | (0x0080..0x00FF); varname_first = [a-zA-Z_] | (0x0080..0x00FF);
@ -162,36 +164,38 @@ func (lex *Lexer) Lex(lval Lval) int {
(dnum | exponent_dnum) => {lex.setTokenPosition(token); tok = T_DNUMBER; fbreak;}; (dnum | exponent_dnum) => {lex.setTokenPosition(token); tok = T_DNUMBER; fbreak;};
bnum => { bnum => {
firstNum := 2 s := strings.Replace(string(lex.data[lex.ts+2:lex.te]), "_", "", -1)
for i := lex.ts + 2; i < lex.te; i++ { _, err := strconv.ParseInt(s, 2, 0)
if lex.data[i] == '0' {
firstNum++
}
}
if lex.te - lex.ts - firstNum < 64 { if err == nil {
lex.setTokenPosition(token); tok = T_LNUMBER; fbreak; lex.setTokenPosition(token); tok = T_LNUMBER; fbreak;
} }
lex.setTokenPosition(token); tok = T_DNUMBER; fbreak; lex.setTokenPosition(token); tok = T_DNUMBER; fbreak;
}; };
lnum => { lnum => {
if lex.te - lex.ts < 20 { base := 10
if lex.data[lex.ts] == '0' {
base = 8
}
s := strings.Replace(string(lex.data[lex.ts:lex.te]), "_", "", -1)
_, err := strconv.ParseInt(s, base, 0)
if err == nil {
lex.setTokenPosition(token); tok = T_LNUMBER; fbreak; lex.setTokenPosition(token); tok = T_LNUMBER; fbreak;
} }
lex.setTokenPosition(token); tok = T_DNUMBER; fbreak; lex.setTokenPosition(token); tok = T_DNUMBER; fbreak;
}; };
hnum => { hnum => {
firstNum := lex.ts + 2 s := strings.Replace(string(lex.data[lex.ts+2:lex.te]), "_", "", -1)
for i := lex.ts + 2; i < lex.te; i++ { _, err := strconv.ParseInt(s, 16, 0)
if lex.data[i] == '0' {
firstNum++
}
}
length := lex.te - firstNum if err == nil {
if length < 16 || (length == 16 && lex.data[firstNum] <= '7') {
lex.setTokenPosition(token); tok = T_LNUMBER; fbreak; lex.setTokenPosition(token); tok = T_LNUMBER; fbreak;
} }
lex.setTokenPosition(token); tok = T_DNUMBER; fbreak; lex.setTokenPosition(token); tok = T_DNUMBER; fbreak;
}; };

84
scanner/scanner_test.go
View File

@ -22,20 +22,6 @@ func TestTokens(t *testing.T) {
<?= ?> <?= ?>
<?php <?php
0.1
.1
1e10
.1e10
0b1
0b1111111111111111111111111111111111111111111111111111111111111111
0x007111111111111111
0x8111111111111111
1234567890123456789
12345678901234567890
abstract abstract
array array
as as
@ -214,20 +200,6 @@ func TestTokens(t *testing.T) {
TokenID(int(';')).String(), TokenID(int(';')).String(),
T_INLINE_HTML.String(), T_INLINE_HTML.String(),
T_DNUMBER.String(),
T_DNUMBER.String(),
T_DNUMBER.String(),
T_DNUMBER.String(),
T_LNUMBER.String(),
T_DNUMBER.String(),
T_LNUMBER.String(),
T_DNUMBER.String(),
T_LNUMBER.String(),
T_DNUMBER.String(),
T_ABSTRACT.String(), T_ABSTRACT.String(),
T_ARRAY.String(), T_ARRAY.String(),
T_AS.String(), T_AS.String(),
@ -405,6 +377,62 @@ func TestTokens(t *testing.T) {
assert.DeepEqual(t, expected, actual) assert.DeepEqual(t, expected, actual)
} }
func TestNumberTokens(t *testing.T) {
src := `<?php
0.1
.1
1e10
.1e10
0b01111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111
0b10111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111
0x0_7FFF_FFFF_FFFF_FFFF
0x8111_1111_1111_1111
92233_72036_85477_5807
0_77777_77777_77777_77777_7
92233_72036_85477_5808
0_77777_77777_77777_77777_70
`
expected := []string{
T_DNUMBER.String(),
T_DNUMBER.String(),
T_DNUMBER.String(),
T_DNUMBER.String(),
T_LNUMBER.String(),
T_DNUMBER.String(),
T_LNUMBER.String(),
T_DNUMBER.String(),
T_LNUMBER.String(),
T_LNUMBER.String(),
T_DNUMBER.String(),
T_DNUMBER.String(),
}
lexer := NewLexer([]byte(src))
lexer.WithFreeFloating = true
lv := &lval{}
actual := []string{}
for {
token := lexer.Lex(lv)
if token == 0 {
break
}
actual = append(actual, TokenID(token).String())
}
assert.DeepEqual(t, expected, actual)
}
func TestConstantStrings(t *testing.T) { func TestConstantStrings(t *testing.T) {
src := `<? src := `<?
'str' 'str'