maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[refactoring] update Token structure

Browse Source
This commit is contained in:
Vadym Slizov 2020-12-08 01:23:48 +02:00
parent 20a42da7c9
commit 8064d940f0
No known key found for this signature in database
GPG Key ID: AEA2A9388EF42A4A
6 changed files with 59 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
internal/scanner/lexer.go
View File

@ -15,7 +15,6 @@ type Lexer struct {
phpVersion string
errHandlerFunc func(*errors.Error)
sts, ste int
p, pe, cs int
ts, te, act int
stack []int
@ -57,26 +56,18 @@ func (lex *Lexer) setTokenPosition(token *token.Token) {
token.Position = pos
}
func (lex *Lexer) addSkippedToken(t *token.Token, id token.ID, ps, pe int) {
if lex.sts == -1 {
lex.sts = lex.ts
}
lex.ste = lex.te
// TODO remove after parser refactoring
func (lex *Lexer) addFreeFloatingToken(t *token.Token, id token.ID, ps, pe int) {
skippedTkn := lex.tokenPool.Get()
skippedTkn.ID = id
skippedTkn.Value = lex.data[ps:pe]
lex.setTokenPosition(skippedTkn)
if t.SkippedTokens == nil {
t.SkippedTokens = make([]*token.Token, 0, 2)
if t.FreeFloating == nil {
t.FreeFloating = make([]*token.Token, 0, 2)
}
t.SkippedTokens = append(t.SkippedTokens, skippedTkn)
t.FreeFloating = append(t.FreeFloating, skippedTkn)
}
func (lex *Lexer) isNotStringVar() bool {

BIN
internal/scanner/scanner.go generated
View File

Binary file not shown.

33
internal/scanner/scanner.rl
View File

@ -26,9 +26,6 @@ func (lex *Lexer) Lex() *token.Token {
tkn := lex.tokenPool.Get()
lex.sts = -1
lex.ste = 0
lblStart := 0
lblEnd := 0
@ -127,7 +124,7 @@ func (lex *Lexer) Lex() *token.Token {
main := |*
"#!" any* :>> newline => {
lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te)
lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
};
any => {
fnext html;
@ -143,12 +140,12 @@ func (lex *Lexer) Lex() *token.Token {
fbreak;
};
'<?' => {
lex.addSkippedToken(tkn, token.T_OPEN_TAG, lex.ts, lex.te)
lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.te)
fnext php;
};
'<?php'i ( [ \t] | newline ) => {
lex.ungetCnt(lex.te - lex.ts - 5)
lex.addSkippedToken(tkn, token.T_OPEN_TAG, lex.ts, lex.ts+5)
lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.ts+5)
fnext php;
};
'<?='i => {
@ -160,7 +157,7 @@ func (lex *Lexer) Lex() *token.Token {
*|;
php := |*
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
'?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;};
';' whitespace_line* '?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;};
@ -320,7 +317,7 @@ func (lex *Lexer) Lex() *token.Token {
('#' | '//') any_line* when is_not_comment_end => {
lex.ungetStr("?>")
lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te)
lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
};
'/*' any_line* :>> '*/' {
isDocComment := false;
@ -329,9 +326,9 @@ func (lex *Lexer) Lex() *token.Token {
}
if isDocComment {
lex.addSkippedToken(tkn, token.T_DOC_COMMENT, lex.ts, lex.te)
lex.addFreeFloatingToken(tkn, token.T_DOC_COMMENT, lex.ts, lex.te)
} else {
lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te)
lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
}
};
@ -378,7 +375,7 @@ func (lex *Lexer) Lex() *token.Token {
*|;
property := |*
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
"->" => {lex.setTokenPosition(tkn); tok = token.T_OBJECT_OPERATOR; fbreak;};
varname => {lex.setTokenPosition(tkn); tok = token.T_STRING; fnext php; fbreak;};
any => {lex.ungetCnt(1); fgoto php;};
@ -474,38 +471,32 @@ func (lex *Lexer) Lex() *token.Token {
*|;
halt_compiller_open_parenthesis := |*
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
"(" => {lex.setTokenPosition(tkn); tok = token.ID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;};
any => {lex.ungetCnt(1); fnext php;};
*|;
halt_compiller_close_parenthesis := |*
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
")" => {lex.setTokenPosition(tkn); tok = token.ID(int(')')); fnext halt_compiller_close_semicolon; fbreak;};
any => {lex.ungetCnt(1); fnext php;};
*|;
halt_compiller_close_semicolon := |*
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
";" => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext halt_compiller_end; fbreak;};
any => {lex.ungetCnt(1); fnext php;};
*|;
halt_compiller_end := |*
any_line* => { lex.addSkippedToken(tkn, token.T_HALT_COMPILER, lex.ts, lex.te); };
any_line* => { lex.addFreeFloatingToken(tkn, token.T_HALT_COMPILER, lex.ts, lex.te); };
*|;
write exec;
}%%
if lex.sts == -1 {
lex.sts = 0
}
tkn.Value = lex.data[lex.ts:lex.te]
tkn.ID = token.ID(tok)
tkn.Skipped = lex.data[lex.sts:lex.ste]
lex.addSkippedToken(tkn, tok, lex.ts, lex.te);
return tkn
}

78
internal/scanner/scanner_test.go
View File

@ -386,8 +386,8 @@ func TestShebang(t *testing.T) {
tkn := lexer.Lex()
assert.Equal(t, tkn.ID, token.T_DNUMBER)
l := len(tkn.SkippedTokens)
for _, tt := range tkn.SkippedTokens[:l-1] {
l := len(tkn.FreeFloating)
for _, tt := range tkn.FreeFloating[:l-1] {
actual = append(actual, string(tt.Value))
}
@ -404,7 +404,7 @@ func TestShebangHtml(t *testing.T) {
tkn := lexer.Lex()
assert.Equal(t, tkn.ID, token.T_INLINE_HTML)
assert.Equal(t, string(tkn.SkippedTokens[0].Value), "#!/usr/bin/env php\n")
assert.Equal(t, string(tkn.FreeFloating[0].Value), "#!/usr/bin/env php\n")
tkn = lexer.Lex()
assert.Equal(t, tkn.ID, token.T_DNUMBER)
@ -1137,8 +1137,8 @@ func TestCommentEnd(t *testing.T) {
tkn := lexer.Lex()
l := len(tkn.SkippedTokens)
actual := tkn.SkippedTokens[:l-1]
l := len(tkn.FreeFloating)
actual := tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1169,8 +1169,8 @@ func TestCommentNewLine(t *testing.T) {
tkn := lexer.Lex()
l := len(tkn.SkippedTokens)
actual := tkn.SkippedTokens[:l-1]
l := len(tkn.FreeFloating)
actual := tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1201,8 +1201,8 @@ func TestCommentNewLine1(t *testing.T) {
tkn := lexer.Lex()
l := len(tkn.SkippedTokens)
actual := tkn.SkippedTokens[:l-1]
l := len(tkn.FreeFloating)
actual := tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1233,8 +1233,8 @@ func TestCommentNewLine2(t *testing.T) {
tkn := lexer.Lex()
l := len(tkn.SkippedTokens)
actual := tkn.SkippedTokens[:l-1]
l := len(tkn.FreeFloating)
actual := tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1266,8 +1266,8 @@ func TestCommentWithPhpEndTag(t *testing.T) {
tkn := lexer.Lex()
l := len(tkn.SkippedTokens)
actual := tkn.SkippedTokens[:l-1]
l := len(tkn.FreeFloating)
actual := tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1299,8 +1299,8 @@ func TestInlineComment(t *testing.T) {
tkn := lexer.Lex()
l := len(tkn.SkippedTokens)
actual := tkn.SkippedTokens[:l-1]
l := len(tkn.FreeFloating)
actual := tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1332,8 +1332,8 @@ func TestInlineComment2(t *testing.T) {
tkn := lexer.Lex()
l := len(tkn.SkippedTokens)
actual := tkn.SkippedTokens[:l-1]
l := len(tkn.FreeFloating)
actual := tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1369,8 +1369,8 @@ func TestEmptyInlineComment(t *testing.T) {
tkn := lexer.Lex()
l := len(tkn.SkippedTokens)
actual := tkn.SkippedTokens[:l-1]
l := len(tkn.FreeFloating)
actual := tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1402,8 +1402,8 @@ func TestEmptyInlineComment2(t *testing.T) {
tkn := lexer.Lex()
l := len(tkn.SkippedTokens)
actual := tkn.SkippedTokens[:l-1]
l := len(tkn.FreeFloating)
actual := tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1428,8 +1428,8 @@ func TestMethodCallTokens(t *testing.T) {
},
}
tkn := lexer.Lex()
l := len(tkn.SkippedTokens)
actual := tkn.SkippedTokens[:l-1]
l := len(tkn.FreeFloating)
actual := tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1442,8 +1442,8 @@ func TestMethodCallTokens(t *testing.T) {
},
}
tkn = lexer.Lex()
l = len(tkn.SkippedTokens)
actual = tkn.SkippedTokens[:l-1]
l = len(tkn.FreeFloating)
actual = tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1456,8 +1456,8 @@ func TestMethodCallTokens(t *testing.T) {
},
}
tkn = lexer.Lex()
l = len(tkn.SkippedTokens)
actual = tkn.SkippedTokens[:l-1]
l = len(tkn.FreeFloating)
actual = tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1470,8 +1470,8 @@ func TestMethodCallTokens(t *testing.T) {
},
}
tkn = lexer.Lex()
l = len(tkn.SkippedTokens)
actual = tkn.SkippedTokens[:l-1]
l = len(tkn.FreeFloating)
actual = tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1484,8 +1484,8 @@ func TestMethodCallTokens(t *testing.T) {
},
}
tkn = lexer.Lex()
l = len(tkn.SkippedTokens)
actual = tkn.SkippedTokens[:l-1]
l = len(tkn.FreeFloating)
actual = tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1498,8 +1498,8 @@ func TestMethodCallTokens(t *testing.T) {
},
}
tkn = lexer.Lex()
l = len(tkn.SkippedTokens)
actual = tkn.SkippedTokens[:l-1]
l = len(tkn.FreeFloating)
actual = tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1512,8 +1512,8 @@ func TestMethodCallTokens(t *testing.T) {
},
}
tkn = lexer.Lex()
l = len(tkn.SkippedTokens)
actual = tkn.SkippedTokens[:l-1]
l = len(tkn.FreeFloating)
actual = tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1537,8 +1537,8 @@ func TestYieldFromTokens(t *testing.T) {
},
}
tkn := lexer.Lex()
l := len(tkn.SkippedTokens)
actual := tkn.SkippedTokens[:l-1]
l := len(tkn.FreeFloating)
actual := tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}
@ -1551,8 +1551,8 @@ func TestYieldFromTokens(t *testing.T) {
},
}
tkn = lexer.Lex()
l = len(tkn.SkippedTokens)
actual = tkn.SkippedTokens[:l-1]
l = len(tkn.FreeFloating)
actual = tkn.FreeFloating[:l-1]
for _, v := range actual {
v.Position = nil
}

3
pkg/ast/visitor/dump.go
View File

@ -113,9 +113,6 @@ func (v *Dump) printToken(key string, t *token.Token) {
v.printIndent(v.indent + 1)
v.print("Value: []byte(" + strconv.Quote(string(t.Value)) + "),\n")
v.printIndent(v.indent + 1)
v.print("Skipped: []byte(" + strconv.Quote(string(t.Skipped)) + "),\n")
v.printIndent(v.indent)
v.print("},\n")
}

3
pkg/token/token.go
View File

@ -150,8 +150,7 @@ type Token struct {
ID ID
Value []byte
Position *position.Position
SkippedTokens []*Token
Skipped []byte
FreeFloating []*Token
}
func (t *Token) GetPosition() *position.Position {