maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[refactoring] update Token structure

Browse Source
This commit is contained in:
Vadym Slizov 2020-12-08 01:23:48 +02:00
parent 20a42da7c9
commit 8064d940f0
No known key found for this signature in database
GPG Key ID: AEA2A9388EF42A4A
6 changed files with 59 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
internal/scanner/lexer.go
View File

@ -15,7 +15,6 @@ type Lexer struct {
phpVersion string phpVersion string
errHandlerFunc func(*errors.Error) errHandlerFunc func(*errors.Error)
sts, ste int
p, pe, cs int p, pe, cs int
ts, te, act int ts, te, act int
stack []int stack []int
@ -57,26 +56,18 @@ func (lex *Lexer) setTokenPosition(token *token.Token) {
token.Position = pos token.Position = pos
} }
func (lex *Lexer) addSkippedToken(t *token.Token, id token.ID, ps, pe int) { func (lex *Lexer) addFreeFloatingToken(t *token.Token, id token.ID, ps, pe int) {
if lex.sts == -1 {
lex.sts = lex.ts
}
lex.ste = lex.te
// TODO remove after parser refactoring
skippedTkn := lex.tokenPool.Get() skippedTkn := lex.tokenPool.Get()
skippedTkn.ID = id skippedTkn.ID = id
skippedTkn.Value = lex.data[ps:pe] skippedTkn.Value = lex.data[ps:pe]
lex.setTokenPosition(skippedTkn) lex.setTokenPosition(skippedTkn)
if t.SkippedTokens == nil { if t.FreeFloating == nil {
t.SkippedTokens = make([]*token.Token, 0, 2) t.FreeFloating = make([]*token.Token, 0, 2)
} }
t.SkippedTokens = append(t.SkippedTokens, skippedTkn) t.FreeFloating = append(t.FreeFloating, skippedTkn)
} }
func (lex *Lexer) isNotStringVar() bool { func (lex *Lexer) isNotStringVar() bool {

BIN
internal/scanner/scanner.go generated
View File

Binary file not shown.

33
internal/scanner/scanner.rl
View File

@ -26,9 +26,6 @@ func (lex *Lexer) Lex() *token.Token {
tkn := lex.tokenPool.Get() tkn := lex.tokenPool.Get()
lex.sts = -1
lex.ste = 0
lblStart := 0 lblStart := 0
lblEnd := 0 lblEnd := 0
@ -127,7 +124,7 @@ func (lex *Lexer) Lex() *token.Token {
main := |* main := |*
"#!" any* :>> newline => { "#!" any* :>> newline => {
lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te) lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
}; };
any => { any => {
fnext html; fnext html;
@ -143,12 +140,12 @@ func (lex *Lexer) Lex() *token.Token {
fbreak; fbreak;
}; };
'<?' => { '<?' => {
lex.addSkippedToken(tkn, token.T_OPEN_TAG, lex.ts, lex.te) lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.te)
fnext php; fnext php;
}; };
'<?php'i ( [ \t] | newline ) => { '<?php'i ( [ \t] | newline ) => {
lex.ungetCnt(lex.te - lex.ts - 5) lex.ungetCnt(lex.te - lex.ts - 5)
lex.addSkippedToken(tkn, token.T_OPEN_TAG, lex.ts, lex.ts+5) lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.ts+5)
fnext php; fnext php;
}; };
'<?='i => { '<?='i => {
@ -160,7 +157,7 @@ func (lex *Lexer) Lex() *token.Token {
*|; *|;
php := |* php := |*
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
'?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;}; '?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;};
';' whitespace_line* '?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;}; ';' whitespace_line* '?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;};
@ -320,7 +317,7 @@ func (lex *Lexer) Lex() *token.Token {
('#' | '//') any_line* when is_not_comment_end => { ('#' | '//') any_line* when is_not_comment_end => {
lex.ungetStr("?>") lex.ungetStr("?>")
lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te) lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
}; };
'/*' any_line* :>> '*/' { '/*' any_line* :>> '*/' {
isDocComment := false; isDocComment := false;
@ -329,9 +326,9 @@ func (lex *Lexer) Lex() *token.Token {
} }
if isDocComment { if isDocComment {
lex.addSkippedToken(tkn, token.T_DOC_COMMENT, lex.ts, lex.te) lex.addFreeFloatingToken(tkn, token.T_DOC_COMMENT, lex.ts, lex.te)
} else { } else {
lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te) lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
} }
}; };
@ -378,7 +375,7 @@ func (lex *Lexer) Lex() *token.Token {
*|; *|;
property := |* property := |*
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
"->" => {lex.setTokenPosition(tkn); tok = token.T_OBJECT_OPERATOR; fbreak;}; "->" => {lex.setTokenPosition(tkn); tok = token.T_OBJECT_OPERATOR; fbreak;};
varname => {lex.setTokenPosition(tkn); tok = token.T_STRING; fnext php; fbreak;}; varname => {lex.setTokenPosition(tkn); tok = token.T_STRING; fnext php; fbreak;};
any => {lex.ungetCnt(1); fgoto php;}; any => {lex.ungetCnt(1); fgoto php;};
@ -474,38 +471,32 @@ func (lex *Lexer) Lex() *token.Token {
*|; *|;
halt_compiller_open_parenthesis := |* halt_compiller_open_parenthesis := |*
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
"(" => {lex.setTokenPosition(tkn); tok = token.ID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;}; "(" => {lex.setTokenPosition(tkn); tok = token.ID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_close_parenthesis := |* halt_compiller_close_parenthesis := |*
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
")" => {lex.setTokenPosition(tkn); tok = token.ID(int(')')); fnext halt_compiller_close_semicolon; fbreak;}; ")" => {lex.setTokenPosition(tkn); tok = token.ID(int(')')); fnext halt_compiller_close_semicolon; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_close_semicolon := |* halt_compiller_close_semicolon := |*
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
";" => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext halt_compiller_end; fbreak;}; ";" => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext halt_compiller_end; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_end := |* halt_compiller_end := |*
any_line* => { lex.addSkippedToken(tkn, token.T_HALT_COMPILER, lex.ts, lex.te); }; any_line* => { lex.addFreeFloatingToken(tkn, token.T_HALT_COMPILER, lex.ts, lex.te); };
*|; *|;
write exec; write exec;
}%% }%%
if lex.sts == -1 {
lex.sts = 0
}
tkn.Value = lex.data[lex.ts:lex.te] tkn.Value = lex.data[lex.ts:lex.te]
tkn.ID = token.ID(tok) tkn.ID = token.ID(tok)
tkn.Skipped = lex.data[lex.sts:lex.ste]
lex.addSkippedToken(tkn, tok, lex.ts, lex.te);
return tkn return tkn
} }

78
internal/scanner/scanner_test.go
View File

@ -386,8 +386,8 @@ func TestShebang(t *testing.T) {
tkn := lexer.Lex() tkn := lexer.Lex()
assert.Equal(t, tkn.ID, token.T_DNUMBER) assert.Equal(t, tkn.ID, token.T_DNUMBER)
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
for _, tt := range tkn.SkippedTokens[:l-1] { for _, tt := range tkn.FreeFloating[:l-1] {
actual = append(actual, string(tt.Value)) actual = append(actual, string(tt.Value))
} }
@ -404,7 +404,7 @@ func TestShebangHtml(t *testing.T) {
tkn := lexer.Lex() tkn := lexer.Lex()
assert.Equal(t, tkn.ID, token.T_INLINE_HTML) assert.Equal(t, tkn.ID, token.T_INLINE_HTML)
assert.Equal(t, string(tkn.SkippedTokens[0].Value), "#!/usr/bin/env php\n") assert.Equal(t, string(tkn.FreeFloating[0].Value), "#!/usr/bin/env php\n")
tkn = lexer.Lex() tkn = lexer.Lex()
assert.Equal(t, tkn.ID, token.T_DNUMBER) assert.Equal(t, tkn.ID, token.T_DNUMBER)
@ -1137,8 +1137,8 @@ func TestCommentEnd(t *testing.T) {
tkn := lexer.Lex() tkn := lexer.Lex()
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
actual := tkn.SkippedTokens[:l-1] actual := tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1169,8 +1169,8 @@ func TestCommentNewLine(t *testing.T) {
tkn := lexer.Lex() tkn := lexer.Lex()
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
actual := tkn.SkippedTokens[:l-1] actual := tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1201,8 +1201,8 @@ func TestCommentNewLine1(t *testing.T) {
tkn := lexer.Lex() tkn := lexer.Lex()
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
actual := tkn.SkippedTokens[:l-1] actual := tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1233,8 +1233,8 @@ func TestCommentNewLine2(t *testing.T) {
tkn := lexer.Lex() tkn := lexer.Lex()
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
actual := tkn.SkippedTokens[:l-1] actual := tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1266,8 +1266,8 @@ func TestCommentWithPhpEndTag(t *testing.T) {
tkn := lexer.Lex() tkn := lexer.Lex()
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
actual := tkn.SkippedTokens[:l-1] actual := tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1299,8 +1299,8 @@ func TestInlineComment(t *testing.T) {
tkn := lexer.Lex() tkn := lexer.Lex()
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
actual := tkn.SkippedTokens[:l-1] actual := tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1332,8 +1332,8 @@ func TestInlineComment2(t *testing.T) {
tkn := lexer.Lex() tkn := lexer.Lex()
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
actual := tkn.SkippedTokens[:l-1] actual := tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1369,8 +1369,8 @@ func TestEmptyInlineComment(t *testing.T) {
tkn := lexer.Lex() tkn := lexer.Lex()
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
actual := tkn.SkippedTokens[:l-1] actual := tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1402,8 +1402,8 @@ func TestEmptyInlineComment2(t *testing.T) {
tkn := lexer.Lex() tkn := lexer.Lex()
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
actual := tkn.SkippedTokens[:l-1] actual := tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1428,8 +1428,8 @@ func TestMethodCallTokens(t *testing.T) {
}, },
} }
tkn := lexer.Lex() tkn := lexer.Lex()
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
actual := tkn.SkippedTokens[:l-1] actual := tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1442,8 +1442,8 @@ func TestMethodCallTokens(t *testing.T) {
}, },
} }
tkn = lexer.Lex() tkn = lexer.Lex()
l = len(tkn.SkippedTokens) l = len(tkn.FreeFloating)
actual = tkn.SkippedTokens[:l-1] actual = tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1456,8 +1456,8 @@ func TestMethodCallTokens(t *testing.T) {
}, },
} }
tkn = lexer.Lex() tkn = lexer.Lex()
l = len(tkn.SkippedTokens) l = len(tkn.FreeFloating)
actual = tkn.SkippedTokens[:l-1] actual = tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1470,8 +1470,8 @@ func TestMethodCallTokens(t *testing.T) {
}, },
} }
tkn = lexer.Lex() tkn = lexer.Lex()
l = len(tkn.SkippedTokens) l = len(tkn.FreeFloating)
actual = tkn.SkippedTokens[:l-1] actual = tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1484,8 +1484,8 @@ func TestMethodCallTokens(t *testing.T) {
}, },
} }
tkn = lexer.Lex() tkn = lexer.Lex()
l = len(tkn.SkippedTokens) l = len(tkn.FreeFloating)
actual = tkn.SkippedTokens[:l-1] actual = tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1498,8 +1498,8 @@ func TestMethodCallTokens(t *testing.T) {
}, },
} }
tkn = lexer.Lex() tkn = lexer.Lex()
l = len(tkn.SkippedTokens) l = len(tkn.FreeFloating)
actual = tkn.SkippedTokens[:l-1] actual = tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1512,8 +1512,8 @@ func TestMethodCallTokens(t *testing.T) {
}, },
} }
tkn = lexer.Lex() tkn = lexer.Lex()
l = len(tkn.SkippedTokens) l = len(tkn.FreeFloating)
actual = tkn.SkippedTokens[:l-1] actual = tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1537,8 +1537,8 @@ func TestYieldFromTokens(t *testing.T) {
}, },
} }
tkn := lexer.Lex() tkn := lexer.Lex()
l := len(tkn.SkippedTokens) l := len(tkn.FreeFloating)
actual := tkn.SkippedTokens[:l-1] actual := tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }
@ -1551,8 +1551,8 @@ func TestYieldFromTokens(t *testing.T) {
}, },
} }
tkn = lexer.Lex() tkn = lexer.Lex()
l = len(tkn.SkippedTokens) l = len(tkn.FreeFloating)
actual = tkn.SkippedTokens[:l-1] actual = tkn.FreeFloating[:l-1]
for _, v := range actual { for _, v := range actual {
v.Position = nil v.Position = nil
} }

3
pkg/ast/visitor/dump.go
View File

@ -113,9 +113,6 @@ func (v *Dump) printToken(key string, t *token.Token) {
v.printIndent(v.indent + 1) v.printIndent(v.indent + 1)
v.print("Value: []byte(" + strconv.Quote(string(t.Value)) + "),\n") v.print("Value: []byte(" + strconv.Quote(string(t.Value)) + "),\n")
v.printIndent(v.indent + 1)
v.print("Skipped: []byte(" + strconv.Quote(string(t.Skipped)) + "),\n")
v.printIndent(v.indent) v.printIndent(v.indent)
v.print("},\n") v.print("},\n")
} }

9
pkg/token/token.go
View File

@ -147,11 +147,10 @@ const (
) )
type Token struct { type Token struct {
ID ID ID ID
Value []byte Value []byte
Position *position.Position Position *position.Position
SkippedTokens []*Token FreeFloating []*Token
Skipped []byte
} }
func (t *Token) GetPosition() *position.Position { func (t *Token) GetPosition() *position.Position {