diff --git a/internal/scanner/lexer.go b/internal/scanner/lexer.go index 7986acb..8c0381f 100644 --- a/internal/scanner/lexer.go +++ b/internal/scanner/lexer.go @@ -15,7 +15,6 @@ type Lexer struct { phpVersion string errHandlerFunc func(*errors.Error) - sts, ste int p, pe, cs int ts, te, act int stack []int @@ -57,26 +56,18 @@ func (lex *Lexer) setTokenPosition(token *token.Token) { token.Position = pos } -func (lex *Lexer) addSkippedToken(t *token.Token, id token.ID, ps, pe int) { - if lex.sts == -1 { - lex.sts = lex.ts - } - - lex.ste = lex.te - - // TODO remove after parser refactoring - +func (lex *Lexer) addFreeFloatingToken(t *token.Token, id token.ID, ps, pe int) { skippedTkn := lex.tokenPool.Get() skippedTkn.ID = id skippedTkn.Value = lex.data[ps:pe] lex.setTokenPosition(skippedTkn) - if t.SkippedTokens == nil { - t.SkippedTokens = make([]*token.Token, 0, 2) + if t.FreeFloating == nil { + t.FreeFloating = make([]*token.Token, 0, 2) } - t.SkippedTokens = append(t.SkippedTokens, skippedTkn) + t.FreeFloating = append(t.FreeFloating, skippedTkn) } func (lex *Lexer) isNotStringVar() bool { diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go index 9f779e6..bd7768b 100644 Binary files a/internal/scanner/scanner.go and b/internal/scanner/scanner.go differ diff --git a/internal/scanner/scanner.rl b/internal/scanner/scanner.rl index b3a0015..e7d1179 100644 --- a/internal/scanner/scanner.rl +++ b/internal/scanner/scanner.rl @@ -26,9 +26,6 @@ func (lex *Lexer) Lex() *token.Token { tkn := lex.tokenPool.Get() - lex.sts = -1 - lex.ste = 0 - lblStart := 0 lblEnd := 0 @@ -127,7 +124,7 @@ func (lex *Lexer) Lex() *token.Token { main := |* "#!" any* :>> newline => { - lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te) + lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te) }; any => { fnext html; @@ -143,12 +140,12 @@ func (lex *Lexer) Lex() *token.Token { fbreak; }; ' { - lex.addSkippedToken(tkn, token.T_OPEN_TAG, lex.ts, lex.te) + lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.te) fnext php; }; ' { lex.ungetCnt(lex.te - lex.ts - 5) - lex.addSkippedToken(tkn, token.T_OPEN_TAG, lex.ts, lex.ts+5) + lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.ts+5) fnext php; }; ' { @@ -160,7 +157,7 @@ func (lex *Lexer) Lex() *token.Token { *|; php := |* - whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; + whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; '?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;}; ';' whitespace_line* '?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;}; @@ -320,7 +317,7 @@ func (lex *Lexer) Lex() *token.Token { ('#' | '//') any_line* when is_not_comment_end => { lex.ungetStr("?>") - lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te) + lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te) }; '/*' any_line* :>> '*/' { isDocComment := false; @@ -329,9 +326,9 @@ func (lex *Lexer) Lex() *token.Token { } if isDocComment { - lex.addSkippedToken(tkn, token.T_DOC_COMMENT, lex.ts, lex.te) + lex.addFreeFloatingToken(tkn, token.T_DOC_COMMENT, lex.ts, lex.te) } else { - lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te) + lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te) } }; @@ -378,7 +375,7 @@ func (lex *Lexer) Lex() *token.Token { *|; property := |* - whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; + whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; "->" => {lex.setTokenPosition(tkn); tok = token.T_OBJECT_OPERATOR; fbreak;}; varname => {lex.setTokenPosition(tkn); tok = token.T_STRING; fnext php; fbreak;}; any => {lex.ungetCnt(1); fgoto php;}; @@ -474,38 +471,32 @@ func (lex *Lexer) Lex() *token.Token { *|; halt_compiller_open_parenthesis := |* - whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; + whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; "(" => {lex.setTokenPosition(tkn); tok = token.ID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;}; any => {lex.ungetCnt(1); fnext php;}; *|; halt_compiller_close_parenthesis := |* - whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; + whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; ")" => {lex.setTokenPosition(tkn); tok = token.ID(int(')')); fnext halt_compiller_close_semicolon; fbreak;}; any => {lex.ungetCnt(1); fnext php;}; *|; halt_compiller_close_semicolon := |* - whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; + whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)}; ";" => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext halt_compiller_end; fbreak;}; any => {lex.ungetCnt(1); fnext php;}; *|; halt_compiller_end := |* - any_line* => { lex.addSkippedToken(tkn, token.T_HALT_COMPILER, lex.ts, lex.te); }; + any_line* => { lex.addFreeFloatingToken(tkn, token.T_HALT_COMPILER, lex.ts, lex.te); }; *|; write exec; }%% - if lex.sts == -1 { - lex.sts = 0 - } - tkn.Value = lex.data[lex.ts:lex.te] tkn.ID = token.ID(tok) - tkn.Skipped = lex.data[lex.sts:lex.ste] - lex.addSkippedToken(tkn, tok, lex.ts, lex.te); return tkn } \ No newline at end of file diff --git a/internal/scanner/scanner_test.go b/internal/scanner/scanner_test.go index 465c40a..c49b785 100644 --- a/internal/scanner/scanner_test.go +++ b/internal/scanner/scanner_test.go @@ -386,8 +386,8 @@ func TestShebang(t *testing.T) { tkn := lexer.Lex() assert.Equal(t, tkn.ID, token.T_DNUMBER) - l := len(tkn.SkippedTokens) - for _, tt := range tkn.SkippedTokens[:l-1] { + l := len(tkn.FreeFloating) + for _, tt := range tkn.FreeFloating[:l-1] { actual = append(actual, string(tt.Value)) } @@ -404,7 +404,7 @@ func TestShebangHtml(t *testing.T) { tkn := lexer.Lex() assert.Equal(t, tkn.ID, token.T_INLINE_HTML) - assert.Equal(t, string(tkn.SkippedTokens[0].Value), "#!/usr/bin/env php\n") + assert.Equal(t, string(tkn.FreeFloating[0].Value), "#!/usr/bin/env php\n") tkn = lexer.Lex() assert.Equal(t, tkn.ID, token.T_DNUMBER) @@ -1137,8 +1137,8 @@ func TestCommentEnd(t *testing.T) { tkn := lexer.Lex() - l := len(tkn.SkippedTokens) - actual := tkn.SkippedTokens[:l-1] + l := len(tkn.FreeFloating) + actual := tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1169,8 +1169,8 @@ func TestCommentNewLine(t *testing.T) { tkn := lexer.Lex() - l := len(tkn.SkippedTokens) - actual := tkn.SkippedTokens[:l-1] + l := len(tkn.FreeFloating) + actual := tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1201,8 +1201,8 @@ func TestCommentNewLine1(t *testing.T) { tkn := lexer.Lex() - l := len(tkn.SkippedTokens) - actual := tkn.SkippedTokens[:l-1] + l := len(tkn.FreeFloating) + actual := tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1233,8 +1233,8 @@ func TestCommentNewLine2(t *testing.T) { tkn := lexer.Lex() - l := len(tkn.SkippedTokens) - actual := tkn.SkippedTokens[:l-1] + l := len(tkn.FreeFloating) + actual := tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1266,8 +1266,8 @@ func TestCommentWithPhpEndTag(t *testing.T) { tkn := lexer.Lex() - l := len(tkn.SkippedTokens) - actual := tkn.SkippedTokens[:l-1] + l := len(tkn.FreeFloating) + actual := tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1299,8 +1299,8 @@ func TestInlineComment(t *testing.T) { tkn := lexer.Lex() - l := len(tkn.SkippedTokens) - actual := tkn.SkippedTokens[:l-1] + l := len(tkn.FreeFloating) + actual := tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1332,8 +1332,8 @@ func TestInlineComment2(t *testing.T) { tkn := lexer.Lex() - l := len(tkn.SkippedTokens) - actual := tkn.SkippedTokens[:l-1] + l := len(tkn.FreeFloating) + actual := tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1369,8 +1369,8 @@ func TestEmptyInlineComment(t *testing.T) { tkn := lexer.Lex() - l := len(tkn.SkippedTokens) - actual := tkn.SkippedTokens[:l-1] + l := len(tkn.FreeFloating) + actual := tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1402,8 +1402,8 @@ func TestEmptyInlineComment2(t *testing.T) { tkn := lexer.Lex() - l := len(tkn.SkippedTokens) - actual := tkn.SkippedTokens[:l-1] + l := len(tkn.FreeFloating) + actual := tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1428,8 +1428,8 @@ func TestMethodCallTokens(t *testing.T) { }, } tkn := lexer.Lex() - l := len(tkn.SkippedTokens) - actual := tkn.SkippedTokens[:l-1] + l := len(tkn.FreeFloating) + actual := tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1442,8 +1442,8 @@ func TestMethodCallTokens(t *testing.T) { }, } tkn = lexer.Lex() - l = len(tkn.SkippedTokens) - actual = tkn.SkippedTokens[:l-1] + l = len(tkn.FreeFloating) + actual = tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1456,8 +1456,8 @@ func TestMethodCallTokens(t *testing.T) { }, } tkn = lexer.Lex() - l = len(tkn.SkippedTokens) - actual = tkn.SkippedTokens[:l-1] + l = len(tkn.FreeFloating) + actual = tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1470,8 +1470,8 @@ func TestMethodCallTokens(t *testing.T) { }, } tkn = lexer.Lex() - l = len(tkn.SkippedTokens) - actual = tkn.SkippedTokens[:l-1] + l = len(tkn.FreeFloating) + actual = tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1484,8 +1484,8 @@ func TestMethodCallTokens(t *testing.T) { }, } tkn = lexer.Lex() - l = len(tkn.SkippedTokens) - actual = tkn.SkippedTokens[:l-1] + l = len(tkn.FreeFloating) + actual = tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1498,8 +1498,8 @@ func TestMethodCallTokens(t *testing.T) { }, } tkn = lexer.Lex() - l = len(tkn.SkippedTokens) - actual = tkn.SkippedTokens[:l-1] + l = len(tkn.FreeFloating) + actual = tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1512,8 +1512,8 @@ func TestMethodCallTokens(t *testing.T) { }, } tkn = lexer.Lex() - l = len(tkn.SkippedTokens) - actual = tkn.SkippedTokens[:l-1] + l = len(tkn.FreeFloating) + actual = tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1537,8 +1537,8 @@ func TestYieldFromTokens(t *testing.T) { }, } tkn := lexer.Lex() - l := len(tkn.SkippedTokens) - actual := tkn.SkippedTokens[:l-1] + l := len(tkn.FreeFloating) + actual := tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } @@ -1551,8 +1551,8 @@ func TestYieldFromTokens(t *testing.T) { }, } tkn = lexer.Lex() - l = len(tkn.SkippedTokens) - actual = tkn.SkippedTokens[:l-1] + l = len(tkn.FreeFloating) + actual = tkn.FreeFloating[:l-1] for _, v := range actual { v.Position = nil } diff --git a/pkg/ast/visitor/dump.go b/pkg/ast/visitor/dump.go index 771ac0c..af4a87a 100644 --- a/pkg/ast/visitor/dump.go +++ b/pkg/ast/visitor/dump.go @@ -113,9 +113,6 @@ func (v *Dump) printToken(key string, t *token.Token) { v.printIndent(v.indent + 1) v.print("Value: []byte(" + strconv.Quote(string(t.Value)) + "),\n") - v.printIndent(v.indent + 1) - v.print("Skipped: []byte(" + strconv.Quote(string(t.Skipped)) + "),\n") - v.printIndent(v.indent) v.print("},\n") } diff --git a/pkg/token/token.go b/pkg/token/token.go index b36c85f..406a577 100644 --- a/pkg/token/token.go +++ b/pkg/token/token.go @@ -147,11 +147,10 @@ const ( ) type Token struct { - ID ID - Value []byte - Position *position.Position - SkippedTokens []*Token - Skipped []byte + ID ID + Value []byte + Position *position.Position + FreeFloating []*Token } func (t *Token) GetPosition() *position.Position {