[refactoring] update Token structure
This commit is contained in:
parent
20a42da7c9
commit
8064d940f0
@ -15,7 +15,6 @@ type Lexer struct {
|
||||
phpVersion string
|
||||
errHandlerFunc func(*errors.Error)
|
||||
|
||||
sts, ste int
|
||||
p, pe, cs int
|
||||
ts, te, act int
|
||||
stack []int
|
||||
@ -57,26 +56,18 @@ func (lex *Lexer) setTokenPosition(token *token.Token) {
|
||||
token.Position = pos
|
||||
}
|
||||
|
||||
func (lex *Lexer) addSkippedToken(t *token.Token, id token.ID, ps, pe int) {
|
||||
if lex.sts == -1 {
|
||||
lex.sts = lex.ts
|
||||
}
|
||||
|
||||
lex.ste = lex.te
|
||||
|
||||
// TODO remove after parser refactoring
|
||||
|
||||
func (lex *Lexer) addFreeFloatingToken(t *token.Token, id token.ID, ps, pe int) {
|
||||
skippedTkn := lex.tokenPool.Get()
|
||||
skippedTkn.ID = id
|
||||
skippedTkn.Value = lex.data[ps:pe]
|
||||
|
||||
lex.setTokenPosition(skippedTkn)
|
||||
|
||||
if t.SkippedTokens == nil {
|
||||
t.SkippedTokens = make([]*token.Token, 0, 2)
|
||||
if t.FreeFloating == nil {
|
||||
t.FreeFloating = make([]*token.Token, 0, 2)
|
||||
}
|
||||
|
||||
t.SkippedTokens = append(t.SkippedTokens, skippedTkn)
|
||||
t.FreeFloating = append(t.FreeFloating, skippedTkn)
|
||||
}
|
||||
|
||||
func (lex *Lexer) isNotStringVar() bool {
|
||||
|
BIN
internal/scanner/scanner.go
generated
BIN
internal/scanner/scanner.go
generated
Binary file not shown.
@ -26,9 +26,6 @@ func (lex *Lexer) Lex() *token.Token {
|
||||
|
||||
tkn := lex.tokenPool.Get()
|
||||
|
||||
lex.sts = -1
|
||||
lex.ste = 0
|
||||
|
||||
lblStart := 0
|
||||
lblEnd := 0
|
||||
|
||||
@ -127,7 +124,7 @@ func (lex *Lexer) Lex() *token.Token {
|
||||
|
||||
main := |*
|
||||
"#!" any* :>> newline => {
|
||||
lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te)
|
||||
lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
|
||||
};
|
||||
any => {
|
||||
fnext html;
|
||||
@ -143,12 +140,12 @@ func (lex *Lexer) Lex() *token.Token {
|
||||
fbreak;
|
||||
};
|
||||
'<?' => {
|
||||
lex.addSkippedToken(tkn, token.T_OPEN_TAG, lex.ts, lex.te)
|
||||
lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.te)
|
||||
fnext php;
|
||||
};
|
||||
'<?php'i ( [ \t] | newline ) => {
|
||||
lex.ungetCnt(lex.te - lex.ts - 5)
|
||||
lex.addSkippedToken(tkn, token.T_OPEN_TAG, lex.ts, lex.ts+5)
|
||||
lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.ts+5)
|
||||
fnext php;
|
||||
};
|
||||
'<?='i => {
|
||||
@ -160,7 +157,7 @@ func (lex *Lexer) Lex() *token.Token {
|
||||
*|;
|
||||
|
||||
php := |*
|
||||
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
|
||||
whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
|
||||
'?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;};
|
||||
';' whitespace_line* '?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;};
|
||||
|
||||
@ -320,7 +317,7 @@ func (lex *Lexer) Lex() *token.Token {
|
||||
|
||||
('#' | '//') any_line* when is_not_comment_end => {
|
||||
lex.ungetStr("?>")
|
||||
lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te)
|
||||
lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
|
||||
};
|
||||
'/*' any_line* :>> '*/' {
|
||||
isDocComment := false;
|
||||
@ -329,9 +326,9 @@ func (lex *Lexer) Lex() *token.Token {
|
||||
}
|
||||
|
||||
if isDocComment {
|
||||
lex.addSkippedToken(tkn, token.T_DOC_COMMENT, lex.ts, lex.te)
|
||||
lex.addFreeFloatingToken(tkn, token.T_DOC_COMMENT, lex.ts, lex.te)
|
||||
} else {
|
||||
lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te)
|
||||
lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
|
||||
}
|
||||
};
|
||||
|
||||
@ -378,7 +375,7 @@ func (lex *Lexer) Lex() *token.Token {
|
||||
*|;
|
||||
|
||||
property := |*
|
||||
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
|
||||
whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
|
||||
"->" => {lex.setTokenPosition(tkn); tok = token.T_OBJECT_OPERATOR; fbreak;};
|
||||
varname => {lex.setTokenPosition(tkn); tok = token.T_STRING; fnext php; fbreak;};
|
||||
any => {lex.ungetCnt(1); fgoto php;};
|
||||
@ -474,38 +471,32 @@ func (lex *Lexer) Lex() *token.Token {
|
||||
*|;
|
||||
|
||||
halt_compiller_open_parenthesis := |*
|
||||
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
|
||||
whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
|
||||
"(" => {lex.setTokenPosition(tkn); tok = token.ID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;};
|
||||
any => {lex.ungetCnt(1); fnext php;};
|
||||
*|;
|
||||
|
||||
halt_compiller_close_parenthesis := |*
|
||||
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
|
||||
whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
|
||||
")" => {lex.setTokenPosition(tkn); tok = token.ID(int(')')); fnext halt_compiller_close_semicolon; fbreak;};
|
||||
any => {lex.ungetCnt(1); fnext php;};
|
||||
*|;
|
||||
|
||||
halt_compiller_close_semicolon := |*
|
||||
whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
|
||||
whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
|
||||
";" => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext halt_compiller_end; fbreak;};
|
||||
any => {lex.ungetCnt(1); fnext php;};
|
||||
*|;
|
||||
|
||||
halt_compiller_end := |*
|
||||
any_line* => { lex.addSkippedToken(tkn, token.T_HALT_COMPILER, lex.ts, lex.te); };
|
||||
any_line* => { lex.addFreeFloatingToken(tkn, token.T_HALT_COMPILER, lex.ts, lex.te); };
|
||||
*|;
|
||||
|
||||
write exec;
|
||||
}%%
|
||||
|
||||
if lex.sts == -1 {
|
||||
lex.sts = 0
|
||||
}
|
||||
|
||||
tkn.Value = lex.data[lex.ts:lex.te]
|
||||
tkn.ID = token.ID(tok)
|
||||
tkn.Skipped = lex.data[lex.sts:lex.ste]
|
||||
lex.addSkippedToken(tkn, tok, lex.ts, lex.te);
|
||||
|
||||
return tkn
|
||||
}
|
@ -386,8 +386,8 @@ func TestShebang(t *testing.T) {
|
||||
tkn := lexer.Lex()
|
||||
assert.Equal(t, tkn.ID, token.T_DNUMBER)
|
||||
|
||||
l := len(tkn.SkippedTokens)
|
||||
for _, tt := range tkn.SkippedTokens[:l-1] {
|
||||
l := len(tkn.FreeFloating)
|
||||
for _, tt := range tkn.FreeFloating[:l-1] {
|
||||
actual = append(actual, string(tt.Value))
|
||||
}
|
||||
|
||||
@ -404,7 +404,7 @@ func TestShebangHtml(t *testing.T) {
|
||||
|
||||
tkn := lexer.Lex()
|
||||
assert.Equal(t, tkn.ID, token.T_INLINE_HTML)
|
||||
assert.Equal(t, string(tkn.SkippedTokens[0].Value), "#!/usr/bin/env php\n")
|
||||
assert.Equal(t, string(tkn.FreeFloating[0].Value), "#!/usr/bin/env php\n")
|
||||
|
||||
tkn = lexer.Lex()
|
||||
assert.Equal(t, tkn.ID, token.T_DNUMBER)
|
||||
@ -1137,8 +1137,8 @@ func TestCommentEnd(t *testing.T) {
|
||||
|
||||
tkn := lexer.Lex()
|
||||
|
||||
l := len(tkn.SkippedTokens)
|
||||
actual := tkn.SkippedTokens[:l-1]
|
||||
l := len(tkn.FreeFloating)
|
||||
actual := tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1169,8 +1169,8 @@ func TestCommentNewLine(t *testing.T) {
|
||||
|
||||
tkn := lexer.Lex()
|
||||
|
||||
l := len(tkn.SkippedTokens)
|
||||
actual := tkn.SkippedTokens[:l-1]
|
||||
l := len(tkn.FreeFloating)
|
||||
actual := tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1201,8 +1201,8 @@ func TestCommentNewLine1(t *testing.T) {
|
||||
|
||||
tkn := lexer.Lex()
|
||||
|
||||
l := len(tkn.SkippedTokens)
|
||||
actual := tkn.SkippedTokens[:l-1]
|
||||
l := len(tkn.FreeFloating)
|
||||
actual := tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1233,8 +1233,8 @@ func TestCommentNewLine2(t *testing.T) {
|
||||
|
||||
tkn := lexer.Lex()
|
||||
|
||||
l := len(tkn.SkippedTokens)
|
||||
actual := tkn.SkippedTokens[:l-1]
|
||||
l := len(tkn.FreeFloating)
|
||||
actual := tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1266,8 +1266,8 @@ func TestCommentWithPhpEndTag(t *testing.T) {
|
||||
|
||||
tkn := lexer.Lex()
|
||||
|
||||
l := len(tkn.SkippedTokens)
|
||||
actual := tkn.SkippedTokens[:l-1]
|
||||
l := len(tkn.FreeFloating)
|
||||
actual := tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1299,8 +1299,8 @@ func TestInlineComment(t *testing.T) {
|
||||
|
||||
tkn := lexer.Lex()
|
||||
|
||||
l := len(tkn.SkippedTokens)
|
||||
actual := tkn.SkippedTokens[:l-1]
|
||||
l := len(tkn.FreeFloating)
|
||||
actual := tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1332,8 +1332,8 @@ func TestInlineComment2(t *testing.T) {
|
||||
|
||||
tkn := lexer.Lex()
|
||||
|
||||
l := len(tkn.SkippedTokens)
|
||||
actual := tkn.SkippedTokens[:l-1]
|
||||
l := len(tkn.FreeFloating)
|
||||
actual := tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1369,8 +1369,8 @@ func TestEmptyInlineComment(t *testing.T) {
|
||||
|
||||
tkn := lexer.Lex()
|
||||
|
||||
l := len(tkn.SkippedTokens)
|
||||
actual := tkn.SkippedTokens[:l-1]
|
||||
l := len(tkn.FreeFloating)
|
||||
actual := tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1402,8 +1402,8 @@ func TestEmptyInlineComment2(t *testing.T) {
|
||||
|
||||
tkn := lexer.Lex()
|
||||
|
||||
l := len(tkn.SkippedTokens)
|
||||
actual := tkn.SkippedTokens[:l-1]
|
||||
l := len(tkn.FreeFloating)
|
||||
actual := tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1428,8 +1428,8 @@ func TestMethodCallTokens(t *testing.T) {
|
||||
},
|
||||
}
|
||||
tkn := lexer.Lex()
|
||||
l := len(tkn.SkippedTokens)
|
||||
actual := tkn.SkippedTokens[:l-1]
|
||||
l := len(tkn.FreeFloating)
|
||||
actual := tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1442,8 +1442,8 @@ func TestMethodCallTokens(t *testing.T) {
|
||||
},
|
||||
}
|
||||
tkn = lexer.Lex()
|
||||
l = len(tkn.SkippedTokens)
|
||||
actual = tkn.SkippedTokens[:l-1]
|
||||
l = len(tkn.FreeFloating)
|
||||
actual = tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1456,8 +1456,8 @@ func TestMethodCallTokens(t *testing.T) {
|
||||
},
|
||||
}
|
||||
tkn = lexer.Lex()
|
||||
l = len(tkn.SkippedTokens)
|
||||
actual = tkn.SkippedTokens[:l-1]
|
||||
l = len(tkn.FreeFloating)
|
||||
actual = tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1470,8 +1470,8 @@ func TestMethodCallTokens(t *testing.T) {
|
||||
},
|
||||
}
|
||||
tkn = lexer.Lex()
|
||||
l = len(tkn.SkippedTokens)
|
||||
actual = tkn.SkippedTokens[:l-1]
|
||||
l = len(tkn.FreeFloating)
|
||||
actual = tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1484,8 +1484,8 @@ func TestMethodCallTokens(t *testing.T) {
|
||||
},
|
||||
}
|
||||
tkn = lexer.Lex()
|
||||
l = len(tkn.SkippedTokens)
|
||||
actual = tkn.SkippedTokens[:l-1]
|
||||
l = len(tkn.FreeFloating)
|
||||
actual = tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1498,8 +1498,8 @@ func TestMethodCallTokens(t *testing.T) {
|
||||
},
|
||||
}
|
||||
tkn = lexer.Lex()
|
||||
l = len(tkn.SkippedTokens)
|
||||
actual = tkn.SkippedTokens[:l-1]
|
||||
l = len(tkn.FreeFloating)
|
||||
actual = tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1512,8 +1512,8 @@ func TestMethodCallTokens(t *testing.T) {
|
||||
},
|
||||
}
|
||||
tkn = lexer.Lex()
|
||||
l = len(tkn.SkippedTokens)
|
||||
actual = tkn.SkippedTokens[:l-1]
|
||||
l = len(tkn.FreeFloating)
|
||||
actual = tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1537,8 +1537,8 @@ func TestYieldFromTokens(t *testing.T) {
|
||||
},
|
||||
}
|
||||
tkn := lexer.Lex()
|
||||
l := len(tkn.SkippedTokens)
|
||||
actual := tkn.SkippedTokens[:l-1]
|
||||
l := len(tkn.FreeFloating)
|
||||
actual := tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
@ -1551,8 +1551,8 @@ func TestYieldFromTokens(t *testing.T) {
|
||||
},
|
||||
}
|
||||
tkn = lexer.Lex()
|
||||
l = len(tkn.SkippedTokens)
|
||||
actual = tkn.SkippedTokens[:l-1]
|
||||
l = len(tkn.FreeFloating)
|
||||
actual = tkn.FreeFloating[:l-1]
|
||||
for _, v := range actual {
|
||||
v.Position = nil
|
||||
}
|
||||
|
@ -113,9 +113,6 @@ func (v *Dump) printToken(key string, t *token.Token) {
|
||||
v.printIndent(v.indent + 1)
|
||||
v.print("Value: []byte(" + strconv.Quote(string(t.Value)) + "),\n")
|
||||
|
||||
v.printIndent(v.indent + 1)
|
||||
v.print("Skipped: []byte(" + strconv.Quote(string(t.Skipped)) + "),\n")
|
||||
|
||||
v.printIndent(v.indent)
|
||||
v.print("},\n")
|
||||
}
|
||||
|
@ -147,11 +147,10 @@ const (
|
||||
)
|
||||
|
||||
type Token struct {
|
||||
ID ID
|
||||
Value []byte
|
||||
Position *position.Position
|
||||
SkippedTokens []*Token
|
||||
Skipped []byte
|
||||
ID ID
|
||||
Value []byte
|
||||
Position *position.Position
|
||||
FreeFloating []*Token
|
||||
}
|
||||
|
||||
func (t *Token) GetPosition() *position.Position {
|
||||
|
Loading…
Reference in New Issue
Block a user