[refactoring] rename struct field scanner.Token.Hidden

This commit is contained in:
Vadym Slizov 2020-05-18 00:04:04 +03:00
parent e98607f5d0
commit 5620697cab
11 changed files with 1179 additions and 1179 deletions

View File

@ -50,7 +50,7 @@ func (l *Parser) Error(msg string) {
}
func (l *Parser) WithTokens() {
l.Lexer.SetWithTokens(true)
l.Lexer.SetWithHiddenTokens(true)
}
// Parse the php7 Parser entrypoint
@ -93,7 +93,7 @@ func isDollar(r rune) bool {
}
func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
if l.Lexer.GetWithFreeFloating() == false {
if l.Lexer.GetWithHiddenTokens() == false {
return
}
@ -106,7 +106,7 @@ func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
}
func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) {
if l.Lexer.GetWithFreeFloating() == false {
if l.Lexer.GetWithHiddenTokens() == false {
return
}
@ -123,7 +123,7 @@ func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []tok
}
func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
if l.Lexer.GetWithFreeFloating() == false {
if l.Lexer.GetWithHiddenTokens() == false {
return []token.Token{}
}
@ -136,7 +136,7 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
}
func (l *Parser) addDollarToken(v ast.Vertex) {
if l.Lexer.GetWithFreeFloating() == false {
if l.Lexer.GetWithHiddenTokens() == false {
return
}
@ -149,7 +149,7 @@ func (l *Parser) addDollarToken(v ast.Vertex) {
}
func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
if l.Lexer.GetWithFreeFloating() == false {
if l.Lexer.GetWithHiddenTokens() == false {
return
}

Binary file not shown.

File diff suppressed because it is too large Load Diff

View File

@ -48,7 +48,7 @@ func (l *Parser) Error(msg string) {
}
func (l *Parser) WithTokens() {
l.Lexer.SetWithTokens(true)
l.Lexer.SetWithHiddenTokens(true)
}
// Parse the php7 Parser entrypoint
@ -91,7 +91,7 @@ func isDollar(r rune) bool {
}
func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
if l.Lexer.GetWithFreeFloating() == false {
if l.Lexer.GetWithHiddenTokens() == false {
return
}
@ -104,7 +104,7 @@ func (l *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
}
func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []token.Token) {
if l.Lexer.GetWithFreeFloating() == false {
if l.Lexer.GetWithHiddenTokens() == false {
return
}
@ -121,7 +121,7 @@ func (l *Parser) setFreeFloating(dst ast.Vertex, p token.Position, strings []tok
}
func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
if l.Lexer.GetWithFreeFloating() == false {
if l.Lexer.GetWithHiddenTokens() == false {
return []token.Token{}
}
@ -134,7 +134,7 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
}
func (l *Parser) addDollarToken(v ast.Vertex) {
if l.Lexer.GetWithFreeFloating() == false {
if l.Lexer.GetWithHiddenTokens() == false {
return
}
@ -147,7 +147,7 @@ func (l *Parser) addDollarToken(v ast.Vertex) {
}
func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
if l.Lexer.GetWithFreeFloating() == false {
if l.Lexer.GetWithHiddenTokens() == false {
return
}

Binary file not shown.

File diff suppressed because it is too large Load Diff

View File

@ -16,8 +16,8 @@ type Scanner interface {
GetPhpDocComment() string
SetPhpDocComment(string)
GetErrors() []*errors.Error
GetWithFreeFloating() bool
SetWithTokens(bool)
GetWithHiddenTokens() bool
SetWithHiddenTokens(bool)
AddError(e *errors.Error)
SetErrors(e []*errors.Error)
}
@ -36,8 +36,8 @@ type Lexer struct {
heredocLabel []byte
TokenPool *TokenPool
Tokens []token.Token
WithTokens bool
HiddenTokens []token.Token
WithHiddenTokens bool
PhpDocComment string
lastToken *Token
Errors []*errors.Error
@ -61,12 +61,12 @@ func (l *Lexer) GetErrors() []*errors.Error {
return l.Errors
}
func (l *Lexer) GetWithFreeFloating() bool {
return l.WithTokens
func (l *Lexer) GetWithHiddenTokens() bool {
return l.WithHiddenTokens
}
func (l *Lexer) SetWithTokens(b bool) {
l.WithTokens = b
func (l *Lexer) SetWithHiddenTokens(b bool) {
l.WithHiddenTokens = b
}
func (l *Lexer) AddError(e *errors.Error) {
@ -85,11 +85,11 @@ func (lex *Lexer) setTokenPosition(token *Token) {
}
func (lex *Lexer) addToken(id TokenID, ps, pe int) {
if !lex.WithTokens {
if !lex.WithHiddenTokens {
return
}
lex.Tokens = append(lex.Tokens, token.Token{
lex.HiddenTokens = append(lex.HiddenTokens, token.Token{
ID: token.ID(id),
Value: lex.data[ps:pe],
})

Binary file not shown.

View File

@ -28,12 +28,12 @@ func NewLexer(data []byte) *Lexer {
}
func (lex *Lexer) Lex(lval Lval) int {
lex.Tokens = nil
lex.HiddenTokens = nil
eof := lex.pe
var tok TokenID
token := lex.TokenPool.Get()
token.Tokens = lex.Tokens
token.Hidden = nil
token.Value = lex.data[0:0]
lblStart := 0
@ -508,7 +508,7 @@ func (lex *Lexer) Lex(lval Lval) int {
write exec;
}%%
token.Tokens = lex.Tokens
token.Hidden = lex.HiddenTokens
token.Value = lex.data[lex.ts:lex.te]
token.ID = tok

View File

@ -360,7 +360,7 @@ func TestTokens(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
@ -389,14 +389,14 @@ func TestShebang(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
token := lexer.Lex(lv)
assert.Equal(t, token, int(T_DNUMBER))
for _, tt := range lv.Tkn.Tokens {
for _, tt := range lv.Tkn.Hidden {
actual = append(actual, string(tt.Value))
}
@ -410,12 +410,12 @@ func TestShebangHtml(t *testing.T) {
`
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
token := lexer.Lex(lv)
assert.Equal(t, token, int(T_INLINE_HTML))
assert.Equal(t, string(lv.Tkn.Tokens[0].Value), "#!/usr/bin/env php\n")
assert.Equal(t, string(lv.Tkn.Hidden[0].Value), "#!/usr/bin/env php\n")
token = lexer.Lex(lv)
assert.Equal(t, token, int(T_DNUMBER))
@ -461,7 +461,7 @@ func TestNumberTokens(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
@ -519,7 +519,7 @@ func TestConstantStrings(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
@ -655,7 +655,7 @@ func TestTeplateStringTokens(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
@ -741,7 +741,7 @@ func TestBackquoteStringTokens(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
@ -836,7 +836,7 @@ CAT;
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
@ -910,7 +910,7 @@ CAT
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
@ -950,7 +950,7 @@ CAT;
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
@ -982,7 +982,7 @@ func TestHereDocTokens73(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
@ -1014,7 +1014,7 @@ CAT;`
lexer := NewLexer([]byte(src))
lexer.PHPVersion = "7.2"
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
@ -1047,7 +1047,7 @@ func TestInlineHtmlNopTokens(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
actual := []string{}
@ -1155,12 +1155,12 @@ func TestCommentEnd(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Tokens
actual := lexer.HiddenTokens
assert.DeepEqual(t, expected, actual)
}
@ -1185,12 +1185,12 @@ func TestCommentNewLine(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
lexer.Lex(lv)
actual := lv.Tkn.Tokens
actual := lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
}
@ -1215,12 +1215,12 @@ func TestCommentNewLine1(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
lexer.Lex(lv)
actual := lv.Tkn.Tokens
actual := lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
}
@ -1245,12 +1245,12 @@ func TestCommentNewLine2(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
lexer.Lex(lv)
actual := lv.Tkn.Tokens
actual := lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
}
@ -1276,12 +1276,12 @@ func TestCommentWithPhpEndTag(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
lexer.Lex(lv)
actual := lv.Tkn.Tokens
actual := lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
}
@ -1307,12 +1307,12 @@ func TestInlineComment(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
lexer.Lex(lv)
actual := lv.Tkn.Tokens
actual := lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
}
@ -1338,12 +1338,12 @@ func TestInlineComment2(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Tokens
actual := lexer.HiddenTokens
assert.DeepEqual(t, expected, actual)
}
@ -1373,12 +1373,12 @@ func TestEmptyInlineComment(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
lexer.Lex(lv)
actual := lexer.Tokens
actual := lexer.HiddenTokens
assert.DeepEqual(t, expected, actual)
}
@ -1404,12 +1404,12 @@ func TestEmptyInlineComment2(t *testing.T) {
}
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
lexer.Lex(lv)
actual := lv.Tkn.Tokens
actual := lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
}
@ -1419,7 +1419,7 @@ func TestMethodCallTokens(t *testing.T) {
$a -> bar ( '' ) ;`
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
expected := []token.Token{
@ -1433,7 +1433,7 @@ func TestMethodCallTokens(t *testing.T) {
},
}
lexer.Lex(lv)
actual := lv.Tkn.Tokens
actual := lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
expected = []token.Token{
@ -1443,7 +1443,7 @@ func TestMethodCallTokens(t *testing.T) {
},
}
lexer.Lex(lv)
actual = lv.Tkn.Tokens
actual = lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
expected = []token.Token{
@ -1453,7 +1453,7 @@ func TestMethodCallTokens(t *testing.T) {
},
}
lexer.Lex(lv)
actual = lv.Tkn.Tokens
actual = lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
expected = []token.Token{
@ -1463,7 +1463,7 @@ func TestMethodCallTokens(t *testing.T) {
},
}
lexer.Lex(lv)
actual = lv.Tkn.Tokens
actual = lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
expected = []token.Token{
@ -1473,7 +1473,7 @@ func TestMethodCallTokens(t *testing.T) {
},
}
lexer.Lex(lv)
actual = lv.Tkn.Tokens
actual = lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
expected = []token.Token{
@ -1483,7 +1483,7 @@ func TestMethodCallTokens(t *testing.T) {
},
}
lexer.Lex(lv)
actual = lv.Tkn.Tokens
actual = lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
expected = []token.Token{
@ -1493,7 +1493,7 @@ func TestMethodCallTokens(t *testing.T) {
},
}
lexer.Lex(lv)
actual = lv.Tkn.Tokens
actual = lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
}
@ -1502,7 +1502,7 @@ func TestYieldFromTokens(t *testing.T) {
yield from $a`
lexer := NewLexer([]byte(src))
lexer.WithTokens = true
lexer.WithHiddenTokens = true
lv := &lval{}
expected := []token.Token{
@ -1516,7 +1516,7 @@ func TestYieldFromTokens(t *testing.T) {
},
}
lexer.Lex(lv)
actual := lv.Tkn.Tokens
actual := lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
expected = []token.Token{
@ -1526,7 +1526,7 @@ func TestYieldFromTokens(t *testing.T) {
},
}
lexer.Lex(lv)
actual = lv.Tkn.Tokens
actual = lv.Tkn.Hidden
assert.DeepEqual(t, expected, actual)
}

View File

@ -9,6 +9,6 @@ import (
type Token struct {
ID TokenID
Value []byte
Tokens []token.Token
Hidden []token.Token
Position position.Position
}