[refactoring] remove param withTokens from parser

internal/php5/parser.go

@@ -14,14 +14,12 @@ type Parser struct {
 	Lexer          *scanner.Lexer
 	currentToken   *scanner.Token
 	rootNode       ast.Vertex
-	withTokens     bool
 	errHandlerFunc func(*errors.Error)
 }
 
 // NewParser creates and returns new Parser
-func NewParser(lexer *scanner.Lexer, withTokens bool, errHandlerFunc func(*errors.Error)) *Parser {
+func NewParser(lexer *scanner.Lexer, errHandlerFunc func(*errors.Error)) *Parser {
 	return &Parser{
-		withTokens:     withTokens,
 		Lexer:          lexer,
 		errHandlerFunc: errHandlerFunc,
 	}
@@ -63,7 +61,7 @@ func lastNode(nn []ast.Vertex) ast.Vertex {
 }
 
 func (p *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
-	if p.withTokens == false {
+	if _, ok := src.GetNode().Tokens[token.Start]; !ok {
 		return
 	}
 
@@ -71,42 +69,64 @@ func (p *Parser) MoveFreeFloating(src ast.Vertex, dst ast.Vertex) {
 		return
 	}
 
-	p.setFreeFloating(dst, token.Start, src.GetNode().Tokens[token.Start])
-	delete(src.GetNode().Tokens, token.Start)
-}
-
-func (p *Parser) setFreeFloating(dst ast.Vertex, pos token.Position, strings []token.Token) {
-	if p.withTokens == false {
-		return
-	}
-
-	if len(strings) == 0 {
-		return
-	}
-
 	dstCollection := &dst.GetNode().Tokens
 	if *dstCollection == nil {
 		*dstCollection = make(token.Collection)
 	}
 
-	(*dstCollection)[pos] = strings
+	(*dstCollection)[token.Start] = src.GetNode().Tokens[token.Start]
+	delete(src.GetNode().Tokens, token.Start)
 }
 
-func (p *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
-	if p.withTokens == false {
-		return []token.Token{}
+func (p *Parser) setFreeFloating(dst ast.Vertex, pos token.Position, tokens []token.Token) {
+	if len(tokens) == 0 {
+		return
 	}
 
-	return []token.Token{
-		{
-			ID:    token.ID(t.ID),
-			Value: t.Value,
-		},
+	dstCollection := &dst.GetNode().Tokens
+	if *dstCollection == nil {
+		*dstCollection = make(token.Collection)
 	}
+
+	l := len(tokens)
+	for _, v := range tokens[0 : l-1] {
+		(*dstCollection)[pos] = append((*dstCollection)[pos], v)
+	}
+}
+
+func (p *Parser) setFreeFloatingTokens(dst ast.Vertex, pos token.Position, tokens []token.Token) {
+	if len(tokens) == 0 {
+		return
+	}
+
+	dstCollection := &dst.GetNode().Tokens
+	if *dstCollection == nil {
+		*dstCollection = make(token.Collection)
+	}
+
+	(*dstCollection)[pos] = make([]token.Token, 0)
+
+	for _, v := range tokens {
+		(*dstCollection)[pos] = append((*dstCollection)[pos], v)
+	}
+}
+
+func (p *Parser) setToken(dst ast.Vertex, pos token.Position, tokens []token.Token) {
+	if len(tokens) == 0 {
+		return
+	}
+
+	dstCollection := &dst.GetNode().Tokens
+	if *dstCollection == nil {
+		*dstCollection = make(token.Collection)
+	}
+
+	l := len(tokens)
+	(*dstCollection)[pos] = append((*dstCollection)[pos], tokens[l-1])
 }
 
 func (p *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
-	if p.withTokens == false {
+	if _, ok := prevNode.GetNode().Tokens[token.SemiColon]; !ok {
 		return
 	}
 
@@ -117,7 +137,7 @@ func (p *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.
 	}
 
 	if semiColon[0].Value[0] == ';' {
-		p.setFreeFloating(prevNode, token.SemiColon, []token.Token{
+		p.setFreeFloatingTokens(prevNode, token.SemiColon, []token.Token{
 			{
 				ID:    token.ID(';'),
 				Value: semiColon[0].Value[0:1],
@@ -126,7 +146,6 @@ func (p *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.
 	}
 
 	vlen := len(semiColon[0].Value)
-
 	tlen := 2
 	if bytes.HasSuffix(semiColon[0].Value, []byte("?>\n")) {
 		tlen = 3
@@ -145,7 +164,7 @@ func (p *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.
 		Value: semiColon[0].Value[vlen-tlen:],
 	})
 
-	p.setFreeFloating(htmlNode, token.Start, append(phpCloseTag, htmlNode.GetNode().Tokens[token.Start]...))
+	p.setFreeFloatingTokens(htmlNode, token.Start, append(phpCloseTag, htmlNode.GetNode().Tokens[token.Start]...))
 }
 
 func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {

internal/php5/php5_bench_test.go

@@ -415,7 +415,7 @@ CAD;
 
 	for n := 0; n < b.N; n++ {
 		lexer := scanner.NewLexer([]byte(src), "5.6", false, nil)
-		php5parser := php5.NewParser(lexer, false, nil)
+		php5parser := php5.NewParser(lexer, nil)
 		php5parser.Parse()
 	}
 }

internal/php5/php5_test.go

@@ -22458,7 +22458,7 @@ func TestPhp5(t *testing.T) {
 	}
 
 	lexer := scanner.NewLexer([]byte(src), "5.6", false, nil)
-	php5parser := php5.NewParser(lexer, false, nil)
+	php5parser := php5.NewParser(lexer, nil)
 	php5parser.Parse()
 	actual := php5parser.GetRootNode()
 	assert.DeepEqual(t, expected, actual)
@@ -22596,7 +22596,7 @@ func TestPhp5Strings(t *testing.T) {
 	}
 
 	lexer := scanner.NewLexer([]byte(src), "5.6", false, nil)
-	php5parser := php5.NewParser(lexer, false, nil)
+	php5parser := php5.NewParser(lexer, nil)
 	php5parser.Parse()
 	actual := php5parser.GetRootNode()
 	assert.DeepEqual(t, expected, actual)
@@ -22823,7 +22823,7 @@ CAD;
 	}
 
 	lexer := scanner.NewLexer([]byte(src), "5.6", false, nil)
-	php5parser := php5.NewParser(lexer, false, nil)
+	php5parser := php5.NewParser(lexer, nil)
 	php5parser.Parse()
 	actual := php5parser.GetRootNode()
 	assert.DeepEqual(t, expected, actual)
@@ -22849,7 +22849,7 @@ func TestPhp5ControlCharsErrors(t *testing.T) {
 	}
 
 	lexer := scanner.NewLexer([]byte(src), "5.6", false, errorHandlerFunc)
-	php5parser := php5.NewParser(lexer, false, errorHandlerFunc)
+	php5parser := php5.NewParser(lexer, errorHandlerFunc)
 	php5parser.Parse()
 	assert.DeepEqual(t, expected, parserErrors)
 }