refactor printer

cmd/php-parser/main.go

@@ -13,8 +13,8 @@ import (
 
 	"github.com/pkg/profile"
 	"github.com/yookoala/realpath"
-	"github.com/z7zmey/php-parser/pkg/parser"
 	"github.com/z7zmey/php-parser/pkg/ast/visitor"
+	"github.com/z7zmey/php-parser/pkg/parser"
 )
 
 var wg sync.WaitGroup

internal/php5/parser.go

@@ -2,6 +2,7 @@ package php5
 
 import (
 	"bytes"
+	"fmt"
 
 	"github.com/z7zmey/php-parser/internal/positionbuilder"
 	"github.com/z7zmey/php-parser/internal/scanner"
@@ -132,10 +133,25 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
 		return []token.Token{}
 	}
 
-	tokens := make([]token.Token, len(t.Tokens))
+	return []token.Token{
-	copy(tokens, t.Tokens)
+		{
+			ID:    token.ID(t.ID),
+			Value: t.Value,
+		},
+	}
+}
 
-	return tokens
+func (l *Parser) addDollarToken(v ast.Vertex) {
+	if l.Lexer.GetWithFreeFloating() == false {
+		return
+	}
+
+	l.setFreeFloating(v, token.Dollar, []token.Token{
+		{
+			ID:    token.ID('$'),
+			Value: []byte("$"),
+		},
+	})
 }
 
 func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
@@ -159,6 +175,8 @@ func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.
 	}
 
 	vlen := len(semiColon[0].Value)
+	fmt.Printf("vlen: %q\n", string(semiColon[0].Value))
+
 	tlen := 2
 	if bytes.HasSuffix(semiColon[0].Value, []byte("?>\n")) {
 		tlen = 3

internal/php5/php5.y

@@ -1431,6 +1431,7 @@ catch_statement:
                 yylex.(*Parser).setFreeFloating(catchNode, token.Start, $1.Tokens)
                 yylex.(*Parser).setFreeFloating(catchNode, token.Catch, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $4.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(catchNode, token.Var, $5.Tokens)
                 yylex.(*Parser).setFreeFloating(catchNode, token.Cond, $6.Tokens)
                 yylex.(*Parser).setFreeFloating(catchNode, token.Stmts, $8.Tokens)
@@ -1508,6 +1509,7 @@ additional_catch:
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
                 yylex.(*Parser).setFreeFloating($$, token.Catch, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $4.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $5.Tokens)
                 yylex.(*Parser).setFreeFloating($$, token.Cond, $6.Tokens)
                 yylex.(*Parser).setFreeFloating($$, token.Stmts, $8.Tokens)
@@ -2309,6 +2311,7 @@ parameter:
                     yylex.(*Parser).setFreeFloating($$, token.Ampersand, $3.Tokens)
                 }
                 yylex.(*Parser).setFreeFloating($$, token.Variadic, $4.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 // normalize
                 if $3 == nil {
@@ -2354,6 +2357,7 @@ parameter:
                 }
                 yylex.(*Parser).setFreeFloating($$, token.Variadic, $4.Tokens)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $5.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 // normalize
                 if $3 == nil {
@@ -2556,6 +2560,7 @@ global_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -2605,6 +2610,7 @@ static_var_list:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Start, $3.Tokens)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
@@ -2623,6 +2629,7 @@ static_var_list:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Start, $3.Tokens)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Var, $4.Tokens)
 
@@ -2641,6 +2648,7 @@ static_var_list:
                 staticVar.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokenPosition($1)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Start, $1.Tokens)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
@@ -2658,6 +2666,7 @@ static_var_list:
                 staticVar.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokenNodePosition($1, $3)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Start, $1.Tokens)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Var, $2.Tokens)
 
@@ -3155,6 +3164,7 @@ class_variable_declaration:
                 property.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokenPosition($3)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(property, token.Start, $3.Tokens)
 
@@ -3174,6 +3184,7 @@ class_variable_declaration:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(property, token.Start, $3.Tokens)
                 yylex.(*Parser).setFreeFloating(property, token.Var, $4.Tokens)
 
@@ -3192,6 +3203,7 @@ class_variable_declaration:
                 property.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokenPosition($1)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(property, token.Start, $1.Tokens)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
@@ -3209,6 +3221,7 @@ class_variable_declaration:
                 property.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokenNodePosition($1, $3)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(property, token.Start, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(property, token.Var, $2.Tokens)
 
@@ -4589,6 +4602,7 @@ lexical_var_list:
                 // save comments
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $3.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -4608,6 +4622,7 @@ lexical_var_list:
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(reference, token.Start, $3.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $4.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -4623,6 +4638,7 @@ lexical_var_list:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -4641,6 +4657,7 @@ lexical_var_list:
                 // save comments
                 yylex.(*Parser).setFreeFloating(reference, token.Start, $1.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $2.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -6381,6 +6398,7 @@ compound_variable:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -6807,6 +6825,7 @@ encaps_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -6822,6 +6841,7 @@ encaps_var:
                 $$.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokensPosition($1, $4)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, append($2.Tokens, yylex.(*Parser).GetFreeFloatingToken($2)...))
                 yylex.(*Parser).setFreeFloating($$, token.Expr, append($4.Tokens, yylex.(*Parser).GetFreeFloatingToken($4)...))
 
@@ -6841,6 +6861,7 @@ encaps_var:
                 $$.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokensPosition($1, $3)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(fetch, token.Start, $3.Tokens)
 
@@ -6950,6 +6971,7 @@ encaps_var_offset:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }

internal/php7/parser.go

@@ -131,10 +131,25 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
 		return []token.Token{}
 	}
 
-	tokens := make([]token.Token, len(t.Tokens))
+	return []token.Token{
-	copy(tokens, t.Tokens)
+		{
+			ID:    token.ID(t.ID),
+			Value: t.Value,
+		},
+	}
+}
 
-	return tokens
+func (l *Parser) addDollarToken(v ast.Vertex) {
+	if l.Lexer.GetWithFreeFloating() == false {
+		return
+	}
+
+	l.setFreeFloating(v, token.Dollar, []token.Token{
+		{
+			ID:    token.ID('$'),
+			Value: []byte("$"),
+		},
+	})
 }
 
 func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {

internal/php7/php7.y

@@ -1368,6 +1368,7 @@ catch_list:
                 yylex.(*Parser).setFreeFloating(catch, token.Start, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(catch, token.Catch, $3.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $5.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(catch, token.Var, $6.Tokens)
                 yylex.(*Parser).setFreeFloating(catch, token.Cond, $7.Tokens)
                 yylex.(*Parser).setFreeFloating(catch, token.Stmts, $9.Tokens)
@@ -2172,6 +2173,8 @@ parameter:
                 }
                 yylex.(*Parser).setFreeFloating($$, token.Variadic, $4.Tokens)
 
+                yylex.(*Parser).addDollarToken(variable)
+
                 // normalize
                 if $3 == nil {
                     yylex.(*Parser).setFreeFloating($$, token.Ampersand, $$.GetNode().Tokens[token.Variadic]); delete($$.GetNode().Tokens, token.Variadic)
@@ -2217,6 +2220,8 @@ parameter:
                 yylex.(*Parser).setFreeFloating($$, token.Variadic, $4.Tokens)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $5.Tokens)
 
+                yylex.(*Parser).addDollarToken(variable)
+
                 // normalize
                 if $3 == nil {
                     yylex.(*Parser).setFreeFloating($$, token.Ampersand, $$.GetNode().Tokens[token.Variadic]); delete($$.GetNode().Tokens, token.Variadic)
@@ -2456,6 +2461,7 @@ static_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -2472,6 +2478,7 @@ static_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $2.Tokens)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
@@ -2981,6 +2988,7 @@ property:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -2997,6 +3005,7 @@ property:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $2.Tokens)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
@@ -4297,6 +4306,7 @@ lexical_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -4314,6 +4324,7 @@ lexical_var:
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $2.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -4943,6 +4954,7 @@ simple_variable:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -5352,6 +5364,7 @@ encaps_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -5367,6 +5380,7 @@ encaps_var:
                 $$.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokensPosition($1, $4)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, append($2.Tokens, yylex.(*Parser).GetFreeFloatingToken($2)...))
                 yylex.(*Parser).setFreeFloating($$, token.Expr, append($4.Tokens, yylex.(*Parser).GetFreeFloatingToken($4)...))
 
@@ -5386,6 +5400,7 @@ encaps_var:
                 $$.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokensPosition($1, $3)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(fetch, token.Start, $3.Tokens)
 
@@ -5521,6 +5536,7 @@ encaps_var_offset:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }

internal/scanner/scanner.rl

@@ -510,6 +510,7 @@ func (lex *Lexer) Lex(lval Lval) int {
 
     token.Tokens = lex.Tokens
 	token.Value = lex.data[lex.ts:lex.te]
+	token.ID = tok
 
     lval.Token(token)
 

internal/scanner/token.go

@@ -6,6 +6,7 @@ import (
 
 // Token value returned by lexer
 type Token struct {
+	ID        TokenID
 	Value     []byte
 	Tokens    []token.Token
 	StartLine int

pkg/printer/printer_parsed_php5_test.go

@@ -2,22 +2,22 @@ package printer_test
 
 import (
 	"bytes"
+	"github.com/z7zmey/php-parser/pkg/ast"
 	"testing"
 
-	"github.com/z7zmey/php-parser/node"
+	"github.com/z7zmey/php-parser/internal/php5"
-	"github.com/z7zmey/php-parser/php5"
+	"github.com/z7zmey/php-parser/pkg/printer"
-	"github.com/z7zmey/php-parser/printer"
 )
 
-func parsePhp5(src string) node.Node {
+func parsePhp5(src string) ast.Vertex {
 	php5parser := php5.NewParser([]byte(src), "5.6")
-	php5parser.WithFreeFloating()
+	php5parser.WithTokens()
 	php5parser.Parse()
 
 	return php5parser.GetRootNode()
 }
 
-func printPhp5(n node.Node) string {
+func printPhp5(n ast.Vertex) string {
 	o := bytes.NewBufferString("")
 
 	p := printer.NewPrinter(o)

pkg/printer/printer_parsed_php7_test.go

@@ -2,14 +2,12 @@ package printer_test
 
 import (
 	"bytes"
+	"github.com/z7zmey/php-parser/pkg/ast"
 	"os"
 	"testing"
 
-	"github.com/z7zmey/php-parser/node"
+	"github.com/z7zmey/php-parser/internal/php7"
-	"github.com/z7zmey/php-parser/node/name"
+	"github.com/z7zmey/php-parser/pkg/printer"
-	"github.com/z7zmey/php-parser/node/stmt"
-	"github.com/z7zmey/php-parser/php7"
-	"github.com/z7zmey/php-parser/printer"
 )
 
 func ExamplePrinter() {
@@ -30,15 +28,15 @@ abstract class Bar extends Baz
 	// parse
 
 	php7parser := php7.NewParser([]byte(src), "7.4")
-	php7parser.WithFreeFloating()
+	php7parser.WithTokens()
 	php7parser.Parse()
 
 	rootNode := php7parser.GetRootNode()
 
 	// change namespace
 
-	parts := &rootNode.(*node.Root).Stmts[0].(*stmt.Namespace).NamespaceName.(*name.Name).Parts
+	parts := &rootNode.(*ast.Root).Stmts[0].(*ast.StmtNamespace).NamespaceName.(*ast.NameName).Parts
-	*parts = append(*parts, &name.NamePart{Value: "Quuz"})
+	*parts = append(*parts, &ast.NameNamePart{Value: []byte("Quuz")})
 
 	// print
 
@@ -60,15 +58,15 @@ abstract class Bar extends Baz
 	// }
 }
 
-func parse(src string) node.Node {
+func parse(src string) ast.Vertex {
 	php7parser := php7.NewParser([]byte(src), "7.4")
-	php7parser.WithFreeFloating()
+	php7parser.WithTokens()
 	php7parser.Parse()
 
 	return php7parser.GetRootNode()
 }
 
-func print(n node.Node) string {
+func print(n ast.Vertex) string {
 	o := bytes.NewBufferString("")
 
 	p := printer.NewPrinter(o)

pkg/token/position.go

@@ -2,8 +2,6 @@ package token
 
 type Position int
 
-type Collection map[Position][]Token
-
 //go:generate stringer -type=Position -output ./position_string.go
 const (
 	Start Position = iota
@@ -86,3 +84,14 @@ const (
 	OpenParenthesisToken
 	CloseParenthesisToken
 )
+
+type Collection map[Position][]Token
+
+func (c Collection) IsEmpty() bool {
+	for _, v := range c {
+		if len(v) > 0 {
+			return false
+		}
+	}
+	return true
+}