refactor printer

cmd/php-parser/main.go

@@ -13,8 +13,8 @@ import (
 
 	"github.com/pkg/profile"
 	"github.com/yookoala/realpath"
-	"github.com/z7zmey/php-parser/pkg/parser"
 	"github.com/z7zmey/php-parser/pkg/ast/visitor"
+	"github.com/z7zmey/php-parser/pkg/parser"
 )
 
 var wg sync.WaitGroup

internal/php5/parser.go

@@ -2,6 +2,7 @@ package php5
 
 import (
 	"bytes"
+	"fmt"
 
 	"github.com/z7zmey/php-parser/internal/positionbuilder"
 	"github.com/z7zmey/php-parser/internal/scanner"
@@ -132,10 +133,25 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
 		return []token.Token{}
 	}
 
-	tokens := make([]token.Token, len(t.Tokens))
+	return []token.Token{
-	copy(tokens, t.Tokens)
+		{
+			ID:    token.ID(t.ID),
+			Value: t.Value,
+		},
+	}
+}
 
-	return tokens
+func (l *Parser) addDollarToken(v ast.Vertex) {
+	if l.Lexer.GetWithFreeFloating() == false {
+		return
+	}
+
+	l.setFreeFloating(v, token.Dollar, []token.Token{
+		{
+			ID:    token.ID('$'),
+			Value: []byte("$"),
+		},
+	})
 }
 
 func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {
@@ -159,6 +175,8 @@ func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.
 	}
 
 	vlen := len(semiColon[0].Value)
+	fmt.Printf("vlen: %q\n", string(semiColon[0].Value))
+
 	tlen := 2
 	if bytes.HasSuffix(semiColon[0].Value, []byte("?>\n")) {
 		tlen = 3

internal/php5/php5.y

@@ -1431,6 +1431,7 @@ catch_statement:
                 yylex.(*Parser).setFreeFloating(catchNode, token.Start, $1.Tokens)
                 yylex.(*Parser).setFreeFloating(catchNode, token.Catch, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $4.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(catchNode, token.Var, $5.Tokens)
                 yylex.(*Parser).setFreeFloating(catchNode, token.Cond, $6.Tokens)
                 yylex.(*Parser).setFreeFloating(catchNode, token.Stmts, $8.Tokens)
@@ -1508,6 +1509,7 @@ additional_catch:
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
                 yylex.(*Parser).setFreeFloating($$, token.Catch, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $4.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $5.Tokens)
                 yylex.(*Parser).setFreeFloating($$, token.Cond, $6.Tokens)
                 yylex.(*Parser).setFreeFloating($$, token.Stmts, $8.Tokens)
@@ -2309,6 +2311,7 @@ parameter:
                     yylex.(*Parser).setFreeFloating($$, token.Ampersand, $3.Tokens)
                 }
                 yylex.(*Parser).setFreeFloating($$, token.Variadic, $4.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 // normalize
                 if $3 == nil {
@@ -2354,6 +2357,7 @@ parameter:
                 }
                 yylex.(*Parser).setFreeFloating($$, token.Variadic, $4.Tokens)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $5.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 // normalize
                 if $3 == nil {
@@ -2556,6 +2560,7 @@ global_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -2605,6 +2610,7 @@ static_var_list:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Start, $3.Tokens)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
@@ -2623,6 +2629,7 @@ static_var_list:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Start, $3.Tokens)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Var, $4.Tokens)
 
@@ -2641,6 +2648,7 @@ static_var_list:
                 staticVar.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokenPosition($1)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Start, $1.Tokens)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
@@ -2658,6 +2666,7 @@ static_var_list:
                 staticVar.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokenNodePosition($1, $3)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Start, $1.Tokens)
                 yylex.(*Parser).setFreeFloating(staticVar, token.Var, $2.Tokens)
 
@@ -3155,6 +3164,7 @@ class_variable_declaration:
                 property.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokenPosition($3)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(property, token.Start, $3.Tokens)
 
@@ -3174,6 +3184,7 @@ class_variable_declaration:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(property, token.Start, $3.Tokens)
                 yylex.(*Parser).setFreeFloating(property, token.Var, $4.Tokens)
 
@@ -3192,6 +3203,7 @@ class_variable_declaration:
                 property.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokenPosition($1)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(property, token.Start, $1.Tokens)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
@@ -3209,6 +3221,7 @@ class_variable_declaration:
                 property.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokenNodePosition($1, $3)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(property, token.Start, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(property, token.Var, $2.Tokens)
 
@@ -4589,6 +4602,7 @@ lexical_var_list:
                 // save comments
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $3.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -4608,6 +4622,7 @@ lexical_var_list:
                 yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(reference, token.Start, $3.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $4.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -4623,6 +4638,7 @@ lexical_var_list:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -4641,6 +4657,7 @@ lexical_var_list:
                 // save comments
                 yylex.(*Parser).setFreeFloating(reference, token.Start, $1.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $2.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -6381,6 +6398,7 @@ compound_variable:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -6807,6 +6825,7 @@ encaps_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -6822,6 +6841,7 @@ encaps_var:
                 $$.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokensPosition($1, $4)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, append($2.Tokens, yylex.(*Parser).GetFreeFloatingToken($2)...))
                 yylex.(*Parser).setFreeFloating($$, token.Expr, append($4.Tokens, yylex.(*Parser).GetFreeFloatingToken($4)...))
 
@@ -6841,6 +6861,7 @@ encaps_var:
                 $$.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokensPosition($1, $3)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(fetch, token.Start, $3.Tokens)
 
@@ -6950,6 +6971,7 @@ encaps_var_offset:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }

internal/php7/parser.go

@@ -131,10 +131,25 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []token.Token {
 		return []token.Token{}
 	}
 
-	tokens := make([]token.Token, len(t.Tokens))
+	return []token.Token{
-	copy(tokens, t.Tokens)
+		{
+			ID:    token.ID(t.ID),
+			Value: t.Value,
+		},
+	}
+}
 
-	return tokens
+func (l *Parser) addDollarToken(v ast.Vertex) {
+	if l.Lexer.GetWithFreeFloating() == false {
+		return
+	}
+
+	l.setFreeFloating(v, token.Dollar, []token.Token{
+		{
+			ID:    token.ID('$'),
+			Value: []byte("$"),
+		},
+	})
 }
 
 func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode ast.Vertex, prevNode ast.Vertex) {

internal/php7/parser_test.go

@@ -982,7 +982,7 @@ func TestPhp7ParameterNode(t *testing.T) {
 						EndPos:    51,
 					},
 				},
-				ReturnsRef:    false,
+				ReturnsRef: false,
 				FunctionName: &ast.Identifier{
 					Node: ast.Node{
 						Position: &position.Position{
@@ -1183,7 +1183,7 @@ func TestPhp7ParameterNode(t *testing.T) {
 								EndPos:    117,
 							},
 						},
-						ReturnsRef:    false,
+						ReturnsRef: false,
 						MethodName: &ast.Identifier{
 							Node: ast.Node{
 								Position: &position.Position{
@@ -1397,8 +1397,8 @@ func TestPhp7ParameterNode(t *testing.T) {
 							EndPos:    162,
 						},
 					},
-					ReturnsRef:    false,
+					ReturnsRef: false,
-					Static:        false,
+					Static:     false,
 					Params: []ast.Vertex{
 						&ast.Parameter{
 							Node: ast.Node{
@@ -1577,8 +1577,8 @@ func TestPhp7ParameterNode(t *testing.T) {
 							EndPos:    214,
 						},
 					},
-					Static:        true,
+					Static:     true,
-					ReturnsRef:    false,
+					ReturnsRef: false,
 					Params: []ast.Vertex{
 						&ast.Parameter{
 							Node: ast.Node{
@@ -4376,7 +4376,7 @@ func TestStmtClassMethod_SimpleClassMethod(t *testing.T) {
 								EndPos:    31,
 							},
 						},
-						ReturnsRef:    false,
+						ReturnsRef: false,
 						MethodName: &ast.Identifier{
 							Node: ast.Node{
 								Position: &position.Position{
@@ -4454,7 +4454,7 @@ func TestStmtClassMethod_PrivateProtectedClassMethod(t *testing.T) {
 								EndPos:    45,
 							},
 						},
-						ReturnsRef:    false,
+						ReturnsRef: false,
 						MethodName: &ast.Identifier{
 							Node: ast.Node{
 								Position: &position.Position{
@@ -4511,7 +4511,7 @@ func TestStmtClassMethod_PrivateProtectedClassMethod(t *testing.T) {
 								EndPos:    73,
 							},
 						},
-						ReturnsRef:    false,
+						ReturnsRef: false,
 						MethodName: &ast.Identifier{
 							Node: ast.Node{
 								Position: &position.Position{
@@ -4602,7 +4602,7 @@ func TestStmtClassMethod_Php7ClassMethod(t *testing.T) {
 								EndPos:    52,
 							},
 						},
-						ReturnsRef:    true,
+						ReturnsRef: true,
 						MethodName: &ast.Identifier{
 							Node: ast.Node{
 								Position: &position.Position{
@@ -4740,7 +4740,7 @@ func TestStmtClassMethod_AbstractClassMethod(t *testing.T) {
 								EndPos:    54,
 							},
 						},
-						ReturnsRef:    false,
+						ReturnsRef: false,
 						MethodName: &ast.Identifier{
 							Node: ast.Node{
 								Position: &position.Position{
@@ -4854,7 +4854,7 @@ func TestStmtClassMethod_Php7AbstractClassMethod(t *testing.T) {
 								EndPos:    51,
 							},
 						},
-						ReturnsRef:    false,
+						ReturnsRef: false,
 						MethodName: &ast.Identifier{
 							Node: ast.Node{
 								Position: &position.Position{
@@ -7300,7 +7300,7 @@ func TestStmtFunction(t *testing.T) {
 						EndPos:    20,
 					},
 				},
-				ReturnsRef:    false,
+				ReturnsRef: false,
 				FunctionName: &ast.Identifier{
 					Node: ast.Node{
 						Position: &position.Position{
@@ -7345,7 +7345,7 @@ func TestStmtFunction_Return(t *testing.T) {
 						EndPos:    27,
 					},
 				},
-				ReturnsRef:    false,
+				ReturnsRef: false,
 				FunctionName: &ast.Identifier{
 					Node: ast.Node{
 						Position: &position.Position{
@@ -7401,7 +7401,7 @@ func TestStmtFunction_ReturnVar(t *testing.T) {
 						EndPos:    51,
 					},
 				},
-				ReturnsRef:    false,
+				ReturnsRef: false,
 				FunctionName: &ast.Identifier{
 					Node: ast.Node{
 						Position: &position.Position{
@@ -7568,7 +7568,7 @@ func TestStmtFunction_Ref(t *testing.T) {
 						EndPos:    30,
 					},
 				},
-				ReturnsRef:    true,
+				ReturnsRef: true,
 				FunctionName: &ast.Identifier{
 					Node: ast.Node{
 						Position: &position.Position{
@@ -7635,7 +7635,7 @@ func TestStmtFunction_ReturnType(t *testing.T) {
 						EndPos:    27,
 					},
 				},
-				ReturnsRef:    true,
+				ReturnsRef: true,
 				FunctionName: &ast.Identifier{
 					Node: ast.Node{
 						Position: &position.Position{
@@ -14535,8 +14535,8 @@ func TestExprArrowFunction(t *testing.T) {
 							EndPos:    13,
 						},
 					},
-					ReturnsRef:    false,
+					ReturnsRef: false,
-					Static:        false,
+					Static:     false,
 					Expr: &ast.ExprVariable{
 						Node: ast.Node{
 							Position: &position.Position{
@@ -14600,8 +14600,8 @@ func TestExprArrowFunction_ReturnType(t *testing.T) {
 							EndPos:    22,
 						},
 					},
-					Static:        false,
+					Static:     false,
-					ReturnsRef:    true,
+					ReturnsRef: true,
 					ReturnType: &ast.NameName{
 						Node: ast.Node{
 							Position: &position.Position{
@@ -15080,9 +15080,9 @@ func TestExprClosure(t *testing.T) {
 							EndPos:    15,
 						},
 					},
-					ReturnsRef:    false,
+					ReturnsRef: false,
-					Static:        false,
+					Static:     false,
-					Stmts:         []ast.Vertex{},
+					Stmts:      []ast.Vertex{},
 				},
 			},
 		},
@@ -15125,8 +15125,8 @@ func TestExprClosure_Use(t *testing.T) {
 							EndPos:    36,
 						},
 					},
-					ReturnsRef:    false,
+					ReturnsRef: false,
-					Static:        false,
+					Static:     false,
 					Params: []ast.Vertex{
 						&ast.Parameter{
 							Node: ast.Node{
@@ -15302,8 +15302,8 @@ func TestExprClosure_Use2(t *testing.T) {
 							EndPos:    36,
 						},
 					},
-					ReturnsRef:    false,
+					ReturnsRef: false,
-					Static:        false,
+					Static:     false,
 					Params: []ast.Vertex{
 						&ast.Parameter{
 							Node: ast.Node{
@@ -15479,8 +15479,8 @@ func TestExprClosure_ReturnType(t *testing.T) {
 							EndPos:    22,
 						},
 					},
-					ReturnsRef:    false,
+					ReturnsRef: false,
-					Static:        false,
+					Static:     false,
 					ReturnType: &ast.NameName{
 						Node: ast.Node{
 							Position: &position.Position{

internal/php7/php7.y

@@ -1368,6 +1368,7 @@ catch_list:
                 yylex.(*Parser).setFreeFloating(catch, token.Start, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(catch, token.Catch, $3.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $5.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating(catch, token.Var, $6.Tokens)
                 yylex.(*Parser).setFreeFloating(catch, token.Cond, $7.Tokens)
                 yylex.(*Parser).setFreeFloating(catch, token.Stmts, $9.Tokens)
@@ -2172,6 +2173,8 @@ parameter:
                 }
                 yylex.(*Parser).setFreeFloating($$, token.Variadic, $4.Tokens)
 
+                yylex.(*Parser).addDollarToken(variable)
+
                 // normalize
                 if $3 == nil {
                     yylex.(*Parser).setFreeFloating($$, token.Ampersand, $$.GetNode().Tokens[token.Variadic]); delete($$.GetNode().Tokens, token.Variadic)
@@ -2217,6 +2220,8 @@ parameter:
                 yylex.(*Parser).setFreeFloating($$, token.Variadic, $4.Tokens)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $5.Tokens)
 
+                yylex.(*Parser).addDollarToken(variable)
+
                 // normalize
                 if $3 == nil {
                     yylex.(*Parser).setFreeFloating($$, token.Ampersand, $$.GetNode().Tokens[token.Variadic]); delete($$.GetNode().Tokens, token.Variadic)
@@ -2456,6 +2461,7 @@ static_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -2472,6 +2478,7 @@ static_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $2.Tokens)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
@@ -2981,6 +2988,7 @@ property:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -2997,6 +3005,7 @@ property:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $2.Tokens)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
@@ -4297,6 +4306,7 @@ lexical_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -4314,6 +4324,7 @@ lexical_var:
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
                 yylex.(*Parser).setFreeFloating(variable, token.Start, $2.Tokens)
+                yylex.(*Parser).addDollarToken(variable)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -4943,6 +4954,7 @@ simple_variable:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -5352,6 +5364,7 @@ encaps_var:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }
@@ -5367,6 +5380,7 @@ encaps_var:
                 $$.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokensPosition($1, $4)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, append($2.Tokens, yylex.(*Parser).GetFreeFloatingToken($2)...))
                 yylex.(*Parser).setFreeFloating($$, token.Expr, append($4.Tokens, yylex.(*Parser).GetFreeFloatingToken($4)...))
 
@@ -5386,6 +5400,7 @@ encaps_var:
                 $$.GetNode().Position = yylex.(*Parser).positionBuilder.NewTokensPosition($1, $3)
 
                 // save comments
+                yylex.(*Parser).addDollarToken(variable)
                 yylex.(*Parser).setFreeFloating($$, token.Var, $2.Tokens)
                 yylex.(*Parser).setFreeFloating(fetch, token.Start, $3.Tokens)
 
@@ -5521,6 +5536,7 @@ encaps_var_offset:
 
                 // save comments
                 yylex.(*Parser).setFreeFloating($$, token.Start, $1.Tokens)
+                yylex.(*Parser).addDollarToken($$)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }

internal/scanner/scanner.rl

@@ -510,6 +510,7 @@ func (lex *Lexer) Lex(lval Lval) int {
 
     token.Tokens = lex.Tokens
 	token.Value = lex.data[lex.ts:lex.te]
+	token.ID = tok
 
     lval.Token(token)
 

internal/scanner/token.go

@@ -6,6 +6,7 @@ import (
 
 // Token value returned by lexer
 type Token struct {
+	ID        TokenID
 	Value     []byte
 	Tokens    []token.Token
 	StartLine int

pkg/ast/visitor/namespace_resolver_test.go

@@ -194,8 +194,8 @@ func TestResolveInstanceCatch(t *testing.T) {
 							nameBC,
 							nameF,
 						},
-						Var: &ast.ExprVariable{VarName: &ast.Identifier{Value: []byte("foo")}},
+						Var:   &ast.ExprVariable{VarName: &ast.Identifier{Value: []byte("foo")}},
-						Stmts:    []ast.Vertex{},
+						Stmts: []ast.Vertex{},
 					},
 				},
 			},
@@ -407,7 +407,7 @@ func TestResolveClassName(t *testing.T) {
 	nameBC := &ast.NameName{Parts: []ast.Vertex{&ast.NameNamePart{Value: []byte("B")}, &ast.NameNamePart{Value: []byte("C")}}}
 
 	class := &ast.StmtClass{
-		ClassName:     &ast.Identifier{Value: []byte("A")},
+		ClassName: &ast.Identifier{Value: []byte("A")},
 		Extends: &ast.StmtClassExtends{
 			ClassName: nameAB,
 		},
@@ -472,8 +472,8 @@ func TestResolveInterfaceName(t *testing.T) {
 
 func TestResolveTraitName(t *testing.T) {
 	traitNode := &ast.StmtTrait{
-		TraitName:     &ast.Identifier{Value: []byte("A")},
+		TraitName: &ast.Identifier{Value: []byte("A")},
-		Stmts:         []ast.Vertex{},
+		Stmts:     []ast.Vertex{},
 	}
 
 	stxTree := &ast.StmtStmtList{
@@ -498,14 +498,14 @@ func TestResolveFunctionName(t *testing.T) {
 	nameBC := &ast.NameName{Parts: []ast.Vertex{&ast.NameNamePart{Value: []byte("B")}, &ast.NameNamePart{Value: []byte("C")}}}
 
 	functionNode := &ast.StmtFunction{
-		ReturnsRef:    false,
+		ReturnsRef:   false,
-		FunctionName:  &ast.Identifier{Value: []byte("A")},
+		FunctionName: &ast.Identifier{Value: []byte("A")},
 		Params: []ast.Vertex{
 			&ast.Parameter{
-				ByRef:        false,
+				ByRef:    false,
-				Variadic:     false,
+				Variadic: false,
-				Type: nameAB,
+				Type:     nameAB,
-				Var:     &ast.ExprVariable{VarName: &ast.Identifier{Value: []byte("foo")}},
+				Var:      &ast.ExprVariable{VarName: &ast.Identifier{Value: []byte("foo")}},
 			},
 		},
 		ReturnType: &ast.Nullable{Expr: nameBC},
@@ -536,14 +536,14 @@ func TestResolveMethodName(t *testing.T) {
 	nameBC := &ast.NameName{Parts: []ast.Vertex{&ast.NameNamePart{Value: []byte("B")}, &ast.NameNamePart{Value: []byte("C")}}}
 
 	methodNode := &ast.StmtClassMethod{
-		ReturnsRef:    false,
+		ReturnsRef: false,
-		MethodName:    &ast.Identifier{Value: []byte("A")},
+		MethodName: &ast.Identifier{Value: []byte("A")},
 		Params: []ast.Vertex{
 			&ast.Parameter{
-				ByRef:        false,
+				ByRef:    false,
-				Variadic:     false,
+				Variadic: false,
-				Type: nameAB,
+				Type:     nameAB,
-				Var:     &ast.ExprVariable{VarName: &ast.Identifier{Value: []byte("foo")}},
+				Var:      &ast.ExprVariable{VarName: &ast.Identifier{Value: []byte("foo")}},
 			},
 		},
 		ReturnType: &ast.Nullable{Expr: nameBC},
@@ -569,14 +569,14 @@ func TestResolveClosureName(t *testing.T) {
 	nameBC := &ast.NameName{Parts: []ast.Vertex{&ast.NameNamePart{Value: []byte("B")}, &ast.NameNamePart{Value: []byte("C")}}}
 
 	closureNode := &ast.ExprClosure{
-		ReturnsRef:    false,
+		ReturnsRef: false,
-		Static:        false,
+		Static:     false,
 		Params: []ast.Vertex{
 			&ast.Parameter{
-				ByRef:        false,
+				ByRef:    false,
-				Variadic:     false,
+				Variadic: false,
-				Type: nameAB,
+				Type:     nameAB,
-				Var:     &ast.ExprVariable{VarName: &ast.Identifier{Value: []byte("foo")}},
+				Var:      &ast.ExprVariable{VarName: &ast.Identifier{Value: []byte("foo")}},
 			},
 		},
 		ClosureUse: nil,
@@ -600,12 +600,12 @@ func TestResolveConstantsName(t *testing.T) {
 	nameAB := &ast.NameName{Parts: []ast.Vertex{&ast.NameNamePart{Value: []byte("A")}, &ast.NameNamePart{Value: []byte("B")}}}
 
 	constantB := &ast.StmtConstant{
-		ConstantName:  &ast.Identifier{Value: []byte("B")},
+		ConstantName: &ast.Identifier{Value: []byte("B")},
-		Expr:          &ast.ScalarLnumber{Value: []byte("1")},
+		Expr:         &ast.ScalarLnumber{Value: []byte("1")},
 	}
 	constantC := &ast.StmtConstant{
-		ConstantName:  &ast.Identifier{Value: []byte("C")},
+		ConstantName: &ast.Identifier{Value: []byte("C")},
-		Expr:          &ast.ScalarLnumber{Value: []byte("1")},
+		Expr:         &ast.ScalarLnumber{Value: []byte("1")},
 	}
 
 	stxTree := &ast.StmtStmtList{
@@ -644,12 +644,12 @@ func TestResolveNamespaces(t *testing.T) {
 	relativeNameCE := &ast.NameRelative{Parts: []ast.Vertex{&ast.NameNamePart{Value: []byte("C")}, &ast.NameNamePart{Value: []byte("E")}}}
 
 	constantB := &ast.StmtConstant{
-		ConstantName:  &ast.Identifier{Value: []byte("B")},
+		ConstantName: &ast.Identifier{Value: []byte("B")},
-		Expr:          &ast.ScalarLnumber{Value: []byte("1")},
+		Expr:         &ast.ScalarLnumber{Value: []byte("1")},
 	}
 	constantC := &ast.StmtConstant{
-		ConstantName:  &ast.Identifier{Value: []byte("C")},
+		ConstantName: &ast.Identifier{Value: []byte("C")},
-		Expr:          &ast.ScalarLnumber{Value: []byte("1")},
+		Expr:         &ast.ScalarLnumber{Value: []byte("1")},
 	}
 
 	stxTree := &ast.StmtStmtList{

pkg/printer/printer_parsed_php5_test.go

@@ -2,22 +2,22 @@ package printer_test
 
 import (
 	"bytes"
+	"github.com/z7zmey/php-parser/pkg/ast"
 	"testing"
 
-	"github.com/z7zmey/php-parser/node"
+	"github.com/z7zmey/php-parser/internal/php5"
-	"github.com/z7zmey/php-parser/php5"
+	"github.com/z7zmey/php-parser/pkg/printer"
-	"github.com/z7zmey/php-parser/printer"
 )
 
-func parsePhp5(src string) node.Node {
+func parsePhp5(src string) ast.Vertex {
 	php5parser := php5.NewParser([]byte(src), "5.6")
-	php5parser.WithFreeFloating()
+	php5parser.WithTokens()
 	php5parser.Parse()
 
 	return php5parser.GetRootNode()
 }
 
-func printPhp5(n node.Node) string {
+func printPhp5(n ast.Vertex) string {
 	o := bytes.NewBufferString("")
 
 	p := printer.NewPrinter(o)

pkg/printer/printer_parsed_php7_test.go

@@ -2,14 +2,12 @@ package printer_test
 
 import (
 	"bytes"
+	"github.com/z7zmey/php-parser/pkg/ast"
 	"os"
 	"testing"
 
-	"github.com/z7zmey/php-parser/node"
+	"github.com/z7zmey/php-parser/internal/php7"
-	"github.com/z7zmey/php-parser/node/name"
+	"github.com/z7zmey/php-parser/pkg/printer"
-	"github.com/z7zmey/php-parser/node/stmt"
-	"github.com/z7zmey/php-parser/php7"
-	"github.com/z7zmey/php-parser/printer"
 )
 
 func ExamplePrinter() {
@@ -30,15 +28,15 @@ abstract class Bar extends Baz
 	// parse
 
 	php7parser := php7.NewParser([]byte(src), "7.4")
-	php7parser.WithFreeFloating()
+	php7parser.WithTokens()
 	php7parser.Parse()
 
 	rootNode := php7parser.GetRootNode()
 
 	// change namespace
 
-	parts := &rootNode.(*node.Root).Stmts[0].(*stmt.Namespace).NamespaceName.(*name.Name).Parts
+	parts := &rootNode.(*ast.Root).Stmts[0].(*ast.StmtNamespace).NamespaceName.(*ast.NameName).Parts
-	*parts = append(*parts, &name.NamePart{Value: "Quuz"})
+	*parts = append(*parts, &ast.NameNamePart{Value: []byte("Quuz")})
 
 	// print
 
@@ -60,15 +58,15 @@ abstract class Bar extends Baz
 	// }
 }
 
-func parse(src string) node.Node {
+func parse(src string) ast.Vertex {
 	php7parser := php7.NewParser([]byte(src), "7.4")
-	php7parser.WithFreeFloating()
+	php7parser.WithTokens()
 	php7parser.Parse()
 
 	return php7parser.GetRootNode()
 }
 
-func print(n node.Node) string {
+func print(n ast.Vertex) string {
 	o := bytes.NewBufferString("")
 
 	p := printer.NewPrinter(o)

pkg/token/position.go

@@ -2,8 +2,6 @@ package token
 
 type Position int
 
-type Collection map[Position][]Token
-
 //go:generate stringer -type=Position -output ./position_string.go
 const (
 	Start Position = iota
@@ -86,3 +84,14 @@ const (
 	OpenParenthesisToken
 	CloseParenthesisToken
 )
+
+type Collection map[Position][]Token
+
+func (c Collection) IsEmpty() bool {
+	for _, v := range c {
+		if len(v) > 0 {
+			return false
+		}
+	}
+	return true
+}