[refactoring] update ast structure of "For", "TraitUseAlias" and "TraitUsePrecedence" nodes

internal/php5/php5.y

@@ -240,7 +240,7 @@ import (
 %type <node> switch_case_list non_empty_function_call_parameter_list assignment_list lexical_var_list
 %type <node> method_body trait_reference_list static_array_pair_list non_empty_static_array_pair_list
 %type <node> foreach_statement for_statement while_statement isset_variables
-%type <node> foreach_variable foreach_optional_arg
+%type <node> foreach_variable foreach_optional_arg for_expr non_empty_for_expr
 %type <node> extends_from interface_list trait_list
 %type <node> implements_list
 %type <node> interface_extends_list
@@ -248,8 +248,8 @@ import (
 
 %type <list> top_statement_list namespace_name use_declarations use_function_declarations use_const_declarations
 %type <list> inner_statement_list encaps_list
-%type <list> elseif_list new_elseif_list non_empty_for_expr
+%type <list> elseif_list new_elseif_list
-%type <list> for_expr case_list catch_statement additional_catches
+%type <list> case_list catch_statement additional_catches
 %type <list> non_empty_additional_catches class_statement_list
 %type <list> class_statement_list variable_modifiers method_modifiers
 %type <list> trait_adaptation_list non_empty_trait_adaptation_list
@@ -938,11 +938,14 @@ unticked_statement:
             {
                 $9.(*ast.StmtFor).ForTkn = $1
                 $9.(*ast.StmtFor).OpenParenthesisTkn = $2
-                $9.(*ast.StmtFor).Init = $3
+                $9.(*ast.StmtFor).Init = $3.(*ast.ParserSeparatedList).Items
+                $9.(*ast.StmtFor).InitSeparatorTkns = $3.(*ast.ParserSeparatedList).SeparatorTkns
                 $9.(*ast.StmtFor).InitSemiColonTkn = $4
-                $9.(*ast.StmtFor).Cond = $5
+                $9.(*ast.StmtFor).Cond = $5.(*ast.ParserSeparatedList).Items
+                $9.(*ast.StmtFor).CondSeparatorTkns = $5.(*ast.ParserSeparatedList).SeparatorTkns
                 $9.(*ast.StmtFor).CondSemiColonTkn = $6
-                $9.(*ast.StmtFor).Loop = $7
+                $9.(*ast.StmtFor).Loop = $7.(*ast.ParserSeparatedList).Items
+                $9.(*ast.StmtFor).LoopSeparatorTkns = $7.(*ast.ParserSeparatedList).SeparatorTkns
                 $9.(*ast.StmtFor).CloseParenthesisTkn = $8
                 $9.(*ast.StmtFor).Node.Position = position.NewTokenNodePosition($1, $9)
 
@@ -2537,19 +2540,15 @@ non_empty_trait_adaptation_list:
 trait_adaptation_statement:
         trait_precedence ';'
             {
-                $$ = $1;
+                $1.(*ast.StmtTraitUsePrecedence).SemiColonTkn = $2
 
-                // save comments
+                $$ = $1;
-                yylex.(*Parser).setFreeFloating($$, token.NameList, $2.SkippedTokens)
-                yylex.(*Parser).setToken($$, token.SemiColon, $2.SkippedTokens)
             }
     |   trait_alias ';'
             {
-                $$ = $1;
+                $1.(*ast.StmtTraitUseAlias).SemiColonTkn = $2
 
-                // save comments
+                $$ = $1;
-                yylex.(*Parser).setFreeFloating($$, token.Alias, $2.SkippedTokens)
-                yylex.(*Parser).setToken($$, token.SemiColon, $2.SkippedTokens)
             }
 ;
 
@@ -2976,7 +2975,7 @@ echo_expr_list:
 for_expr:
         /* empty */
             {
-                $$ = nil
+                $$ = &ast.ParserSeparatedList{}
             }
     |   non_empty_for_expr
             {
@@ -2987,14 +2986,16 @@ for_expr:
 non_empty_for_expr:
         non_empty_for_expr ',' expr
             {
-                $$ = append($1, $3)
+                $1.(*ast.ParserSeparatedList).SeparatorTkns = append($1.(*ast.ParserSeparatedList).SeparatorTkns, $2)
+                $1.(*ast.ParserSeparatedList).Items = append($1.(*ast.ParserSeparatedList).Items, $3)
 
-                // save comments
+                $$ = $1
-                yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.SkippedTokens)
             }
     |   expr
             {
-                $$ = []ast.Vertex{$1}
+                $$ = &ast.ParserSeparatedList{
+                    Items: []ast.Vertex{$1},
+                }
             }
 ;
 
@@ -4461,10 +4462,9 @@ dynamic_class_name_variable_properties:
 dynamic_class_name_variable_property:
         T_OBJECT_OPERATOR object_property
             {
-                $$ = $2
+                $2[0].(*ast.ExprPropertyFetch).ObjectOperatorTkn = $1
 
-                // save comments
+                $$ = $2
-                yylex.(*Parser).setFreeFloating($2[0], token.Var, $1.SkippedTokens)
             }
 ;
 

internal/php7/php7.y

@@ -234,7 +234,7 @@ import (
 %type <node> interface_declaration_statement
 %type <node> group_use_declaration inline_use_declaration
 %type <node> mixed_group_use_declaration use_declaration unprefixed_use_declaration
-%type <node> const_decl inner_statement
+%type <node> const_decl inner_statement for_exprs non_empty_for_exprs
 %type <node> expr optional_expr parameter_list non_empty_parameter_list
 %type <node> declare_statement finally_statement unset_variable variable
 %type <node> parameter optional_type argument expr_without_variable global_var_list global_var
@@ -272,7 +272,6 @@ import (
 
 
 %type <list> encaps_list backticks_expr namespace_name catch_list class_const_list
-%type <list> for_exprs non_empty_for_exprs
 %type <list> unprefixed_use_declarations inline_use_declarations
 %type <list> case_list trait_adaptation_list
 %type <list> use_declarations
@@ -900,11 +899,14 @@ statement:
             {
                 $9.(*ast.StmtFor).ForTkn = $1
                 $9.(*ast.StmtFor).OpenParenthesisTkn = $2
-                $9.(*ast.StmtFor).Init = $3
+                $9.(*ast.StmtFor).Init = $3.(*ast.ParserSeparatedList).Items
+                $9.(*ast.StmtFor).InitSeparatorTkns = $3.(*ast.ParserSeparatedList).SeparatorTkns
                 $9.(*ast.StmtFor).InitSemiColonTkn = $4
-                $9.(*ast.StmtFor).Cond = $5
+                $9.(*ast.StmtFor).Cond = $5.(*ast.ParserSeparatedList).Items
+                $9.(*ast.StmtFor).CondSeparatorTkns = $5.(*ast.ParserSeparatedList).SeparatorTkns
                 $9.(*ast.StmtFor).CondSemiColonTkn = $6
-                $9.(*ast.StmtFor).Loop = $7
+                $9.(*ast.StmtFor).Loop = $7.(*ast.ParserSeparatedList).Items
+                $9.(*ast.StmtFor).LoopSeparatorTkns = $7.(*ast.ParserSeparatedList).SeparatorTkns
                 $9.(*ast.StmtFor).CloseParenthesisTkn = $8
                 $9.(*ast.StmtFor).Node.Position = position.NewTokenNodePosition($1, $9)
 
@@ -2277,19 +2279,15 @@ trait_adaptation_list:
 trait_adaptation:
         trait_precedence ';'
             {
-                $$ = $1;
+                $1.(*ast.StmtTraitUsePrecedence).SemiColonTkn = $2
 
-                // save comments
+                $$ = $1;
-                yylex.(*Parser).setFreeFloating($$, token.NameList, $2.SkippedTokens)
-                yylex.(*Parser).setToken($$, token.SemiColon, $2.SkippedTokens)
             }
     |   trait_alias ';'
             {
-                $$ = $1;
+                $1.(*ast.StmtTraitUseAlias).SemiColonTkn = $2
 
-                // save comments
+                $$ = $1;
-                yylex.(*Parser).setFreeFloating($$, token.Alias, $2.SkippedTokens)
-                yylex.(*Parser).setToken($$, token.SemiColon, $2.SkippedTokens)
             }
 ;
 
@@ -2688,7 +2686,7 @@ echo_expr:
 for_exprs:
         /* empty */
             {
-                $$ = nil;
+                $$ = &ast.ParserSeparatedList{}
             }
     |   non_empty_for_exprs
             {
@@ -2699,14 +2697,16 @@ for_exprs:
 non_empty_for_exprs:
         non_empty_for_exprs ',' expr
             {
-                $$ = append($1, $3)
+                $1.(*ast.ParserSeparatedList).SeparatorTkns = append($1.(*ast.ParserSeparatedList).SeparatorTkns, $2)
+                $1.(*ast.ParserSeparatedList).Items = append($1.(*ast.ParserSeparatedList).Items, $3)
 
-                // save comments
+                $$ = $1
-                yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.SkippedTokens)
             }
     |   expr
             {
-                $$ = []ast.Vertex{$1}
+                $$ = &ast.ParserSeparatedList{
+                    Items: []ast.Vertex{$1},
+                }
             }
 ;
 

pkg/ast/node.go

@@ -438,10 +438,13 @@ type StmtFor struct {
 	ForTkn              *token.Token
 	OpenParenthesisTkn  *token.Token
 	Init                []Vertex
+	InitSeparatorTkns   []*token.Token
 	InitSemiColonTkn    *token.Token
 	Cond                []Vertex
+	CondSeparatorTkns   []*token.Token
 	CondSemiColonTkn    *token.Token
 	Loop                []Vertex
+	LoopSeparatorTkns   []*token.Token
 	CloseParenthesisTkn *token.Token
 	ColonTkn            *token.Token
 	Stmt                Vertex
@@ -792,10 +795,11 @@ func (n *StmtTraitUse) Accept(v NodeVisitor) {
 // StmtTraitUseAlias node
 type StmtTraitUseAlias struct {
 	Node
-	Ref      Vertex
+	Ref          Vertex
-	AsTkn    *token.Token
+	AsTkn        *token.Token
-	Modifier Vertex
+	Modifier     Vertex
-	Alias    Vertex
+	Alias        Vertex
+	SemiColonTkn *token.Token
 }
 
 func (n *StmtTraitUseAlias) Accept(v NodeVisitor) {
@@ -809,6 +813,7 @@ type StmtTraitUsePrecedence struct {
 	InsteadofTkn  *token.Token
 	Insteadof     []Vertex
 	SeparatorTkns []*token.Token
+	SemiColonTkn  *token.Token
 }
 
 func (n *StmtTraitUsePrecedence) Accept(v NodeVisitor) {