[refactoring] update ast structure of "Static", "Global" and "StaticVar" nodes

internal/php5/php5.y

@@ -239,7 +239,7 @@ import (
 %type <node> optional_class_type parameter class_entry_type class_statement class_constant_declaration
 %type <node> trait_use_statement function_call_parameter trait_adaptation_statement trait_precedence trait_alias
 %type <node> trait_method_reference_fully_qualified trait_method_reference trait_modifiers member_modifier method
-%type <node> static_scalar_value static_operation
+%type <node> static_scalar_value static_operation static_var_list global_var_list
 %type <node> ctor_arguments function_call_parameter_list
 %type <node> trait_adaptations
 %type <node> switch_case_list
@@ -253,7 +253,7 @@ import (
 %type <ClosureUse> lexical_vars
 
 %type <list> top_statement_list namespace_name use_declarations use_function_declarations use_const_declarations
-%type <list> inner_statement_list global_var_list static_var_list encaps_list isset_variables non_empty_array_pair_list
+%type <list> inner_statement_list encaps_list isset_variables non_empty_array_pair_list
 %type <list> array_pair_list assignment_list lexical_var_list elseif_list new_elseif_list non_empty_for_expr
 %type <list> for_expr case_list echo_expr_list unset_variables declare_list catch_statement additional_catches
 %type <list> non_empty_additional_catches parameter_list non_empty_parameter_list class_statement_list
@@ -1036,27 +1036,21 @@ unticked_statement:
             }
     |   T_GLOBAL global_var_list ';'
             {
-                $$ = &ast.StmtGlobal{ast.Node{}, $2}
+                $2.(*ast.StmtGlobal).GlobalTkn = $1
+                $2.(*ast.StmtGlobal).SemiColonTkn = $3
+                $2.(*ast.StmtGlobal).SeparatorTkns = append($2.(*ast.StmtGlobal).SeparatorTkns, nil)
+                $2.(*ast.StmtGlobal).Node.Position = position.NewTokensPosition($1, $3)
 
-                // save position
+                $$ = $2
-                $$.GetNode().Position = position.NewTokensPosition($1, $3)
-
-                // save comments
-                yylex.(*Parser).setFreeFloating($$, token.Start, $1.SkippedTokens)
-                yylex.(*Parser).setFreeFloating($$, token.VarList, $3.SkippedTokens)
-                yylex.(*Parser).setToken($$, token.SemiColon, $3.SkippedTokens)
             }
     |   T_STATIC static_var_list ';'
             {
-                $$ = &ast.StmtStatic{ast.Node{}, $2}
+                $2.(*ast.StmtStatic).StaticTkn = $1
+                $2.(*ast.StmtStatic).SemiColonTkn = $3
+                $2.(*ast.StmtStatic).SeparatorTkns = append($2.(*ast.StmtStatic).SeparatorTkns, nil)
+                $2.(*ast.StmtStatic).Node.Position = position.NewTokensPosition($1, $3)
 
-                // save position
+                $$ = $2
-                $$.GetNode().Position = position.NewTokensPosition($1, $3)
-
-                // save comments
-                yylex.(*Parser).setFreeFloating($$, token.Start, $1.SkippedTokens)
-                yylex.(*Parser).setFreeFloating($$, token.VarList, $3.SkippedTokens)
-                yylex.(*Parser).setToken($$, token.SemiColon, $3.SkippedTokens)
             }
     |   T_ECHO echo_expr_list ';'
             {
@@ -2218,14 +2212,16 @@ function_call_parameter:
 global_var_list:
         global_var_list ',' global_var
             {
-                $$ = append($1, $3)
+                $1.(*ast.StmtGlobal).Vars = append($1.(*ast.StmtGlobal).Vars, $3)
+                $1.(*ast.StmtGlobal).SeparatorTkns = append($1.(*ast.StmtGlobal).SeparatorTkns, $2)
 
-                // save comments
+                $$ = $1
-                yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.SkippedTokens)
             }
     |   global_var
             {
-                $$ = []ast.Vertex{$1}
+                $$ = &ast.StmtGlobal{
+                    Vars: []ast.Vertex{$1},
+                }
             }
 ;
 
@@ -2273,65 +2269,95 @@ static_var_list:
             {
                 identifier := &ast.Identifier{ast.Node{}, $3.Value}
                 variable := &ast.ExprVariable{ast.Node{}, identifier}
-                staticVar := &ast.StmtStaticVar{ast.Node{}, variable, nil}
+
-                $$ = append($1, staticVar)
+                $1.(*ast.StmtStatic).Vars = append($1.(*ast.StmtStatic).Vars, &ast.StmtStaticVar{
+                    Node: ast.Node{
+                        Position:  position.NewTokenPosition($3),
+                    },
+                    Var: variable,
+                })
+                $1.(*ast.StmtStatic).SeparatorTkns = append($1.(*ast.StmtStatic).SeparatorTkns, $2)
+
+                $$ = $1
 
                 // save position
                 identifier.GetNode().Position = position.NewTokenPosition($3)
                 variable.GetNode().Position = position.NewTokenPosition($3)
-                staticVar.GetNode().Position = position.NewTokenPosition($3)
 
                 // save comments
-                yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.SkippedTokens)
+                yylex.(*Parser).setFreeFloating(variable, token.Start, $3.SkippedTokens)
-                yylex.(*Parser).setFreeFloating(staticVar, token.Start, $3.SkippedTokens)
             }
     |   static_var_list ',' T_VARIABLE '=' static_scalar
             {
                 identifier := &ast.Identifier{ast.Node{}, $3.Value}
                 variable := &ast.ExprVariable{ast.Node{}, identifier}
-                staticVar := &ast.StmtStaticVar{ast.Node{}, variable, $5}
+
-                $$ = append($1, staticVar)
+                $1.(*ast.StmtStatic).Vars = append($1.(*ast.StmtStatic).Vars, &ast.StmtStaticVar{
+                    Node: ast.Node{
+                        Position:  position.NewTokenNodePosition($3, $5),
+                    },
+                    Var:      variable,
+                    EqualTkn: $4,
+                    Expr:     $5,
+                })
+                $1.(*ast.StmtStatic).SeparatorTkns = append($1.(*ast.StmtStatic).SeparatorTkns, $2)
+
+                $$ = $1
 
                 // save position
                 identifier.GetNode().Position = position.NewTokenPosition($3)
                 variable.GetNode().Position = position.NewTokenPosition($3)
-                staticVar.GetNode().Position = position.NewTokenNodePosition($3, $5)
 
                 // save comments
-                yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.SkippedTokens)
+                yylex.(*Parser).setFreeFloating(variable, token.Start, $3.SkippedTokens)
-                yylex.(*Parser).setFreeFloating(staticVar, token.Start, $3.SkippedTokens)
-                yylex.(*Parser).setFreeFloating(staticVar, token.Var, $4.SkippedTokens)
             }
     |   T_VARIABLE
             {
                 identifier := &ast.Identifier{ast.Node{}, $1.Value}
                 variable := &ast.ExprVariable{ast.Node{}, identifier}
-                staticVar := &ast.StmtStaticVar{ast.Node{}, variable, nil}
+
-                $$ = []ast.Vertex{staticVar}
+                $$ = &ast.StmtStatic{
+                    Vars: []ast.Vertex{
+                        &ast.StmtStaticVar{
+                            Node: ast.Node{
+                                Position:  position.NewTokenPosition($1),
+                            },
+                            Var: variable,
+                        },
+                    },
+                }
 
                 // save position
                 identifier.GetNode().Position = position.NewTokenPosition($1)
                 variable.GetNode().Position = position.NewTokenPosition($1)
-                staticVar.GetNode().Position = position.NewTokenPosition($1)
 
                 // save comments
-                yylex.(*Parser).setFreeFloating(staticVar, token.Start, $1.SkippedTokens)
+                yylex.(*Parser).setFreeFloating(variable, token.Start, $1.SkippedTokens)
             }
     |   T_VARIABLE '=' static_scalar
             {
                 identifier := &ast.Identifier{ast.Node{}, $1.Value}
                 variable := &ast.ExprVariable{ast.Node{}, identifier}
-                staticVar := &ast.StmtStaticVar{ast.Node{}, variable, $3}
+
-                $$ = []ast.Vertex{staticVar}
+                $$ = &ast.StmtStatic{
+                    Vars: []ast.Vertex{
+                        &ast.StmtStaticVar{
+                            Node: ast.Node{
+                                Position:  position.NewTokenNodePosition($1, $3),
+                            },
+                            Var:      variable,
+                            EqualTkn: $2,
+                            Expr:     $3,
+                        },
+                    },
+                }
 
                 // save position
                 identifier.GetNode().Position = position.NewTokenPosition($1)
                 variable.GetNode().Position = position.NewTokenPosition($1)
-                staticVar.GetNode().Position = position.NewTokenNodePosition($1, $3)
 
                 // save comments
-                yylex.(*Parser).setFreeFloating(staticVar, token.Start, $1.SkippedTokens)
+                yylex.(*Parser).setFreeFloating(variable, token.Start, $1.SkippedTokens)
-                yylex.(*Parser).setFreeFloating(staticVar, token.Var, $2.SkippedTokens)
             }
 ;
 

internal/php7/php7.y

@@ -241,8 +241,8 @@ import (
 %type <node> const_decl inner_statement
 %type <node> expr optional_expr
 %type <node> declare_statement finally_statement unset_variable variable
-%type <node> parameter optional_type argument expr_without_variable global_var
+%type <node> parameter optional_type argument expr_without_variable global_var_list global_var
-%type <node> static_var class_statement trait_adaptation trait_precedence trait_alias
+%type <node> static_var_list static_var class_statement trait_adaptation trait_precedence trait_alias
 %type <node> absolute_trait_method_reference trait_method_reference property echo_expr
 %type <node> new_expr anonymous_class class_name class_name_reference simple_variable
 %type <node> internal_functions_in_yacc
@@ -275,8 +275,8 @@ import (
 
 
 %type <list> encaps_list backticks_expr namespace_name catch_name_list catch_list class_const_list
-%type <list> const_list echo_expr_list for_exprs non_empty_for_exprs global_var_list
+%type <list> const_list echo_expr_list for_exprs non_empty_for_exprs
-%type <list> unprefixed_use_declarations inline_use_declarations property_list static_var_list
+%type <list> unprefixed_use_declarations inline_use_declarations property_list
 %type <list> case_list trait_adaptation_list unset_variables
 %type <list> use_declarations lexical_var_list isset_variables non_empty_array_pair_list
 %type <list> array_pair_list non_empty_argument_list top_statement_list
@@ -952,27 +952,21 @@ statement:
             }
     |   T_GLOBAL global_var_list ';'
             {
-                $$ = &ast.StmtGlobal{ast.Node{}, $2}
+                $2.(*ast.StmtGlobal).GlobalTkn = $1
+                $2.(*ast.StmtGlobal).SemiColonTkn = $3
+                $2.(*ast.StmtGlobal).SeparatorTkns = append($2.(*ast.StmtGlobal).SeparatorTkns, nil)
+                $2.(*ast.StmtGlobal).Node.Position = position.NewTokensPosition($1, $3)
 
-                // save position
+                $$ = $2
-                $$.GetNode().Position = position.NewTokensPosition($1, $3)
-
-                // save comments
-                yylex.(*Parser).setFreeFloating($$, token.Start, $1.SkippedTokens)
-                yylex.(*Parser).setFreeFloating($$, token.VarList, $3.SkippedTokens)
-                yylex.(*Parser).setToken($$, token.SemiColon, $3.SkippedTokens)
             }
     |   T_STATIC static_var_list ';'
             {
-                $$ = &ast.StmtStatic{ast.Node{}, $2}
+                $2.(*ast.StmtStatic).StaticTkn = $1
+                $2.(*ast.StmtStatic).SemiColonTkn = $3
+                $2.(*ast.StmtStatic).SeparatorTkns = append($2.(*ast.StmtStatic).SeparatorTkns, nil)
+                $2.(*ast.StmtStatic).Node.Position = position.NewTokensPosition($1, $3)
 
-                // save position
+                $$ = $2
-                $$.GetNode().Position = position.NewTokensPosition($1, $3)
-
-                // save comments
-                yylex.(*Parser).setFreeFloating($$, token.Start, $1.SkippedTokens)
-                yylex.(*Parser).setFreeFloating($$, token.VarList, $3.SkippedTokens)
-                yylex.(*Parser).setToken($$, token.SemiColon, $3.SkippedTokens)
             }
     |   T_ECHO echo_expr_list ';'
             {
@@ -2063,14 +2057,16 @@ argument:
 global_var_list:
         global_var_list ',' global_var
             {
-                $$ = append($1, $3)
+                $1.(*ast.StmtGlobal).Vars = append($1.(*ast.StmtGlobal).Vars, $3)
+                $1.(*ast.StmtGlobal).SeparatorTkns = append($1.(*ast.StmtGlobal).SeparatorTkns, $2)
 
-                // save comments
+                $$ = $1
-                yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.SkippedTokens)
             }
     |   global_var
             {
-                $$ = []ast.Vertex{$1}
+                $$ = &ast.StmtGlobal{
+                    Vars: []ast.Vertex{$1},
+                }
             }
 ;
 
@@ -2084,14 +2080,16 @@ global_var:
 static_var_list:
         static_var_list ',' static_var
             {
-                $$ = append($1, $3)
+                $1.(*ast.StmtStatic).Vars = append($1.(*ast.StmtStatic).Vars, $3)
+                $1.(*ast.StmtStatic).SeparatorTkns = append($1.(*ast.StmtStatic).SeparatorTkns, $2)
 
-                // save comments
+                $$ = $1
-                yylex.(*Parser).setFreeFloating(lastNode($1), token.End, $2.SkippedTokens)
             }
     |   static_var
             {
-                $$ = []ast.Vertex{$1}
+                $$ = &ast.StmtStatic{
+                    Vars: []ast.Vertex{$1},
+                }
             }
 ;
 
@@ -2100,30 +2098,40 @@ static_var:
             {
                 identifier := &ast.Identifier{ast.Node{}, $1.Value}
                 variable := &ast.ExprVariable{ast.Node{}, identifier}
-                $$ = &ast.StmtStaticVar{ast.Node{}, variable, nil}
+
+                $$ = &ast.StmtStaticVar{
+                    Node: ast.Node{
+                        Position: position.NewTokenPosition($1),
+                    },
+                    Var: variable,
+                }
 
                 // save position
                 identifier.GetNode().Position = position.NewTokenPosition($1)
                 variable.GetNode().Position = position.NewTokenPosition($1)
-                $$.GetNode().Position = position.NewTokenPosition($1)
 
                 // save comments
-                yylex.(*Parser).setFreeFloating($$, token.Start, $1.SkippedTokens)
+                yylex.(*Parser).setFreeFloating(variable, token.Start, $1.SkippedTokens)
             }
     |   T_VARIABLE '=' expr
             {
                 identifier := &ast.Identifier{ast.Node{}, $1.Value}
                 variable := &ast.ExprVariable{ast.Node{}, identifier}
-                $$ = &ast.StmtStaticVar{ast.Node{}, variable, $3}
+                $$ = &ast.StmtStaticVar{
+                    Node: ast.Node{
+                        Position: position.NewTokenNodePosition($1, $3),
+                    },
+                    Var:      variable,
+                    EqualTkn: $2,
+                    Expr:     $3,
+                }
 
                 // save position
                 identifier.GetNode().Position = position.NewTokenPosition($1)
                 variable.GetNode().Position = position.NewTokenPosition($1)
-                $$.GetNode().Position = position.NewTokenNodePosition($1, $3)
 
                 // save comments
-                yylex.(*Parser).setFreeFloating($$, token.Start, $1.SkippedTokens)
+                yylex.(*Parser).setFreeFloating(variable, token.Start, $1.SkippedTokens)
-                yylex.(*Parser).setFreeFloating($$, token.Var, $2.SkippedTokens)
             }
 ;
 

pkg/ast/node.go

@@ -476,7 +476,10 @@ func (n *StmtFunction) Accept(v NodeVisitor) {
 // StmtGlobal node
 type StmtGlobal struct {
 	Node
-	Vars []Vertex
+	GlobalTkn     *token.Token
+	Vars          []Vertex
+	SeparatorTkns []*token.Token
+	SemiColonTkn  *token.Token
 }
 
 func (n *StmtGlobal) Accept(v NodeVisitor) {
@@ -630,7 +633,10 @@ func (n *StmtReturn) Accept(v NodeVisitor) {
 // StmtStatic node
 type StmtStatic struct {
 	Node
-	Vars []Vertex
+	StaticTkn     *token.Token
+	Vars          []Vertex
+	SeparatorTkns []*token.Token
+	SemiColonTkn  *token.Token
 }
 
 func (n *StmtStatic) Accept(v NodeVisitor) {
@@ -640,8 +646,9 @@ func (n *StmtStatic) Accept(v NodeVisitor) {
 // StmtStaticVar node
 type StmtStaticVar struct {
 	Node
-	Var  Vertex
+	Var      Vertex
-	Expr Vertex
+	EqualTkn *token.Token
+	Expr     Vertex
 }
 
 func (n *StmtStaticVar) Accept(v NodeVisitor) {

pkg/ast/visitor/filter_tokens.go

@@ -178,3 +178,19 @@ func (v *FilterTokens) StmtReturn(n *ast.StmtReturn) {
 	n.ReturnTkn = nil
 	n.SemiColonTkn = nil
 }
+
+func (v *FilterTokens) StmtGlobal(n *ast.StmtGlobal) {
+	n.GlobalTkn = nil
+	n.SeparatorTkns = nil
+	n.SemiColonTkn = nil
+}
+
+func (v *FilterTokens) StmtStatic(n *ast.StmtStatic) {
+	n.StaticTkn = nil
+	n.SeparatorTkns = nil
+	n.SemiColonTkn = nil
+}
+
+func (v *FilterTokens) StmtStaticVar(n *ast.StmtStaticVar) {
+	n.EqualTkn = nil
+}

pkg/printer/printer.go

@@ -65,6 +65,26 @@ func (p *Printer) joinPrintRefactored(glue string, nn []ast.Vertex) {
 	}
 }
 
+func (p *Printer) printSeparatedList(nodeList []ast.Vertex, separatorList []*token.Token, def string) {
+	var separators []*token.Token
+
+	if cap(separatorList) >= len(nodeList) {
+		separators = separatorList[:len(nodeList)]
+	} else {
+		separators = make([]*token.Token, len(nodeList))
+		copy(separators, separatorList)
+	}
+
+	for k, n := range nodeList {
+		p.Print(n)
+		if k < len(nodeList)-1 {
+			p.printToken(separators[k], def)
+		} else {
+			p.printToken(separators[k], "")
+		}
+	}
+}
+
 func (p *Printer) printNodes(nn []ast.Vertex) {
 	for _, n := range nn {
 		p.Print(n)
@@ -2476,20 +2496,11 @@ func (p *Printer) printStmtFunction(n ast.Vertex) {
 	p.printFreeFloating(nn, token.End)
 }
 
-func (p *Printer) printStmtGlobal(n ast.Vertex) {
+func (p *Printer) printStmtGlobal(n *ast.StmtGlobal) {
-	nn := n.(*ast.StmtGlobal)
+	p.printToken(n.GlobalTkn, "global")
-	p.printFreeFloating(nn, token.Start)
+	p.bufStart = " "
-
+	p.printSeparatedList(n.Vars, n.SeparatorTkns, ",")
-	io.WriteString(p.w, "global")
+	p.printToken(n.SemiColonTkn, ";")
-	p.joinPrint(",", nn.Vars)
-	p.printFreeFloating(nn, token.VarList)
-
-	p.printFreeFloating(nn, token.SemiColon)
-	if nn.GetNode().Tokens.IsEmpty() {
-		io.WriteString(p.w, ";")
-	}
-
-	p.printFreeFloating(nn, token.End)
 }
 
 func (p *Printer) printStmtGoto(n ast.Vertex) {
@@ -2701,35 +2712,20 @@ func (p *Printer) printStmtReturn(n *ast.StmtReturn) {
 	p.printToken(n.SemiColonTkn, ";")
 }
 
-func (p *Printer) printStmtStaticVar(n ast.Vertex) {
+func (p *Printer) printStmtStaticVar(n *ast.StmtStaticVar) {
-	nn := n.(*ast.StmtStaticVar)
+	p.Print(n.Var)
-	p.printFreeFloating(nn, token.Start)
 
-	p.Print(nn.Var)
+	if n.Expr != nil {
-
+		p.printToken(n.EqualTkn, "=")
-	if nn.Expr != nil {
+		p.Print(n.Expr)
-		p.printFreeFloating(nn, token.Var)
-		io.WriteString(p.w, "=")
-		p.Print(nn.Expr)
 	}
-
-	p.printFreeFloating(nn, token.End)
 }
 
-func (p *Printer) printStmtStatic(n ast.Vertex) {
+func (p *Printer) printStmtStatic(n *ast.StmtStatic) {
-	nn := n.(*ast.StmtStatic)
+	p.printToken(n.StaticTkn, "static")
-	p.printFreeFloating(nn, token.Start)
+	p.bufStart = " "
-	io.WriteString(p.w, "static")
+	p.printSeparatedList(n.Vars, n.SeparatorTkns, ",")
-
+	p.printToken(n.SemiColonTkn, ";")
-	p.joinPrint(",", nn.Vars)
-	p.printFreeFloating(nn, token.VarList)
-
-	p.printFreeFloating(nn, token.SemiColon)
-	if n.GetNode().Tokens.IsEmpty() {
-		io.WriteString(p.w, ";")
-	}
-
-	p.printFreeFloating(nn, token.End)
 }
 
 func (p *Printer) printStmtStmtList(n *ast.StmtStmtList) {

pkg/printer/printer_parsed_php5_test.go

@@ -1036,7 +1036,8 @@ func TestParseAndPrintPhp5Function(t *testing.T) {
 }
 
 func TestParseAndPrintPhp5Global(t *testing.T) {
-	src := `<?php
+	// TODO: remove ; after <?php
+	src := `<?php ;
 	global $a , $b ;`
 
 	actual := printPhp5(parsePhp5(src))
@@ -1208,7 +1209,8 @@ func TestParseAndPrintPhp5Return(t *testing.T) {
 }
 
 func TestParseAndPrintPhp5StaticVar(t *testing.T) {
-	src := `<?php
+	// TODO: remove ; after <?php
+	src := `<?php ;
 	static $a , $b = ' ' ;
 	`
 
@@ -1363,7 +1365,8 @@ func TestParseAndPrintPhp5While(t *testing.T) {
 // other
 
 func TestParseAndPrintPhp5Parentheses(t *testing.T) {
-	src := `<?php
+	// TODO: remove ; after <?php
+	src := `<?php ;
 	global $ { $b } ;
 	$b = (($a));
 	$b = ( ($a) );

pkg/printer/printer_parsed_php7_test.go

@@ -1154,7 +1154,8 @@ func TestParseAndPrintFunction(t *testing.T) {
 }
 
 func TestParseAndPrintGlobal(t *testing.T) {
-	src := `<?php
+	// TODO: remove ; after <?php
+	src := `<?php ;
 	global $a , $b ;`
 
 	actual := print(parse(src))
@@ -1341,7 +1342,8 @@ func TestParseAndPrintReturn(t *testing.T) {
 }
 
 func TestParseAndPrintStaticVar(t *testing.T) {
-	src := `<?php
+	// TODO: remove ; after <?php
+	src := `<?php ;
 	static $a , $b = foo ( ) ;
 	`
 

pkg/printer/printer_test.go

@@ -3658,7 +3658,7 @@ func TestPrinterPrintStmtGlobal(t *testing.T) {
 		},
 	})
 
-	expected := `global$a,$b;`
+	expected := `global $a,$b;`
 	actual := o.String()
 
 	if expected != actual {
@@ -4029,7 +4029,7 @@ func TestPrinterPrintStatic(t *testing.T) {
 		},
 	})
 
-	expected := `static$a,$b;`
+	expected := `static $a,$b;`
 	actual := o.String()
 
 	if expected != actual {