refactor token structure

pkg/ast/ast.go

@@ -1,16 +1,8 @@
 package ast
 
-import (
-	"github.com/z7zmey/php-parser/freefloating"
-	"github.com/z7zmey/php-parser/position"
-)
-
 type Vertex interface {
 	Accept(v NodeVisitor)
-
+	GetNode() *Node
-	SetPosition(p *position.Position)
-	GetPosition() *position.Position
-	GetFreeFloating() *freefloating.Collection
 }
 
 type Traverser interface {

pkg/ast/ast_test.go

@@ -1,8 +1,6 @@
 package ast_test
 
 import (
-	"bytes"
-	"encoding/json"
 	"fmt"
 	"github.com/z7zmey/php-parser/pkg/ast"
 	"github.com/z7zmey/php-parser/pkg/ast/traverser"
@@ -11,92 +9,6 @@ import (
 	"strings"
 )
 
-func ExampleJSON() {
-	stxTree := &ast.Root{
-		Stmts: []ast.Vertex{
-			&ast.Nullable{
-				Expr: &ast.Parameter{
-					Type:         nil,
-					Var:          nil,
-					DefaultValue: nil,
-				},
-			},
-			&ast.Identifier{},
-			&ast.ArgumentList{
-				Arguments: []ast.Vertex{
-					&ast.Argument{},
-					&ast.Argument{
-						Expr: &ast.ScalarDnumber{},
-					},
-				},
-			},
-		},
-	}
-
-	jsonStxTree, err := json.Marshal(stxTree)
-	if err != nil {
-		panic(err)
-	}
-
-	buf := bytes.NewBuffer(nil)
-	err = json.Indent(buf, jsonStxTree, "", "  ")
-	if err != nil {
-		panic(err)
-	}
-
-	fmt.Fprint(os.Stdout, buf.String())
-
-	// output:
-	// {
-	//   "FreeFloating": null,
-	//   "Position": null,
-	//   "Stmts": [
-	//     {
-	//       "FreeFloating": null,
-	//       "Position": null,
-	//       "Expr": {
-	//         "FreeFloating": null,
-	//         "Position": null,
-	//         "ByRef": false,
-	//         "Variadic": false,
-	//         "Type": null,
-	//         "Var": null,
-	//         "DefaultValue": null
-	//       }
-	//     },
-	//     {
-	//       "FreeFloating": null,
-	//       "Position": null,
-	//       "Value": ""
-	//     },
-	//     {
-	//       "FreeFloating": null,
-	//       "Position": null,
-	//       "Arguments": [
-	//         {
-	//           "FreeFloating": null,
-	//           "Position": null,
-	//           "Variadic": false,
-	//           "IsReference": false,
-	//           "Expr": null
-	//         },
-	//         {
-	//           "FreeFloating": null,
-	//           "Position": null,
-	//           "Variadic": false,
-	//           "IsReference": false,
-	//           "Expr": {
-	//             "FreeFloating": null,
-	//             "Position": null,
-	//             "Value": ""
-	//           }
-	//         }
-	//       ]
-	//     }
-	//   ]
-	// }
-}
-
 func ExampleStxTree() {
 	stxTree := &ast.Root{
 		Stmts: []ast.Vertex{

pkg/ast/node.go

@@ -1,27 +1,18 @@
 package ast
 
 import (
-	"github.com/z7zmey/php-parser/freefloating"
+	"github.com/z7zmey/php-parser/pkg/token"
 	"github.com/z7zmey/php-parser/position"
 )
 
 type Node struct {
-	FreeFloating freefloating.Collection
+	StartTokens  []token.Token
+	EndTokens    []token.Token
 	Position     *position.Position
 }
 
-// SetPosition sets node position
+func (n *Node) GetNode() *Node {
-func (n *Node) SetPosition(p *position.Position) {
+	return n
-	n.Position = p
-}
-
-// GetPosition returns node positions
-func (n *Node) GetPosition() *position.Position {
-	return n.Position
-}
-
-func (n *Node) GetFreeFloating() *freefloating.Collection {
-	return &n.FreeFloating
 }
 
 // Root node

pkg/token/token.go

@@ -0,0 +1,149 @@
+package token
+
+type TokenID int
+
+const (
+	T_INCLUDE TokenID = iota + 57346
+	T_INCLUDE_ONCE
+	T_EXIT
+	T_IF
+	T_LNUMBER
+	T_DNUMBER
+	T_STRING
+	T_STRING_VARNAME
+	T_VARIABLE
+	T_NUM_STRING
+	T_INLINE_HTML
+	T_CHARACTER
+	T_BAD_CHARACTER
+	T_ENCAPSED_AND_WHITESPACE
+	T_CONSTANT_ENCAPSED_STRING
+	T_ECHO
+	T_DO
+	T_WHILE
+	T_ENDWHILE
+	T_FOR
+	T_ENDFOR
+	T_FOREACH
+	T_ENDFOREACH
+	T_DECLARE
+	T_ENDDECLARE
+	T_AS
+	T_SWITCH
+	T_ENDSWITCH
+	T_CASE
+	T_DEFAULT
+	T_BREAK
+	T_CONTINUE
+	T_GOTO
+	T_FUNCTION
+	T_FN
+	T_CONST
+	T_RETURN
+	T_TRY
+	T_CATCH
+	T_FINALLY
+	T_THROW
+	T_USE
+	T_INSTEADOF
+	T_GLOBAL
+	T_VAR
+	T_UNSET
+	T_ISSET
+	T_EMPTY
+	T_HALT_COMPILER
+	T_CLASS
+	T_TRAIT
+	T_INTERFACE
+	T_EXTENDS
+	T_IMPLEMENTS
+	T_OBJECT_OPERATOR
+	T_DOUBLE_ARROW
+	T_LIST
+	T_ARRAY
+	T_CALLABLE
+	T_CLASS_C
+	T_TRAIT_C
+	T_METHOD_C
+	T_FUNC_C
+	T_LINE
+	T_FILE
+	T_COMMENT
+	T_DOC_COMMENT
+	T_OPEN_TAG
+	T_OPEN_TAG_WITH_ECHO
+	T_CLOSE_TAG
+	T_WHITESPACE
+	T_START_HEREDOC
+	T_END_HEREDOC
+	T_DOLLAR_OPEN_CURLY_BRACES
+	T_CURLY_OPEN
+	T_PAAMAYIM_NEKUDOTAYIM
+	T_NAMESPACE
+	T_NS_C
+	T_DIR
+	T_NS_SEPARATOR
+	T_ELLIPSIS
+	T_EVAL
+	T_REQUIRE
+	T_REQUIRE_ONCE
+	T_LOGICAL_OR
+	T_LOGICAL_XOR
+	T_LOGICAL_AND
+	T_INSTANCEOF
+	T_NEW
+	T_CLONE
+	T_ELSEIF
+	T_ELSE
+	T_ENDIF
+	T_PRINT
+	T_YIELD
+	T_STATIC
+	T_ABSTRACT
+	T_FINAL
+	T_PRIVATE
+	T_PROTECTED
+	T_PUBLIC
+	T_INC
+	T_DEC
+	T_YIELD_FROM
+	T_INT_CAST
+	T_DOUBLE_CAST
+	T_STRING_CAST
+	T_ARRAY_CAST
+	T_OBJECT_CAST
+	T_BOOL_CAST
+	T_UNSET_CAST
+	T_COALESCE
+	T_SPACESHIP
+	T_NOELSE
+	T_PLUS_EQUAL
+	T_MINUS_EQUAL
+	T_MUL_EQUAL
+	T_POW_EQUAL
+	T_DIV_EQUAL
+	T_CONCAT_EQUAL
+	T_MOD_EQUAL
+	T_AND_EQUAL
+	T_OR_EQUAL
+	T_XOR_EQUAL
+	T_SL_EQUAL
+	T_SR_EQUAL
+	T_COALESCE_EQUAL
+	T_BOOLEAN_OR
+	T_BOOLEAN_AND
+	T_POW
+	T_SL
+	T_SR
+	T_IS_IDENTICAL
+	T_IS_NOT_IDENTICAL
+	T_IS_EQUAL
+	T_IS_NOT_EQUAL
+	T_IS_SMALLER_OR_EQUAL
+	T_IS_GREATER_OR_EQUAL
+)
+
+type Token struct {
+	ID    TokenID
+	Value []byte
+}