[refactoring] update Token structure

2020-12-08 01:23:48 +02:00
parent 20a42da7c9
commit 8064d940f0
6 changed files with 617 additions and 648 deletions
--- a/pkg/token/token.go
+++ b/pkg/token/token.go
@@ -147,11 +147,10 @@ const (
 )

 type Token struct {
-	ID            ID
-	Value         []byte
-	Position      *position.Position
-	SkippedTokens []*Token
-	Skipped       []byte
+	ID           ID
+	Value        []byte
+	Position     *position.Position
+	FreeFloating []*Token
 }

 func (t *Token) GetPosition() *position.Position {