From 8064d940f0ac0f998a19ce6afadb333599235ca6 Mon Sep 17 00:00:00 2001
From: Vadym Slizov <z7zmey@gmail.com>
Date: Tue, 8 Dec 2020 01:23:48 +0200
Subject: [PATCH] [refactoring] update Token structure

---
 internal/scanner/lexer.go        |  17 ++-----
 internal/scanner/scanner.go      | Bin 412497 -> 412478 bytes
 internal/scanner/scanner.rl      |  33 +++++--------
 internal/scanner/scanner_test.go |  78 +++++++++++++++----------------
 pkg/ast/visitor/dump.go          |   3 --
 pkg/token/token.go               |   9 ++--
 6 files changed, 59 insertions(+), 81 deletions(-)

diff --git a/internal/scanner/lexer.go b/internal/scanner/lexer.go
index 7986acb..8c0381f 100644
--- a/internal/scanner/lexer.go
+++ b/internal/scanner/lexer.go
@@ -15,7 +15,6 @@ type Lexer struct {
 	phpVersion     string
 	errHandlerFunc func(*errors.Error)
 
-	sts, ste    int
 	p, pe, cs   int
 	ts, te, act int
 	stack       []int
@@ -57,26 +56,18 @@ func (lex *Lexer) setTokenPosition(token *token.Token) {
 	token.Position = pos
 }
 
-func (lex *Lexer) addSkippedToken(t *token.Token, id token.ID, ps, pe int) {
-	if lex.sts == -1 {
-		lex.sts = lex.ts
-	}
-
-	lex.ste = lex.te
-
-	// TODO remove after parser refactoring
-
+func (lex *Lexer) addFreeFloatingToken(t *token.Token, id token.ID, ps, pe int) {
 	skippedTkn := lex.tokenPool.Get()
 	skippedTkn.ID = id
 	skippedTkn.Value = lex.data[ps:pe]
 
 	lex.setTokenPosition(skippedTkn)
 
-	if t.SkippedTokens == nil {
-		t.SkippedTokens = make([]*token.Token, 0, 2)
+	if t.FreeFloating == nil {
+		t.FreeFloating = make([]*token.Token, 0, 2)
 	}
 
-	t.SkippedTokens = append(t.SkippedTokens, skippedTkn)
+	t.FreeFloating = append(t.FreeFloating, skippedTkn)
 }
 
 func (lex *Lexer) isNotStringVar() bool {
diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go
index 9f779e69b85f30e18eea65d069f5eb189c65f1f8..bd7768b5025b17be7f396b99cf02f27296015bfd 100644
GIT binary patch
delta 7002
zcma)BcUYCj(tqZhT`5uxCbmS3h9)Te0Ahkl^wVRk*!?W`#;CE7C>8{SLs3BybPZ#}
z?$y|$L4D2jTCXLFC@9f@s9=d=Z;v-7M(*tU9u9&e&;5t9@9fO(%x`CQXNIDjrU!GH
z?!3&zf7no6upzHM|JzKRf2b%g{YXrj?Jpc=hl$L@#=<q<iM}V&*oS@tL};QH1d58e
zo;v?vbY8~W*>#;VW=z+K(W7I=PaZ4U-w6}16OV%*w(@}H;#yKU1Z%^~re3;$5L{$}
z-Qrqutj;$Col_w|WTr$}=$M*JlxAs0ajn$NarEb(*IGL~y`Q+976!q>?_6tKJ(Jrk
zp5L<e_TlMXqR)H}abiw;VuRa3qV3#0s=i*9?+lSYuLJnkqkMS!LfnPCjquHP7mYLK
zli=D6P0$z1r!w{@`&DAmU>+z|X9b8g3p!h@S(+I@E1b#v7y|K07W2|<uUu%-1&3hK
za++I{6=UH(olV?t@Rcnx>3l;qKCyTjaodOmI)96#K~yer$$NK+0_dE~-Ehkk=8y+D
z%XO9&`!17c7B364P>wC1MTXPKc!}{VBEVOkc3qg!X%-I`?ax}}wOIM69c^3m!iBux
zB=`|`D`9hHCj`-`2l@`+Vfd9Tc#AI!+TwE?AhvrUqN2bgUY#?FQKbz;-q{5F(H4v%
z?wkqy>DYRRZiOl0n*yU~T$ChyiLJaSMVu})iXG=oU@-4x5OEihM3)ky`0CjJv7*>p
zm2D|*7q6^++!;T0t4(Nsr=4hUv5Tr<tGo#pUjliO5l30WU}04fuG-K;`~0QEhm0#P
z?Ma%>)&)t=KSgqywFcpO*g*tc?k;+kc?<nDSF$@zynn?TO-|r~uCWAB!-5gwKUcg-
z>xwFgkU>OW6{7G(V>#UsuUul*SXIny#F%RnRkuRlSneY}EFZ6$^M6teYf)A{8jlX(
z-t`~DcNNkpc4dLrUoa(J4<r>`#F`r}*e;4Ei0>+F#ONDsrLKL|V-kI;yhWFr-lFqE
z7qRosCnEiiA)@&$FKnuZ&&1+eNWsx+j$9!U|8Bz@i6y^G5TnmFCPEY8c&BMykUNRY
z+if*(_53ToNn-7<9wPgd=i);gOS*73RguU_Z(;MuDE3z-i#At{V&2_UG37U-7<4aL
z*xfUVHusY;(2CqiYWnhiJU(qiE_C=oGJYfnnGfT{&-Z$Ys7EPc)nlXh@Nt}odC*f-
zJ{Dr-9T$=P<PaWwhZvVXO+!a(=u2K?61|@G6dj)@OH4fSZa=>ZV#x0vc`sjV2J!Tn
zie28RUpE7tW~({c1#sZEl@u&%?Rg)~VRWxLn9rgX+4&D2g0-i}_#+jwI5|hsARZw`
zo#JBb>uI9fEu(O+$}(dm{YyQlCVTARZy?HEt488cfCb{xBO@-;L9+O&(uiN^Aq9Kt
z;9Us<e#{_IOuG55^fI*NkcNjC^uj9~(lHvKKmO1F;_#lHFnjzu1e2^F3p>RyZ*;YV
zkEHgZ+KXyYhvF$4x(YVlkHT)YkR<G1%lkeQ8#jbBsi-vN3pK<s!VbJKHy^BVvpw&L
zhwUI9d?h$hxX7Ny{uDUsN#jTdNYbYIp}!-<OULbx-~>q+<xHv;J3%6Dy$UwMbB>*Q
z#IA=qZgYlMItU|v)rb)2c-M#>-vN{Gy<k*qT%a+iH!MDE2~CtQ8^ci>vd5<-WXh{A
zP?B#~-*Gs<6%5zl)-GH;@gXE*G7>GmH5|6^J#7PiwfJ;yFpCtY+c69kH}JD?u5yQD
z@K<Q!U!V&P#cQ6hLKkSjMKf6tKJXz-uK18(r5D7TW%ghTZ-^z@+bn&Q{ywllZ&`f1
zAIK1|@Pk^<1itnstt*=G6PVQzJh0CKX0OBr!s!~x-$PVM%gciHVfBg_^vaQs;Uw4l
zg=$YO9*l%}76o2?3JE%YgZdaNJG#ONpvU{`Sn`|bF<`F@HbPq+KJox7rKAVsw4*mk
zq1i<sL(3Y+%qEZjAdZgQ51&*)0On>wQ{`S9Owm~wmJkp}CNnRkO(Ogobbbc8yc;^2
zAjz!7Eka;MGK9Zz9{VsS1!iI2jl@~r5*%>YJP5+rROn#ActRTZz9F#(<K{qu<|y8C
z;cJ>&HJ1XU))KpE5vO!mAsxaEr)NTUY?c8pExNj3`}vTlGZ-w^*y4lva8eU%j)e=P
zcPo7}Awx&BvMg9ZKdN=>vY~~JOg*%iXaO26O~6>4&QE>DgK!_hdK_tj26!e1yh*>8
zIi#P=9dIwgPZq;EFRSI6d$9j<a;=b1g_4GbU||z0>#%LeCHq}HC{jOL1)XXA>{WEo
zfwcQRdO#vqLm|;*;(^`Qz%cVVyK(4RFwyFGWi5nTi09Tp0vXe46?iML>!Ab4%gq}9
zwE>ps0z$E3Jsr65A7H0sZG_Ew!r+6w@Bt3|mK60>5BMY8`iiFc7NA=-7!PkJM>)C`
z(nwBKA$}VSwd}TFJNdH86k?(6*g;|Ei+<m|p;hah;9r+7au>`dyeT}}Di3$T4^XYl
zi9Av!Fhq4`W&Cc~ZJt?SaS2D|;a+InpynI99(V(oK?eyKvZnN)rot{0+k*v%2pBb|
zZL3T_414tir_V8%K`LxK2GhyB7RO;aM%)BPTzeeG(PLqIf(Xrz6Cvvaj6<6fG-_~?
zrh+m0Buv9ZNqBY=#^dH98cp~S#^MV}tN4-FC!M6x;8QevOO6Upk)R)jo~AWJP7~qa
zX=3Veibg^CG!=lW@@Z<oIU=+^LsM?~G+KU!*4!c4oeGGSafU{7<!FK(*x;`P5RI_~
zM4Nt=XpiOSfgFX~)0zMOIhq}NmS~sH5$&oR^(&<H^5q^s7LsPy9EdNp2qserf!`L<
z{tlAjex4*-o+qx2=Sgz-k2Gp=K{Y(>0vT?1fe1FmFrEyWS`5>qXWcC(>0TFUBKjwi
z`CYR7@)Pa$ja+B=McS>hm`24HX?E04G%{XN&E9>9&=114Ka+g;C8F>AnK&*<4)+q0
zf3Ji_xg{ijT&~`@lxBB0l9p+uFd2C%(R7!Iw&rIV#a<@9EGhAXXr$iIGFm+Vi|7X@
zTqgR!E5tjbj79}=)a?q5BCpb@P}25ar5z7Qn%^}tFBn%`BlChnuu(a!6Nu^ZCn%?a
zR!fjOr&SRD+;STIUO_vbx<R9<*NOI8(kibL?Lh^NM&BUXG&#C-gJ_qp)2R1NSO5kY
zsx9!3O%RTKIzf9JwFzvMNw*;GUClw8BYOoV)z#-xWGY)<>EcVx?`NsLhqCdKp02b&
z^exq=Q$eT&>7!EpLm7~&=PxPKTV70+u<?q1HaZ_5&<9`DKeg~dc^$q+6?z$=iz@V!
zAu#7UtuOD_Il2Jl;C21rY@H#5lJEKp5wwqCHn?CaYb#Ue0K7Yu4OHs{4=miot!gEv
zYULsYtZ{z~tIkjV2k!aPC{>b~5#_pE7J5PW{&ZSE%LUgw3B%F|{yrX?&c@P(L`hFS
zsu^sRc{7-`IWhSxo;6dh&Scvpg7f3pLJe3VSE~2f5xO`YB(PyLtG(Bf;E~9ZHNLBf
zOifG()qObF#Hj32`4l1qfV_t3azm#SrWKb`*d~=xQev5KQck9_JMu14*QYGEKe}#V
zP0*0Ry5quh_OZ--HJU;+PtXU2hN39|jw!B-*+e}Slrt+_p2bXboCmX6OYD%%2AS8>
zNV#-|)dFj@JVa_TBZv7{@4HA6(veg%vlcF8Z?L-O)?p1@Rl}OJnyu4Q+Sx{y_V3`D
z2=0za>sho~v}lD0N7oH3Pp!d(Q(>s$x{>*rZT7@-dE_Zx>zSRBy_wN_<+`4|i!`jl
z@-6HrnXv0QJC0S`*&OBJHr9*QTF-3p&iAZ2j{crSX(p)4IN_u3*?5u|lE-Ecy(o|6
z)|G#x!*BPoP3F_xi`M^Smf#>1_F}XBDwMT0)r6e!_92#Imiz%jj?|ntNgCuURgctk
zCH5rSZY#sU2M;&kop8x{HkoR8s;V*RB5Q$R8(Bk)xxfYxvDF+O?7g1#K%0w1>{-l)
z;^_;ljaq`LrKuuIQ%1)5-KWmPh+7miwilQkmZuY|g9HBn50$V%e>UNPr%Txp3S5OW
z^<s*0jpgbkvlGs~$y{*e4HktL<!9Lq3YwZ-)OmTI-o8oD`r^o2Y`Sc7T)M?_Wx9JD
zU;oZhkq+oR#pwm>qR02{F<%SHRrl(k?EHY4>X9qc;n^n)EwR_)Sslk~_nczy?fp`m
zsA84Hyp!5rk~n{-H7+a=+hVyDA0&^#VDZr=xZRp6C4)8B5LQ07;m#7=wN1G{raOUA
z3&w{IygNoY@b(yO#a%2|cSVCEmmW&dbsYON;wj3A_WY(!adY7-^pe*aZ#3b}@#cGc
z0IeF<gv&RC7N_U32_H|v7S)u`pzGj_{PDxcW_+b=u+>`ch7Rpp@@kLWgA-hN?YK>7
z#VHT0SIP-HxbYe|)7|TcuJGU{T)UN>z}}5`s?=WFZH98#hxfLxZUZQzI`F2_zbRpE
zfv-1Fez-22+v#PyPXg|Ue{|+UFg${L3)eaAqz6;ismKb?M$r1>Bj^DjhGBT66K{(t
zT?j(8)iR_rA1`NR887=4>NVkl4?f|&abOotnRaz`YVJeKuDQnDv8oIA#NQ(M5nYHm
zaV=>K4$9+C`B*L!J^8HO-$$j8$;db9sZ&BdBdi=qK5(Wt_r_29)4s0xcGMwC!1n#<
zsB-&|cHa!-vY0k#2k+gN50{0os<3Pw1>hPfZ0JXSCJw4LWOqM4N(&ZGxj>Xf*cVIt
zYkPUf7SArZmfG4`B+9FNWSQ?S9^LWO7X5M{nH;RXQ&P`lwX3BT=VC;u%HxTSgUR81
zwW5y?;h!=!RH!$bGlHj(Bc+buHMPL>kwjBFx>UoB;{Ht5EN)8t=X|70?M>sjw_^PT
zpQ!_Xg+6x3%1DLn@n_6Y88(4;&}+R$dB%SDU^4HZHk{Ne%1?F*F*BNHt9O~&*bTs#
zuXq@J!);&8j^pjn{%d|f7hteBj)U@oC_rGzQdKFL%Ae6g@Mmd6Bu30s_cS+-C#g?}
zbe?}!H)QJTzeStRLZ%Q}+vY8<@>x7l?JTQ$-Y1kyvw1HiKc2fu<(?++g<65PGoIr2
z+-!cf8ffgF#A`Py7Mgg9WDdfv$^4LoZ+{A})fvWuRBqA-h8nPM8lPlgi<!fdWu-~P
z4j3?(2WpVNr7^X4J7V!%-kExC>Qn2F6X%hLY<z3a!>{wWEdAbU?#ukDBi>0@W1pT4
zPhH5La_!4dH>in7Iz=BGJf9!Z;OM=OH&s4gz&#{NnVE!=nn(6o$nRJzdYDxQAkRev
z5b*_K`66D^$r&i>05on10TfK>4PoJphjQp%asP(<TNZZAQT@4kwu>6m_2!=)F&cR%
z2r;*X)mOm}8!RPfqE2^}8qOVYoHaP(hNXO>>{O~I*N-}_WrZ2bxJ*qYe)yyjs6E!d
zU>~}i0zvLC1DL6on+oN#<piaQ*}Yuebni;;E|K)e<(aZ#B_|BR`?<WPEtbBz4w%Q*
zkiP_JeIJZl`v#s**44pNzo7=Nq5mK>k8XGa%`F>gHbBFgW;fK%$**pr?ndY)9%;5&
z7h2DMUXZw^(yjBh%h5z6=5OW|vLuoJFT%2|^a{9a;Zelja4T<#^R{rAr>YJ~ixVwZ
z%c_ZUc=eg*?BE=qZR6FMwfespBAWe4*-^uar@pIW|K*)zVW4IE_R5i6{NGFI=G)q%
Y!6%NkHdDsLOq(*<HD<zO+XUPH0?KiT$p8QV

delta 7102
zcma)BdtB8;)BnslyTCziDk`KsV#Xcxa1&5dB&#=}yr))L_VKO{q9CG(+{C+rZZclL
zG&RXIF@&GmtEqWc5Jg2n5%6xBkUhS6e`kNca{#eEpXVQ#GdnXo`<>a@+5PRyaz2yg
zoL9k|M#p!GiH&i2$Hmvr+AvA|HJW}IMHw5R3pVBl^RMC|Ktx}36Z;b<K>)6I07p?(
z;vnwM4iIx@hiWVz`Z*ke@k|DIioLUatw)X;HE`ni=;+a-#Io}(#r~vw5Qx1zpshHT
zTm_*Tc_`&$U7(++UR)rKr^f62{LpU>O%|6nNMjd@)H%t}Md;HbAV^HP=-Slm%(-nu
zUNy&{mAs=kpWYQhg<oaItXO6c-RHKqZ0y0zxltnQrjK~Akc-aqGeqt@Z<D_?W5d7+
zQTNCLs}^v}CfbcAn!BJo_TRwX7PTer!gk>zvz<Vb9gileJ;k&|2cQeqt%g9c{cC@b
zvp8H^;4KIi$Cvbw2CgwL0%EuLZRspsu%B5rSe(mv#|*~tOuq)nLQ%0S2?(*h1Wo4h
zX=KSlEY=0ot48Fv{8|B=vI*Sr<Wy#x|6;{zof+=Ft5OKpGpo811M5cc0&#Hl4D7WQ
zyv3L`VT8!5VB`8&)?Mtcw#@%m_JdC1z=B9I>|`dbX3E75qS>iD2qCJ!uq|3BPL)K6
znxa(k;+CUuco`z{ic<xii4cjWQ^cs^2=QS_nizdHLbN%PjCF&#qsXi4A?}=+rG~DL
z$UB>YS6V`(NGwgJB~7{5M`S)6EOO5I;7@iWT~%Tu?wreRgzyKAXqjkj#h{`#!t;WU
z>QsmP=@%Y@nmv``d1(YDTftEA$0c8r@%zVZ@UJ2KO%Yb)E%ufB;5rB3BKG13!ezgu
zc&8#Bmkr>3=!b)_zdTKb-GvsME<CUJP>fWEg0cqit|I!Sk6@MF;`6(aBB4@<V>PYF
z*&tC>WhK76nuw8yzzT<LVC^VtTvcwTs_~R;a!FEZJb~Wprgx(ZH-jFa<(Jn#3K+!N
z8t)YV9yojuPoyvz#Pn<4(mfvxvj*3C|At-n!K;k+Z^`YR;``^rgx?J>@#MuvBKHQ;
zB($~rtDdd8ISc%;ZlNmaq-~koj!kfOUw+y`Y`*0~ip{4n&;L>(<F>EJu9F_@$8LIv
z5EXY)u|-QFa_`O&kAI60(|=19o)02K?|Z4}Xbl5}^Zi+3Uu}e_x}PdWJdYGV{ytM=
z+=&#i57ID7N1EZa3F6U%NMWcGrYyQuH(#{)GeYe7BV8uhK#`Z-OyoR_#ojF_wTAzh
zD*Q{O503ek|G5hiDUW>T|H}pO)$bbn&R_Jy`?pAO<w>@z2qN!Unt1zJs1^!;@%WJo
zv3`_jek`^=PsQ8TCRkHmBwz;64~LgWOweHMVD4Ms3ts`wwSa+G1F#sI$vDsk!96-i
z5yNjp;9NbV;Q&i$EyIH&7$o5~4*k%VL#pWXLMBQGKIO28%1nPOvw%c#<ep|@oh4+5
zo)7!W>=v4jUgqDQ;vEBciL9bls@<{HkSzR8M`A!Tn5TjB;JCGU5BT7hHsFH}#b71=
z=jMla#fH2%Xu%yZ+7|X=kF#LF#|yzmHI-rqv(=>RhyCnfHs+L@wAwpBDz>ktUm8xH
zBJzL%pE^Li++Kr3NJW2iY5~3kcLZj(pyjQ&AA!SL5)hOBh)_Hop%s|wCEB|{8)acD
zIB1}CebgS7$cpBc<pTdP)iT`L0X~y!)+%TL-g*mCR3R+s4fdGR5q{A`cVY`yXjDRX
z;#+PolMXOCm%3t^8~B+`{^Cw1s2chxlRRJ<=z{(6r6;V>1sT=Cj3r)BAS>sN6ez&%
zJ}`rf$m%W!7AP^kuu*R|RqO};<Q@HKROSn?eE_6Dpt3#y4l%Q6yHHiMej%N@nq~Z$
zQr!cNa=pJ%J0r2OH_X@c3h`g>LxL{As2-k5@%wNDuqKAPW9DeEQQ{-OTZi_ZV5vOr
z16iH4Bg0K8oB@-79Gg8G+G39cNQS@_&2<5|cL_Kv&1S(=oi@0JmR23byp-Na@GI#2
zjks(ExMAlMNH!sGg9I@Zy3;@Dx&&nx7N!xhLDHoSCb0rsln%y5LIt>HE+kT%NhlCM
zng`J|xNjaT(DcS(-h4=c5Hv1;HCjBgmcWPDbs;<^LDlcRiy%oCVl+E$jkb&7sHV~u
zYZud^0+cCBV4;q9&oW>Q{ZrR+FcaG8Xkk~E6R(TL+azGDj;feEP_YbQBPOMQ1>Va7
zS>l|L{8EqbAiRiBTED2TR@JAJmJTtifetgxjO!XGtC!5rZX$Uohn62Av(Fp<y%xf0
zn!L3X)*!0AyTA)aeFG)LQxj|aI{1t%$zBN)))V%j{`h=7bk}%0@xix{C>tm;=A*3M
z09}DBpWjH!3zV+7DYyyt>xtK+5dMX6-@`1OpBhS6Jo${$#B~eg0=Zqa4IHuN2S_J9
zb>i%;@Tqy?ZQCe8RhbYo>-3M5qXF1|dxI<Q<~5PpiKBK<rRburRI&RBc1!0}M{eem
zBSC&Lr(Bf<J7K43WDUhYPOsQ0(9y#51IO%b03m)Kfj~=mvaez0d%w^$YH#VSWbKDS
zJ)!r}A()O!k5VBhJp|)$-4UYt9ENeiSZRfO4#Q+@BL_o{z&Mg#e+1&lH>aZzhZBy2
z9cCXTrK;oLfR4wA@+~FVMaN()4mm+Q-EpGcJ4w`x<21VWB#qWe>huYs#-31ZmYg7C
zKb#`Q;FC18*oGQ`!zXFtV`pfB@KZ1zR}~Sp`xK3SW=j>;s|ccSZZVn7EusmYOR9Y_
zjov#=R<ej9L|zt?t6ebeG+8e%A^j(($x<gfvOckd^k3MKrJ53$OhN<Bz%)!aMrO*+
zkeR}>Wai_uL`7X7#Y1OFvF$mcx|I@TI8W41r8N5JJZX5HBPzFwG}fOZXRco$p3`~a
zeO^Yq3`vDtBFf?djV4Kr#0$i8D<|HcMA7ua%gDz)SBX(mMvM@QxJZme6*SN3i!|f*
zOGLeY3C76+n|DbKfqOZLWL1;S)^ai&f0+z-tRQO46{1#G&=j?oX|(xeqAITtHD6Mv
zDv5e2slk$pyP}5R>J`=ICzbRs6pJbejUs!3C*mr>6N*1yg=xAje&}CCbK2IBGlf;;
z)>BFORugsS8c`do$-!VF+SQO$pNnK&)Q~@4Tql35uhBdK7;}vb(P44-8ch_4Bd^11
zU5Fo6r&C$<$pv4G?ggDOc{5lmOKw0yYb`(9;@i(?7X_)iU><Ep<>WKnta9z3tI!Ws
z4wvbvgau>2a{U77fx0p6EA+pccYpq6y*y-0C&x}abVV;Krz#g;ssF<)=Th~m9Q|e(
zRIN{-AnmBuN0AMe8k!@-h|TBe0+h-c{m@KW!QWg5dShM;^N|g40NTg0Pp~3}$qvJ*
zNe4TbY^H4BUU)f%$=<5z_<Q*CG$y-Oc{MYjaV58q_PjANj(O7>)ca;$T^}^`;&0-^
zI5t+ci_%h0L5+)|OlMa0%|hZV=B(JxU|S_xdlJ}E6+^M^YkQ=j+XMsFtzd=t=4|$<
z2``N?G>K_F1IgN<Z8F<$hAb_Gy#npmsVs>itrqQ`X-r<^G!vn7$b{?}kk}5&qdDxh
ztUv17&8@gQ_T9pqaLhvXA?{tkdQgP49$gzyXuuQnU5x%H8v~{9ayChi2Q%1w0^)uq
zYmXB$*$|Uajg>=@Gs~`%O^0-bu9ezRO!RL2Uu3fQs!}gkGEHiIc7qhryK&Om2C41e
zuy6HfpU<rD@nCL)Uq<teXgtTl@Qd$QGfX(goUvphv&4rRn78TD+YmYn{9+^9AeSav
z)?nOzfwfb{Z(?mE9F(G-U=#=bk}b6-y4dkIaO@VAi~GK3beO$5qR_|$c_K*P3yWEr
zUQ7tI!_ZUARtd~w!8(1Q(TEQD>@Cdvi48KD*J-@zq&GjGO_%GiN4uRYqrp&|9xv`<
zn<ZQ+yBmNYdN)S?OhD96sYy1+*85qONxTq89bo1*QtB6C+(DMoR3}c!J<7IO%N^p2
z4=i{u{Pir0!kp`b>PO|wrXCw4Y?|SQQZ^XRonx*T9nD)xv^O=n9m+dobL=TM$M$YK
zScyH)el@}AMHl&AH~UE_yW#JbC_i5P?SYTW*#QVq4%}znOu2cL<>)CL)?Q~Faq~4o
zEeM}oV{xXG(MG0I@oIg8up@6BFy{tsx+m9Ib8K~!Eu|qfm69*BF?uY%%W~^+>2$9N
zE@Afxm!|T~@p&CXQ>qqV#KZaxCUds{hyTe^UsH-#QXjK0TQN1s5o=>8{|M3TSlfov
zA#TSTfyu4-NDMddx9f52iI=VT811pad?R0VvoV_025v@Ey~n<?DKW2{b-*6>+ym3>
zc;_Ztwtn=k;?S1+FlCS<&z5~wT^l}~Fdgs2P0)Jk{8jE-=<Cb}6V7SQe7anLJsRHN
z<8ho5cfiy)_!8>SDO87Xb_+fSU7oVm%H(!D*1We1aJ36JS0oLvtsQs@ono&mUD4qL
zH{JmCF83yyCp<_q_-{0O>y-z-ysvFTE-^)P<8&>f4mzEAC}+BJ8@=`@hi|^i$Kk{<
zs${j{yj6X*a2L1gTFSCLO(wF8nkzbm@quU^&V9`fXs(zX&ZnqXY=?OUtc~K<viw_U
zlIib~(Jo|^%Kf;k_qZnx>CHQ9cTQn?x@nKb-gMdVe2*W{g_>I0$F0ClY5zVS%hk8E
zUErbQMexmfdLEz`C2T!}55uQ@xsPdw+0c_tB90h9$#kk8_eQsVyo2ah>8(}k&-(Mv
zWyPPt`_c<n0G=jA95aCam>vPu=N!B?fPZPKK+-@fblAd%ASoNsFi4$BeV^H(dOxJW
z(&520qwQcmK-k{#){L+Egp5ZGB&eK*@KpH%rQY4u^Nt>?=)4P1U>J{=H%Iw;SBQQi
zD3C#zKZ45}fkM1spglyVl$SE_OYYAo-ww3mZpzkCe1z=t)oW1D7(P=c&#ZRn@G-Mj
z%E$3JI@t)xQM@;bVzKHVyp5}uK?Fum;MR)mB)*F-PO=}s#QEG0+fC-(Xw~wRZWQ?K
z#g@2tGS47MdXJSm@!#sQz0GI58-`4wcm*2O-bits%J<64D8%tdrE@GlMYZj3dXDZm
zDW1P8mOoK9cK}+<pocgA(hudM6*W&hQEqadCJl(HWQ9v+a`nMio<DS_`bW915_mV7
zL2XqUy!|VIA0#~0E6IadJWM)koXwXKtZI`*yf@G`i5JU=_<chW5y`y1#ZVc=DHNhE
zNO23qIjP+ACQ*n_)42ICq1#;{+NSd)eGr{tYKyTaovST|YGUVHo=i8?FuLO*4LIPG
zc|1^y#%qi2ypUVsoAY^hb@R~)9**1R(+TlP&URZsn3>wCrj1#5Bope}BSD7ETf*OU
zcdC#9Z5L4><TIIR4}0OaiwH0iI5|tXld@wm_mHTSFX2l?cv>K4FXeaY(RwGd30m>X
z2(1wHNdtYCbJ>m|e9d=buHc=>jBIrkVg%9Sj8Q|R84@C~E4Y8X)e%`_MBN46NTa>g
z?eF>;RQr*K)0Uw3PG^x<)J&$U9}ZnfiA~q|Qfk$bt-u){t>lwruk&~6R#?5NA-zKW
zS*&n1r;ikkpH|hr$qggd5MHK^{e!jKS@}GhyGy+0<nSf(o~x!AjmRcep=@xFgAxUq
zZoAQSJx|ni(cXZtY`qGYnofN;yqaDkH#T5mx2aKbS$(JGr<!#%;DCp}<0~4!`QnVt
ze6%*HC&H%Jx_$YfN6K`-z6sPcZrx1fba5^fH{GV!I{zlJ{@?Q(GLG`|4?f>Pr)<I&
zK9mYt+z-4xKHS1((<@s?U8u^-G57U$nsnS&?kl5Cb*}(F-PY)olZ;6}@>Fv!y|tZx
zUN0Gx*F^Gz9V8j7x}lI8f2?#5wYD~lALF8Z8KduH@<W+Rg0;c)txSG7Gg=!Gv9^yz
z&#{KsiBX+QADw7GDvcT$JMuG=c30E)wc*;AwuVn+gC<WL9pypFJ}$A7C;F(0u`#Cq
Vqdm!msiR}3O^tFP5$i<j{|AXz_lE!g

diff --git a/internal/scanner/scanner.rl b/internal/scanner/scanner.rl
index b3a0015..e7d1179 100644
--- a/internal/scanner/scanner.rl
+++ b/internal/scanner/scanner.rl
@@ -26,9 +26,6 @@ func (lex *Lexer) Lex() *token.Token {
 
     tkn := lex.tokenPool.Get()
 
-    lex.sts = -1
-    lex.ste = 0
-
     lblStart := 0
     lblEnd   := 0
 
@@ -127,7 +124,7 @@ func (lex *Lexer) Lex() *token.Token {
 
         main := |*
             "#!" any* :>> newline => {
-                lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te)
+                lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
             };
             any => {
                 fnext html;
@@ -143,12 +140,12 @@ func (lex *Lexer) Lex() *token.Token {
                 fbreak;
             };
             '<?' => {
-                lex.addSkippedToken(tkn, token.T_OPEN_TAG, lex.ts, lex.te)
+                lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.te)
                 fnext php;
             };
             '<?php'i ( [ \t] | newline ) => {
                 lex.ungetCnt(lex.te - lex.ts - 5)
-                lex.addSkippedToken(tkn, token.T_OPEN_TAG, lex.ts, lex.ts+5)
+                lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.ts+5)
                 fnext php;
             };
             '<?='i => {
@@ -160,7 +157,7 @@ func (lex *Lexer) Lex() *token.Token {
         *|;
 
         php := |*
-            whitespace_line*                   => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
+            whitespace_line*                   => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
             '?>' newline?                      => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;};
             ';' whitespace_line* '?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;};
 
@@ -320,7 +317,7 @@ func (lex *Lexer) Lex() *token.Token {
 
             ('#' | '//') any_line* when is_not_comment_end => {
                 lex.ungetStr("?>")
-                lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te)
+                lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
             };
             '/*' any_line* :>> '*/' {
                 isDocComment := false;
@@ -329,9 +326,9 @@ func (lex *Lexer) Lex() *token.Token {
                 }
 
                 if isDocComment {
-                    lex.addSkippedToken(tkn, token.T_DOC_COMMENT, lex.ts, lex.te)
+                    lex.addFreeFloatingToken(tkn, token.T_DOC_COMMENT, lex.ts, lex.te)
                 } else {
-                    lex.addSkippedToken(tkn, token.T_COMMENT, lex.ts, lex.te)
+                    lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
                 }
             };
 
@@ -378,7 +375,7 @@ func (lex *Lexer) Lex() *token.Token {
         *|;
 
         property := |*
-            whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
+            whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
             "->"             => {lex.setTokenPosition(tkn); tok = token.T_OBJECT_OPERATOR; fbreak;};
             varname          => {lex.setTokenPosition(tkn); tok = token.T_STRING; fnext php; fbreak;};
             any              => {lex.ungetCnt(1); fgoto php;};
@@ -474,38 +471,32 @@ func (lex *Lexer) Lex() *token.Token {
         *|;
 
         halt_compiller_open_parenthesis := |*
-            whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
+            whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
             "("              => {lex.setTokenPosition(tkn); tok = token.ID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;};
             any              => {lex.ungetCnt(1); fnext php;};
         *|;
 
         halt_compiller_close_parenthesis := |*
-            whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
+            whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
             ")"              => {lex.setTokenPosition(tkn); tok = token.ID(int(')')); fnext halt_compiller_close_semicolon; fbreak;};
             any              => {lex.ungetCnt(1); fnext php;};
         *|;
 
         halt_compiller_close_semicolon := |*
-            whitespace_line* => {lex.addSkippedToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
+            whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
             ";"              => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext halt_compiller_end; fbreak;};
             any              => {lex.ungetCnt(1); fnext php;};
         *|;
 
         halt_compiller_end := |*
-            any_line* => { lex.addSkippedToken(tkn, token.T_HALT_COMPILER, lex.ts, lex.te); };
+            any_line* => { lex.addFreeFloatingToken(tkn, token.T_HALT_COMPILER, lex.ts, lex.te); };
         *|;
 
         write exec;
     }%%
 
-    if lex.sts == -1 {
-        lex.sts = 0
-    }
-
     tkn.Value = lex.data[lex.ts:lex.te]
     tkn.ID = token.ID(tok)
-    tkn.Skipped = lex.data[lex.sts:lex.ste]
-    lex.addSkippedToken(tkn, tok, lex.ts, lex.te);
 
     return tkn
 }
\ No newline at end of file
diff --git a/internal/scanner/scanner_test.go b/internal/scanner/scanner_test.go
index 465c40a..c49b785 100644
--- a/internal/scanner/scanner_test.go
+++ b/internal/scanner/scanner_test.go
@@ -386,8 +386,8 @@ func TestShebang(t *testing.T) {
 	tkn := lexer.Lex()
 	assert.Equal(t, tkn.ID, token.T_DNUMBER)
 
-	l := len(tkn.SkippedTokens)
-	for _, tt := range tkn.SkippedTokens[:l-1] {
+	l := len(tkn.FreeFloating)
+	for _, tt := range tkn.FreeFloating[:l-1] {
 		actual = append(actual, string(tt.Value))
 	}
 
@@ -404,7 +404,7 @@ func TestShebangHtml(t *testing.T) {
 
 	tkn := lexer.Lex()
 	assert.Equal(t, tkn.ID, token.T_INLINE_HTML)
-	assert.Equal(t, string(tkn.SkippedTokens[0].Value), "#!/usr/bin/env php\n")
+	assert.Equal(t, string(tkn.FreeFloating[0].Value), "#!/usr/bin/env php\n")
 
 	tkn = lexer.Lex()
 	assert.Equal(t, tkn.ID, token.T_DNUMBER)
@@ -1137,8 +1137,8 @@ func TestCommentEnd(t *testing.T) {
 
 	tkn := lexer.Lex()
 
-	l := len(tkn.SkippedTokens)
-	actual := tkn.SkippedTokens[:l-1]
+	l := len(tkn.FreeFloating)
+	actual := tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1169,8 +1169,8 @@ func TestCommentNewLine(t *testing.T) {
 
 	tkn := lexer.Lex()
 
-	l := len(tkn.SkippedTokens)
-	actual := tkn.SkippedTokens[:l-1]
+	l := len(tkn.FreeFloating)
+	actual := tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1201,8 +1201,8 @@ func TestCommentNewLine1(t *testing.T) {
 
 	tkn := lexer.Lex()
 
-	l := len(tkn.SkippedTokens)
-	actual := tkn.SkippedTokens[:l-1]
+	l := len(tkn.FreeFloating)
+	actual := tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1233,8 +1233,8 @@ func TestCommentNewLine2(t *testing.T) {
 
 	tkn := lexer.Lex()
 
-	l := len(tkn.SkippedTokens)
-	actual := tkn.SkippedTokens[:l-1]
+	l := len(tkn.FreeFloating)
+	actual := tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1266,8 +1266,8 @@ func TestCommentWithPhpEndTag(t *testing.T) {
 
 	tkn := lexer.Lex()
 
-	l := len(tkn.SkippedTokens)
-	actual := tkn.SkippedTokens[:l-1]
+	l := len(tkn.FreeFloating)
+	actual := tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1299,8 +1299,8 @@ func TestInlineComment(t *testing.T) {
 
 	tkn := lexer.Lex()
 
-	l := len(tkn.SkippedTokens)
-	actual := tkn.SkippedTokens[:l-1]
+	l := len(tkn.FreeFloating)
+	actual := tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1332,8 +1332,8 @@ func TestInlineComment2(t *testing.T) {
 
 	tkn := lexer.Lex()
 
-	l := len(tkn.SkippedTokens)
-	actual := tkn.SkippedTokens[:l-1]
+	l := len(tkn.FreeFloating)
+	actual := tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1369,8 +1369,8 @@ func TestEmptyInlineComment(t *testing.T) {
 
 	tkn := lexer.Lex()
 
-	l := len(tkn.SkippedTokens)
-	actual := tkn.SkippedTokens[:l-1]
+	l := len(tkn.FreeFloating)
+	actual := tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1402,8 +1402,8 @@ func TestEmptyInlineComment2(t *testing.T) {
 
 	tkn := lexer.Lex()
 
-	l := len(tkn.SkippedTokens)
-	actual := tkn.SkippedTokens[:l-1]
+	l := len(tkn.FreeFloating)
+	actual := tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1428,8 +1428,8 @@ func TestMethodCallTokens(t *testing.T) {
 		},
 	}
 	tkn := lexer.Lex()
-	l := len(tkn.SkippedTokens)
-	actual := tkn.SkippedTokens[:l-1]
+	l := len(tkn.FreeFloating)
+	actual := tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1442,8 +1442,8 @@ func TestMethodCallTokens(t *testing.T) {
 		},
 	}
 	tkn = lexer.Lex()
-	l = len(tkn.SkippedTokens)
-	actual = tkn.SkippedTokens[:l-1]
+	l = len(tkn.FreeFloating)
+	actual = tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1456,8 +1456,8 @@ func TestMethodCallTokens(t *testing.T) {
 		},
 	}
 	tkn = lexer.Lex()
-	l = len(tkn.SkippedTokens)
-	actual = tkn.SkippedTokens[:l-1]
+	l = len(tkn.FreeFloating)
+	actual = tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1470,8 +1470,8 @@ func TestMethodCallTokens(t *testing.T) {
 		},
 	}
 	tkn = lexer.Lex()
-	l = len(tkn.SkippedTokens)
-	actual = tkn.SkippedTokens[:l-1]
+	l = len(tkn.FreeFloating)
+	actual = tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1484,8 +1484,8 @@ func TestMethodCallTokens(t *testing.T) {
 		},
 	}
 	tkn = lexer.Lex()
-	l = len(tkn.SkippedTokens)
-	actual = tkn.SkippedTokens[:l-1]
+	l = len(tkn.FreeFloating)
+	actual = tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1498,8 +1498,8 @@ func TestMethodCallTokens(t *testing.T) {
 		},
 	}
 	tkn = lexer.Lex()
-	l = len(tkn.SkippedTokens)
-	actual = tkn.SkippedTokens[:l-1]
+	l = len(tkn.FreeFloating)
+	actual = tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1512,8 +1512,8 @@ func TestMethodCallTokens(t *testing.T) {
 		},
 	}
 	tkn = lexer.Lex()
-	l = len(tkn.SkippedTokens)
-	actual = tkn.SkippedTokens[:l-1]
+	l = len(tkn.FreeFloating)
+	actual = tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1537,8 +1537,8 @@ func TestYieldFromTokens(t *testing.T) {
 		},
 	}
 	tkn := lexer.Lex()
-	l := len(tkn.SkippedTokens)
-	actual := tkn.SkippedTokens[:l-1]
+	l := len(tkn.FreeFloating)
+	actual := tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
@@ -1551,8 +1551,8 @@ func TestYieldFromTokens(t *testing.T) {
 		},
 	}
 	tkn = lexer.Lex()
-	l = len(tkn.SkippedTokens)
-	actual = tkn.SkippedTokens[:l-1]
+	l = len(tkn.FreeFloating)
+	actual = tkn.FreeFloating[:l-1]
 	for _, v := range actual {
 		v.Position = nil
 	}
diff --git a/pkg/ast/visitor/dump.go b/pkg/ast/visitor/dump.go
index 771ac0c..af4a87a 100644
--- a/pkg/ast/visitor/dump.go
+++ b/pkg/ast/visitor/dump.go
@@ -113,9 +113,6 @@ func (v *Dump) printToken(key string, t *token.Token) {
 	v.printIndent(v.indent + 1)
 	v.print("Value:   []byte(" + strconv.Quote(string(t.Value)) + "),\n")
 
-	v.printIndent(v.indent + 1)
-	v.print("Skipped: []byte(" + strconv.Quote(string(t.Skipped)) + "),\n")
-
 	v.printIndent(v.indent)
 	v.print("},\n")
 }
diff --git a/pkg/token/token.go b/pkg/token/token.go
index b36c85f..406a577 100644
--- a/pkg/token/token.go
+++ b/pkg/token/token.go
@@ -147,11 +147,10 @@ const (
 )
 
 type Token struct {
-	ID            ID
-	Value         []byte
-	Position      *position.Position
-	SkippedTokens []*Token
-	Skipped       []byte
+	ID           ID
+	Value        []byte
+	Position     *position.Position
+	FreeFloating []*Token
 }
 
 func (t *Token) GetPosition() *position.Position {