From 291dc7e884a152956e8ae2f72a6ded50a8a74f07 Mon Sep 17 00:00:00 2001 From: Vadym Slizov Date: Mon, 18 May 2020 20:07:17 +0300 Subject: [PATCH] [refactoring] move errors from scanner to parser --- internal/php5/parser.go | 30 +++++++------ internal/php7/parser.go | 30 +++++++------ internal/scanner/lexer.go | 45 +++++++++++-------- internal/scanner/scanner.go | Bin 384744 -> 384574 bytes internal/scanner/scanner.rl | 15 ++----- internal/scanner/scanner_test.go | 75 ++++++++++++++++++------------- 6 files changed, 105 insertions(+), 90 deletions(-) diff --git a/internal/php5/parser.go b/internal/php5/parser.go index fe68521..1315a54 100644 --- a/internal/php5/parser.go +++ b/internal/php5/parser.go @@ -21,19 +21,21 @@ type Parser struct { currentToken *scanner.Token positionBuilder *positionbuilder.PositionBuilder rootNode ast.Vertex + errors []*errors.Error } // NewParser creates and returns new Parser func NewParser(src []byte, v string) *Parser { - lexer := scanner.NewLexer(src) + parser := &Parser{} + + lexer := scanner.NewLexer(src, func(e *errors.Error) { + parser.errors = append(parser.errors, e) + }) lexer.PHPVersion = v - return &Parser{ - lexer, - nil, - nil, - nil, - } + parser.Lexer = lexer + + return parser } // Lex proxy to scanner Lex @@ -49,7 +51,12 @@ func (l *Parser) Lex(lval *yySymType) int { func (l *Parser) Error(msg string) { var pos = l.currentToken.Position - l.Lexer.AddError(errors.NewError(msg, &pos)) + l.errors = append(l.errors, errors.NewError(msg, &pos)) +} + +// GetErrors returns errors list +func (l *Parser) GetErrors() []*errors.Error { + return l.errors } func (l *Parser) WithTokens() { @@ -59,7 +66,7 @@ func (l *Parser) WithTokens() { // Parse the php7 Parser entrypoint func (l *Parser) Parse() int { // init - l.Lexer.SetErrors(nil) + l.errors = nil l.rootNode = nil l.positionBuilder = &positionbuilder.PositionBuilder{} @@ -73,11 +80,6 @@ func (l *Parser) GetRootNode() ast.Vertex { return l.rootNode } -// GetErrors returns errors list -func (l *Parser) GetErrors() []*errors.Error { - return l.Lexer.GetErrors() -} - // helpers func lastNode(nn []ast.Vertex) ast.Vertex { diff --git a/internal/php7/parser.go b/internal/php7/parser.go index b2a0cf6..72dab68 100644 --- a/internal/php7/parser.go +++ b/internal/php7/parser.go @@ -20,19 +20,21 @@ type Parser struct { currentToken *scanner.Token positionBuilder *positionbuilder.PositionBuilder rootNode ast.Vertex + errors []*errors.Error } // NewParser creates and returns new Parser func NewParser(src []byte, v string) *Parser { - lexer := scanner.NewLexer(src) + parser := &Parser{} + + lexer := scanner.NewLexer(src, func(e *errors.Error) { + parser.errors = append(parser.errors, e) + }) lexer.PHPVersion = v - return &Parser{ - lexer, - nil, - nil, - nil, - } + parser.Lexer = lexer + + return parser } func (l *Parser) Lex(lval *yySymType) int { @@ -47,7 +49,12 @@ func (l *Parser) Lex(lval *yySymType) int { func (l *Parser) Error(msg string) { var pos = l.currentToken.Position - l.Lexer.AddError(errors.NewError(msg, &pos)) + l.errors = append(l.errors, errors.NewError(msg, &pos)) +} + +// GetErrors returns errors list +func (l *Parser) GetErrors() []*errors.Error { + return l.errors } func (l *Parser) WithTokens() { @@ -57,7 +64,7 @@ func (l *Parser) WithTokens() { // Parse the php7 Parser entrypoint func (l *Parser) Parse() int { // init - l.Lexer.SetErrors(nil) + l.errors = nil l.rootNode = nil l.positionBuilder = &positionbuilder.PositionBuilder{} @@ -71,11 +78,6 @@ func (l *Parser) GetRootNode() ast.Vertex { return l.rootNode } -// GetErrors returns errors list -func (l *Parser) GetErrors() []*errors.Error { - return l.Lexer.GetErrors() -} - // helpers func lastNode(nn []ast.Vertex) ast.Vertex { diff --git a/internal/scanner/lexer.go b/internal/scanner/lexer.go index 5c8dfad..60aaec9 100644 --- a/internal/scanner/lexer.go +++ b/internal/scanner/lexer.go @@ -13,15 +13,14 @@ import ( type Scanner interface { Lex() *Token ReturnTokenToPool(t *Token) - GetErrors() []*errors.Error GetWithHiddenTokens() bool SetWithHiddenTokens(bool) - AddError(e *errors.Error) - SetErrors(e []*errors.Error) } type Lexer struct { - data []byte + data []byte + errHandlerFunc func(*errors.Error) + p, pe, cs int ts, te, act int stack []int @@ -31,17 +30,29 @@ type Lexer struct { TokenPool *TokenPool HiddenTokens []token.Token WithHiddenTokens bool - Errors []*errors.Error NewLines NewLines PHPVersion string } -func (l *Lexer) ReturnTokenToPool(t *Token) { - l.TokenPool.Put(t) +func NewLexer(data []byte, errHandlerFunc func(*errors.Error)) *Lexer { + lex := &Lexer{ + data: data, + errHandlerFunc: errHandlerFunc, + + pe: len(data), + stack: make([]int, 0), + + TokenPool: &TokenPool{}, + NewLines: NewLines{make([]int, 0, 128)}, + } + + initLexer(lex) + + return lex } -func (l *Lexer) GetErrors() []*errors.Error { - return l.Errors +func (l *Lexer) ReturnTokenToPool(t *Token) { + l.TokenPool.Put(t) } func (l *Lexer) GetWithHiddenTokens() bool { @@ -52,14 +63,6 @@ func (l *Lexer) SetWithHiddenTokens(b bool) { l.WithHiddenTokens = b } -func (l *Lexer) AddError(e *errors.Error) { - l.Errors = append(l.Errors, e) -} - -func (l *Lexer) SetErrors(e []*errors.Error) { - l.Errors = e -} - func (lex *Lexer) setTokenPosition(token *Token) { token.Position.StartLine = lex.NewLines.GetLine(lex.ts) token.Position.EndLine = lex.NewLines.GetLine(lex.te - 1) @@ -230,7 +233,11 @@ func (lex *Lexer) ungetCnt(n int) { lex.te = lex.te - n } -func (lex *Lexer) Error(msg string) { +func (lex *Lexer) error(msg string) { + if lex.errHandlerFunc == nil { + return + } + pos := position.NewPosition( lex.NewLines.GetLine(lex.ts), lex.NewLines.GetLine(lex.te-1), @@ -238,7 +245,7 @@ func (lex *Lexer) Error(msg string) { lex.te, ) - lex.Errors = append(lex.Errors, errors.NewError(msg, pos)) + lex.errHandlerFunc(errors.NewError(msg, pos)) } func isValidVarNameStart(r byte) bool { diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go index 95db2b2208248f3035172eed40ec71fb126387fa..576f19af7fa11f6a1cac2a43292aba794790ce26 100644 GIT binary patch delta 5951 zcmZ`-d3=vo*8iO6xld(l61zx5LEO zXK1V~LZ8~Q5OkSNk}9pSgi?e?qeiEMX3o98weP&|U(bEc_ug~QIrrSN+}p=nmmF)o zZ@Jof+U#lb|2R2)^6On@PEL=Clf$l23wgB`tC$UZ3d>Loj^cYP02Hge__5aSO(h|K z&)pQhj{EZ!StkDOqFy-H4`Sp`r!4u?f|p%xLxeGk#eyH?vi_>UXv1>5X627A$MRL# z6^g}()omf1&(5h)tTs6=o}T+6sAiLkeG8tov=8r`?@-J(J~6*cG1>T#CC7Np;;vq{ zlJ-zBo1EOUmS*q)OIu=n12ghI%T{Kx4u12%y}ml zVtH&qj%sokF}IgyAfO%pwVDJIs%_|7_vdw`Fk{P ztcvAVYG2}KzUapN&vYnm{$+1aZC>H#{k#6 zZd-WB)nvZp;cBn>EFx}nD_-`?Sm4VaxhHny=3fhVgZd|1`0$&_{M^$uq+IpSyLd=rFZZ5`L6X)#2l0H^ z<5bKB@LOLB={%+(S^O*&A+2A{AaeSlSfoI_VzA){Dh$RL6&8Y#ztcDf@2QZDC41?* zCf5wa;|vy&PYK{{4Kis@8uHK|7>;}fGO_TOYhFwf$PrS@5R6q#;Qb8&kb5F;(kWI&UzhNLn4L^NI00evJ72R>8*ipot#ifDJtqn;D7L2_w=W zNUuwS@stiNIA|JdRuZf@aj{~+!ub%cTc^WZ#RY|9#Z1T+@)^#nar$es;3Oz^qm$9z zS0R|c5*mPWUxR5b9l9W!1DQ&KL8cQ7n+NUj>pAe82e!>6*kE_6b9*j~^dQq;C!a9M zoF>+7QEXNm{U(It*?B?)n@9Q-9C%O(#;W;H>k;$(8#Kpm_oxws8$%%k2Tg|wsJ;oq zUCcHs27~0IlNBbAmR8KlAOl#eP6hifgqDhGa2RmRBG^O&vy&d31(OxwJ{CNd17}?_ zZ&NI0r=Ev%U^|vAh9=k|A7s3xble5g@<~~nRMw1Pi{V?3vTc_LD>+>Yk0fa!cwz~7 zQZ>deg++7&MNJWbC1Y7Io?8l)ipgnK9@gwxtc|2&tX~E_6mfh}1@NBC+5C1vuwGvP z`&Gqc#VzY063bW9US_+~ei@f&*_<^{1XPS*Bq#oscL=9L6-5Kycn8L~kZ>Hk4suj8 zx$)I?@Vtw<2@+(fBc;~A=Wv)f$(=SUW{cCf<2FFCn;MCRjlv49yKsFYL_g#DGwAmB z;RK^df=~Vi3zP&i&ff-a2)o>+FMd@pe89%}$~W-wV@l?M?{Bul5o~F~ar{VKa6UcJw3S zuxJ;Jj33iz)E*k`q>-qxn(w1o1NRaI_Yt=FV;TkTCth1s@Wbr=FdO%LM6@USX}-yV zQw|8FtazEexO^W;HK~}STCktyd{;~yS{|p-;Da!YR6TrLa8?&qHL8;`HJX8udFNFbD2FL0DfQXU~(;RGUtcT`87_R?y`)JH#cAz;5?h zu)bmqi~_yqX;{|M<(CwIE<(0yahRlYFQXfzXIxSi{!_8bbYsxF=;~R;b+uzL=7^f5 znB=85AVDu+`=jc14_tSYV5^K9X08OZLj9O}$%7s{u4emQFq~4N{=tJ*m(bb-vp%F$ zU9ncRJ8SZFyf^k7%KWfTD(glmvHDC5u1RH+u_2st<>gB;vKACm{Bhh6=BWnpaX1A0 z4mZiwBJH-rs8(7477k^@ok>=dP6nLy5}V9teeJG4V)*zfe?5K}^OOC85d%lDaGqig z)K|XDKJZ#(_(+x~Zkgyj^x#pflQ;8@d4(WqePniPKAM$zbbT_Kk)15gSQ9svMd|t& zwm_y)(T)iVQ=uHcL4_EsuV8KUnu#oy7*nuX&MGiu4x5b$@3K~U>Z|NURpf>(Y3vQb zUUqXPC!sxNOl4VQ@&8^_p`bR4nLPqcnnj}8E&s>Nw-hXz%d&BE8VkTh6PfHxLd7A_ zr72;3aP#ZzW$McuqJMh>)8?_6BFlT~d_8smeOT!~YG^(s*tH7k+Kh0V&4CxNfV%9fIwj?jlCJR)-BHD5sUCTDo z@P31gS@OEXideIZh^`hb6vx-HsW|B_i|5zQzl7KBvZ)dk9UH?nL&;zvciAwk`H`$^ zkxj+wnmEk5!{YJdI@V3Gh;B@DDzSpt6*KOUnJ5IyayTpqg7rB+u{2GTFYbbia^pkx zp-NHTx5HmQUmJnRk7ygwG0Fxq$lK0y{AZo}@a|Qx2<-Eeb;oZWvu?QhE*r{6pERw1 zUyH|m4Ma)#(K&UCD$!__XCyG3hraCwKUbrv5Tfa_rhjss`4v zbouwFQRqw^Z_OYQI1eAxI?sdJW)ZPF)1gzhux06fMr5Vu-LvXZ=#3Hda=L zvgV)stoD|o*i6nsI3QG8Oa-URm~j{pL`h z=+L65Ez^4AoZp;^d*fp)oy;?MyOp*EFE+R@kX-O(Yg#}wp|QL!arkN*lAB_;J7ZI^ z(^m7>M~7(#C<@B@vokLk6sY$N*KVq+&HX~t8^=X!e)x8Kt(SNlnwTGtp&KbIU5(Zv zuyvHy+vnjW5sRWUDp}=w-vk^l7Ov``xie>yFVaucvJ}>PAxYd38>6)nQ4Y#f_n@QJ z$D8U#chd4)4TD*4-dXGHg(k%kL@lArf?-`WZx&R#5<=CkcxXgwK|`^BoED1ZUA5Dy z-EHIESlXRLyc4JOqHOKCHeSQ0cO(5oWgs7out~hOh#pwPd*cKw6HD*K`)KEb^ppHt z7tNs=aY7H$k18su->{y(`W;IUqRI}%tj{<2>UYvekh6I)VTQ>^zdxG^v6y`H+ioHK zqAfx{_ucD_Q4SLFMXTljvv_uK1zG7M7p$60i_Z73-|jcV44XECo`U2x5w(v62ii4* z`=0$C>EO`%I(ho|Sc!vt%wW|?$$8#GYWF;eYPRbiJg>dtrN$%@)!E{i@M5BmY88F7 ze99!=lY6G07OLm>)!vn>Db*)|gHFq7U6M0U-|z>m8Hln|E~2(D7JvL7Z5Xv+7K9|N sJx=Pc4RFflP~)R?5?)JUzK5QG_)GBGhZJ|$&Rj5b7F z+r`hUEU`;Sl9Ze{o%nf{vwp`SQfn~U!v$@YpC2Y9LnuF~nc2{2^YfjSn3^^%nWUKE z?8W>*ve}&Ra*vN88kgQ<9eG@qi9g7UfoPsk*@6Bh6XIq%)2Rsm>~P6y<~JYr=7!7R zeC-^Dj`ngrKImE~znxt!Sq=DQCuq&fXJ40WR(0(xb-V+T#fY&MS%!TR$)Zv_H|1XdgtidkT)TLs25h__GsHP| z*^?;AswWlu@WRq1lHDZi+MW+M76iRHd^?_39n%rdoKwNhwq-#e8C|rw$8*4lO;)l# zIQt><=7Y;K=|5otwB)fTI`M!Chh#LWT(4E+^0?FOxubF;Q5+k@-r`fPcHqe;awVfM zZ69twnZup;<2am}E!oU2zuq_#%+Ed?#@n3MNhZOloipilSF(&M$*Hd&Q6v~Wqz5IN z!L=?uyOkW~fd>oO!gy7sj(b%_ttbPA{2mMDf_dzDFC6F%1NfP$NXcqf1=+f$$AJxRq_ap7d}hApf)5{hDZ=gd|3XFmQzy<3$L2Nev&VD!awMG} zz}MdLcaFRq1pMTa!TiLPIlRTI<|OB7>>tmf@R0_Bg*g115+A)GbJ%^|K7>8?o!G~wxn8g#X$MGIF zbNCO>;<(?f96q*E$h4ma(0|cw2k-r|Kc8@CCV%H@9FMr0!*9Ln$DiKKW9HIsW`FI%yZDf0t#pYg z`8corItKXi3L=(?c;SB*@Vr}slZlUc=HSu)ajmaac=M>|nf%D>!TiPZoaKwjEM+g| z@-yXeeDTYLTvP5o?H{i)c+BxQuB%x>Iu|e)_xX?ckSUocu*8mL z46?AnUKg$}-e5%fu*~zjO$NHps_6&fOfPWI1@sYI`tr=G!C2@GI-c>OA2#%X9KQW- z9A5GP2UdDRoI(-j50c%g<}+Ly0D=4KL8-4RtA*hMf5;J;%%Tt~7(C$W<}R=qzYK&@ z7c>;}T0n+mGN2(=)+zdy&|Z@41{~TNzL0EcX3W4@ZJ`(L2qLY`c6@Av?Kr;;%+T2M zYC_FW2!BuGIb~5WnK0UfYY(t;MH>#|G-wKiED!ohD4~gs2i??H*nsk~2?v>BG#)pq z$0&46C@{0$mCH>EAuGV0t2<)jq0m5iZU>#TB89S6?1#XVS$0_7NhD}u))CK5hx$s- z@i3MWp&6T`L2E3TKn6EhFkz!)!W(Jew;~l9Dfvk-RjQ?FHHm1fW_%pZB9w|ra2%v) zwVHIqUCEH+G1`a}(#W7!3kRl5gLW7;1v-1M`BMmMq(~H6b(;$Bdk`g4Aqz+vckvQC zS4(ya?nx){!l#R)RiAcLNc>oj`nRGX0niIRvBBQ|otMUQ|DbA+c<6B-0!19qJU-uRDP zP&5fW|!fV zsQJ)W5*|rk0ISr}%QF_%S9;}vQzMxz_;Hi76NQkKF>w`U9( zQvjpgz}Dz20Efmxp=n(PUEI(hT)K>|B9ucAHtsDqQtTWK5+C)5w%BAjWQt5rZovgc ziiuN;Ar;VupcgbrNMhJs&4UuX5`w+le+Ff75tK2JAsSvK54PP2UO0Un49EUkp%GqM z2NUq_W?BtgPpc(9bn{ErLo$xuLa;6yV1i^Y4q;( zJ`P+o6x75=;F)}g!f}`2ZS1-PYH)Hj_$f;&AS=KfzJ(gAUQNUu=hIbFs;)}2u1L{p zIx;BP`!yFN&wZYGP?Ig0)oW=$!t}?qLz)d9Y|q1lwW%q@g3k_X9(xc?i#4LCoeoT zgpHM=-9?%@06)I|cw@eEo)u3HWo51+TQZE53*%tHBo>A{-eWn`+l$<6MnbhvGT&zn zL=hJFaNIkBIm7^^7KfyfthNvj8O6wOX0;IG($TE3(qJ@8XTnA|ANRt{sjRu$hyHst zYpqxkSSayy)zSUQtVY7dlh`|0lg+$w>PO6hSyP!W9vIJBVAJvJ-8Z=S3Fr#4fhfVX zTh*#Z0&bC_%^pmqhGr=-gLGJ7m7C!dWS#NRR5qNTYU`Ya#nad%O6UCXiN2Ws1q)IZ zPiKB=D?lO>S+qip&;S7S+Yqoz20JD)5glEzpR-&IPck=DhGjFIL;^V+jP8M|uGlO7 zJD_1N452?8?$2Rqw6J2QT(*h|>*F=7l6Z{K;Lr2e7jFB8p>sajpDvaA2p2j?NzG^F zepH3rF0SX%Xe}--rce^2$g=O7@Eym!$-8!Ru4Iu!O|e!NCVrSdjT!LW&)H=6@TTgF zLTANL$U;4e;}>U{AGTac-MbhZ$YD!x)md`Ko1g@IdyO!LULi^COo>>}k|jab1g~#m z&G5)ZHcZIoi*3$PRJY&6KJqw`*f$kK<3*gsZD#2r@t1C9tI5v9tQiv)X@J=~m}tFS zVQJk-W{Ecc0pCcu_Z8ddN7htt)gz1CLAbdKrW|ASu)$F_0-xOWOm42Z$+y-O&W=xS^$Wg3swB(sU{`85)?cgP=AMa<4` zlz8#x*Mrbe!DQT6&N{*WLs2Q_InurySD$2am8B=x2eM?c;iGTJK2-E3oOvU-T<{Lq z?<^ZCnao&pRuGx6`8l>qq_*Gj`C4+>^X4}n>Lbxo&2+f+HNzPKMsF;K~b3UT*Z0bF+^y$#+v7}>so-U-l8bf{>H|k z_e&Om9|V$8V(7=YFDZ?>Znj!E7!0c28--=BJSU3T*1vAFS5NiDRW-zn4iE~*&}dgr zoUxD+hB{@5xg4*&pb0(j~=m?O<(D41o zw9-MG>4n{z%k*X>-e*J#HR4-=a*Wt|b0v!rTeWy7kR+o1{``+jr)+K^9~NV}=tPyg zR`M@&j#WIhjl_Spm3vUzS<7ikJ2{44dmf+Yqt3cD?c_{BOBT_Ica*MpTPwR?_b3s+ zfk_w5Yu))ng6Vu3|3tY``gM@IyDC+P9FESnWQThgFe^zNw<0 zVJoAY;~q&&SY@nLC>#%)gaN1u&}fV=&7{^0vrx+})GDMQL%eVZ$I`1Rt<=XZ@qm&k zNIagf;A)#}sMF^U7$2N%r+6Y>Z#cz2nw(-(r*E@TFQ!hBFih=C(z<3Mv!d(rrXsVt z63R81Q9kdcDk2_5F{ryt?*U@?(YB=aadfQQSh?I?{!I8&^cIpuuRbiHc!X7|d&o`c ziPtLB4#PIRlxr)r`@bEg_mXKMRL!K)_ZAUR5JH>tdy_=eLlUP5 QCAyD{s7_`VuE`Vs53y^XV*mgE diff --git a/internal/scanner/scanner.rl b/internal/scanner/scanner.rl index abd03b4..f885098 100644 --- a/internal/scanner/scanner.rl +++ b/internal/scanner/scanner.rl @@ -14,17 +14,8 @@ import ( variable pe lex.pe; }%% -func NewLexer(data []byte) *Lexer { - lex := &Lexer{ - data: data, - pe: len(data), - stack: make([]int, 0), - - TokenPool: &TokenPool{}, - NewLines: NewLines{make([]int, 0, 128)}, - } +func initLexer(lex *Lexer) { %% write init; - return lex } func (lex *Lexer) Lex() *Token { @@ -382,7 +373,7 @@ func (lex *Lexer) Lex() *Token { any_line => { c := lex.data[lex.p] - lex.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c)); + lex.error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c)); }; *|; @@ -473,7 +464,7 @@ func (lex *Lexer) Lex() *Token { ']' > (svi, 2) => {lex.setTokenPosition(token); tok = TokenID(int(']')); lex.ret(2); goto _out;}; any_line => { c := lex.data[lex.p] - lex.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c)); + lex.error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c)); }; *|; diff --git a/internal/scanner/scanner_test.go b/internal/scanner/scanner_test.go index ec61cbc..c9f63af 100644 --- a/internal/scanner/scanner_test.go +++ b/internal/scanner/scanner_test.go @@ -1,6 +1,8 @@ package scanner import ( + "github.com/z7zmey/php-parser/pkg/errors" + "github.com/z7zmey/php-parser/pkg/position" "testing" "github.com/z7zmey/php-parser/pkg/token" @@ -351,7 +353,7 @@ func TestTokens(t *testing.T) { T_UNSET_CAST.String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true actual := []string{} @@ -379,7 +381,7 @@ func TestShebang(t *testing.T) { "\n", } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true actual := []string{} @@ -399,7 +401,7 @@ func TestShebangHtml(t *testing.T) { 0.1 ` - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true tkn := lexer.Lex() @@ -449,7 +451,7 @@ func TestNumberTokens(t *testing.T) { T_DNUMBER.String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true actual := []string{} @@ -506,7 +508,7 @@ func TestConstantStrings(t *testing.T) { T_CONSTANT_ENCAPSED_STRING.String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true actual := []string{} @@ -553,7 +555,7 @@ func TestSingleQuoteStringTokens(t *testing.T) { T_CONSTANT_ENCAPSED_STRING.String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) actual := []string{} for { @@ -640,7 +642,7 @@ func TestTeplateStringTokens(t *testing.T) { TokenID(int('"')).String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true actual := []string{} @@ -725,7 +727,7 @@ func TestBackquoteStringTokens(t *testing.T) { TokenID(int('`')).String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true actual := []string{} @@ -819,7 +821,7 @@ CAT; TokenID(int(';')).String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true actual := []string{} @@ -892,7 +894,7 @@ CAT T_END_HEREDOC.String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true actual := []string{} @@ -931,7 +933,7 @@ CAT; TokenID(int(';')).String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true actual := []string{} @@ -962,7 +964,7 @@ func TestHereDocTokens73(t *testing.T) { T_VARIABLE.String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true actual := []string{} @@ -992,7 +994,7 @@ CAT;` TokenID(int(';')).String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.PHPVersion = "7.2" lexer.WithHiddenTokens = true actual := []string{} @@ -1025,7 +1027,7 @@ func TestInlineHtmlNopTokens(t *testing.T) { T_INLINE_HTML.String(), } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true actual := []string{} @@ -1060,7 +1062,7 @@ func TestStringTokensAfterVariable(t *testing.T) { "\"", } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) actual := []string{} actualTokens := []string{} @@ -1093,7 +1095,7 @@ func TestSlashAfterVariable(t *testing.T) { "3", } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) actual := []string{} actualTokens := []string{} @@ -1130,7 +1132,7 @@ func TestCommentEnd(t *testing.T) { }, } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true lexer.Lex() @@ -1159,7 +1161,7 @@ func TestCommentNewLine(t *testing.T) { }, } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true tkn := lexer.Lex() @@ -1188,7 +1190,7 @@ func TestCommentNewLine1(t *testing.T) { }, } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true tkn := lexer.Lex() @@ -1217,7 +1219,7 @@ func TestCommentNewLine2(t *testing.T) { }, } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true tkn := lexer.Lex() @@ -1247,7 +1249,7 @@ func TestCommentWithPhpEndTag(t *testing.T) { }, } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true tkn := lexer.Lex() @@ -1277,7 +1279,7 @@ func TestInlineComment(t *testing.T) { }, } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true tkn := lexer.Lex() @@ -1307,7 +1309,7 @@ func TestInlineComment2(t *testing.T) { }, } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true lexer.Lex() @@ -1341,7 +1343,7 @@ func TestEmptyInlineComment(t *testing.T) { }, } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true lexer.Lex() @@ -1371,7 +1373,7 @@ func TestEmptyInlineComment2(t *testing.T) { }, } - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true tkn := lexer.Lex() @@ -1385,7 +1387,7 @@ func TestMethodCallTokens(t *testing.T) { src := ` bar ( '' ) ;` - lexer := NewLexer([]byte(src)) + lexer := NewLexer([]byte(src), nil) lexer.WithHiddenTokens = true expected := []token.Token{ @@ -1467,7 +1469,7 @@ func TestYieldFromTokens(t *testing.T) { src := `