From 867095823da3b05689a043f22160b515eb356131 Mon Sep 17 00:00:00 2001 From: z7zmey Date: Sat, 14 Jul 2018 18:00:48 +0300 Subject: [PATCH] fixed bug when heredoc starts by `\`; removing `\n` from a heredoc value end --- node/scalar/t_heredoc_test.go | 26 +- php5/php5_test.go | 21 +- php7/php7_test.go | 21 +- printer/printer.go | 1 + printer/printer_test.go | 4 +- scanner/lexer.go | 2 + scanner/lexer_tokens.go | 5 +- scanner/lexer_tokens_string.go | 17 + scanner/scanner.go | 375 ++++++++-------- scanner/scanner.l | 387 ++++++++-------- scanner/scanner_test.go | 780 +++++++++++++++++---------------- 11 files changed, 848 insertions(+), 791 deletions(-) create mode 100644 scanner/lexer_tokens_string.go diff --git a/node/scalar/t_heredoc_test.go b/node/scalar/t_heredoc_test.go index 47d8bd8..8a56a8d 100644 --- a/node/scalar/t_heredoc_test.go +++ b/node/scalar/t_heredoc_test.go @@ -70,15 +70,6 @@ LBL; Value: "var", }, }, - &scalar.EncapsedStringPart{ - Position: &position.Position{ - StartLine: 2, - EndLine: 2, - StartPos: 20, - EndPos: 20, - }, - Value: "\n", - }, }, }, }, @@ -152,15 +143,6 @@ LBL; Value: "var", }, }, - &scalar.EncapsedStringPart{ - Position: &position.Position{ - StartLine: 2, - EndLine: 2, - StartPos: 22, - EndPos: 22, - }, - Value: "\n", - }, }, }, }, @@ -213,9 +195,9 @@ LBL; StartLine: 2, EndLine: 2, StartPos: 13, - EndPos: 22, + EndPos: 21, }, - Value: "test $var\n", + Value: "test $var", }, }, }, @@ -313,9 +295,9 @@ CAD; StartLine: 2, EndLine: 2, StartPos: 11, - EndPos: 17, + EndPos: 16, }, - Value: "\thello\n", + Value: "\thello", }, }, }, diff --git a/php5/php5_test.go b/php5/php5_test.go index c8ba1fe..3e172db 100644 --- a/php5/php5_test.go +++ b/php5/php5_test.go @@ -18597,9 +18597,9 @@ CAD; StartLine: 5, EndLine: 5, StartPos: 27, - EndPos: 33, + EndPos: 32, }, - Value: "\thello\n", + Value: "\thello", }, }, }, @@ -18625,9 +18625,9 @@ CAD; StartLine: 8, EndLine: 8, StartPos: 50, - EndPos: 56, + EndPos: 55, }, - Value: "\thello\n", + Value: "\thello", }, }, }, @@ -18674,15 +18674,6 @@ CAD; Value: "world", }, }, - &scalar.EncapsedStringPart{ - Position: &position.Position{ - StartLine: 11, - EndLine: 11, - StartPos: 86, - EndPos: 86, - }, - Value: "\n", - }, }, }, }, @@ -18707,9 +18698,9 @@ CAD; StartLine: 14, EndLine: 14, StartPos: 103, - EndPos: 116, + EndPos: 115, }, - Value: "\thello $world\n", + Value: "\thello $world", }, }, }, diff --git a/php7/php7_test.go b/php7/php7_test.go index a2bcc71..07bb196 100644 --- a/php7/php7_test.go +++ b/php7/php7_test.go @@ -16312,9 +16312,9 @@ CAD; StartLine: 5, EndLine: 5, StartPos: 27, - EndPos: 33, + EndPos: 32, }, - Value: "\thello\n", + Value: "\thello", }, }, }, @@ -16340,9 +16340,9 @@ CAD; StartLine: 8, EndLine: 8, StartPos: 50, - EndPos: 56, + EndPos: 55, }, - Value: "\thello\n", + Value: "\thello", }, }, }, @@ -16389,15 +16389,6 @@ CAD; Value: "world", }, }, - &scalar.EncapsedStringPart{ - Position: &position.Position{ - StartLine: 11, - EndLine: 11, - StartPos: 86, - EndPos: 86, - }, - Value: "\n", - }, }, }, }, @@ -16422,9 +16413,9 @@ CAD; StartLine: 14, EndLine: 14, StartPos: 103, - EndPos: 116, + EndPos: 115, }, - Value: "\thello $world\n", + Value: "\thello $world", }, }, }, diff --git a/printer/printer.go b/printer/printer.go index 30ba013..26febed 100644 --- a/printer/printer.go +++ b/printer/printer.go @@ -665,6 +665,7 @@ func (p *Printer) printScalarHeredoc(n node.Node) { } } + io.WriteString(p.w, "\n") io.WriteString(p.w, strings.Trim(nn.Label, "\"'")) p.printMeta(nn, meta.NodeEnd) diff --git a/printer/printer_test.go b/printer/printer_test.go index 5f5e155..2f3e097 100644 --- a/printer/printer_test.go +++ b/printer/printer_test.go @@ -575,7 +575,7 @@ func TestPrinterPrintScalarHeredoc(t *testing.T) { Parts: []node.Node{ &scalar.EncapsedStringPart{Value: "hello "}, &expr.Variable{VarName: &node.Identifier{Value: "var"}}, - &scalar.EncapsedStringPart{Value: " world\n"}, + &scalar.EncapsedStringPart{Value: " world"}, }, }) @@ -602,7 +602,7 @@ func TestPrinterPrintScalarNowdoc(t *testing.T) { }, Label: "'LBL'", Parts: []node.Node{ - &scalar.EncapsedStringPart{Value: "hello world\n"}, + &scalar.EncapsedStringPart{Value: "hello world"}, }, }) diff --git a/scanner/lexer.go b/scanner/lexer.go index 0c43ec8..dd6c05a 100644 --- a/scanner/lexer.go +++ b/scanner/lexer.go @@ -124,6 +124,8 @@ func (l *Lexer) createToken(chars []lex.Char) *Token { token.Meta = l.Meta token.Value = l.tokenString(chars) + // fmt.Println(l.tokenString(chars)) + token.StartLine = l.File.Line(firstChar.Pos()) token.EndLine = l.File.Line(lastChar.Pos()) token.StartPos = int(firstChar.Pos()) diff --git a/scanner/lexer_tokens.go b/scanner/lexer_tokens.go index 8f53114..82b7633 100644 --- a/scanner/lexer_tokens.go +++ b/scanner/lexer_tokens.go @@ -1,7 +1,10 @@ package scanner +type LexerToken int + +//go:generate stringer -type=LexerToken -output ./lexer_tokens_string.go const ( - T_INCLUDE int = iota + 57346 + T_INCLUDE LexerToken = iota + 57346 T_INCLUDE_ONCE T_EXIT T_IF diff --git a/scanner/lexer_tokens_string.go b/scanner/lexer_tokens_string.go new file mode 100644 index 0000000..02e938f --- /dev/null +++ b/scanner/lexer_tokens_string.go @@ -0,0 +1,17 @@ +// Code generated by "stringer -type=LexerToken -output ./lexer_tokens_string.go"; DO NOT EDIT. + +package scanner + +import "strconv" + +const _LexerToken_name = "T_INCLUDET_INCLUDE_ONCET_EXITT_IFT_LNUMBERT_DNUMBERT_STRINGT_STRING_VARNAMET_VARIABLET_NUM_STRINGT_INLINE_HTMLT_CHARACTERT_BAD_CHARACTERT_ENCAPSED_AND_WHITESPACET_CONSTANT_ENCAPSED_STRINGT_ECHOT_DOT_WHILET_ENDWHILET_FORT_ENDFORT_FOREACHT_ENDFOREACHT_DECLARET_ENDDECLARET_AST_SWITCHT_ENDSWITCHT_CASET_DEFAULTT_BREAKT_CONTINUET_GOTOT_FUNCTIONT_CONSTT_RETURNT_TRYT_CATCHT_FINALLYT_THROWT_USET_INSTEADOFT_GLOBALT_VART_UNSETT_ISSETT_EMPTYT_HALT_COMPILERT_CLASST_TRAITT_INTERFACET_EXTENDST_IMPLEMENTST_OBJECT_OPERATORT_DOUBLE_ARROWT_LISTT_ARRAYT_CALLABLET_CLASS_CT_TRAIT_CT_METHOD_CT_FUNC_CT_LINET_FILET_COMMENTT_DOC_COMMENTT_OPEN_TAGT_OPEN_TAG_WITH_ECHOT_CLOSE_TAGT_WHITESPACET_START_HEREDOCT_END_HEREDOCT_DOLLAR_OPEN_CURLY_BRACEST_CURLY_OPENT_PAAMAYIM_NEKUDOTAYIMT_NAMESPACET_NS_CT_DIRT_NS_SEPARATORT_ELLIPSIST_EVALT_REQUIRET_REQUIRE_ONCET_LOGICAL_ORT_LOGICAL_XORT_LOGICAL_ANDT_INSTANCEOFT_NEWT_CLONET_ELSEIFT_ELSET_ENDIFT_PRINTT_YIELDT_STATICT_ABSTRACTT_FINALT_PRIVATET_PROTECTEDT_PUBLICT_INCT_DECT_YIELD_FROMT_INT_CASTT_DOUBLE_CASTT_STRING_CASTT_ARRAY_CASTT_OBJECT_CASTT_BOOL_CASTT_UNSET_CASTT_COALESCET_SPACESHIPT_NOELSET_PLUS_EQUALT_MINUS_EQUALT_MUL_EQUALT_POW_EQUALT_DIV_EQUALT_CONCAT_EQUALT_MOD_EQUALT_AND_EQUALT_OR_EQUALT_XOR_EQUALT_SL_EQUALT_SR_EQUALT_BOOLEAN_ORT_BOOLEAN_ANDT_POWT_SLT_SRT_IS_IDENTICALT_IS_NOT_IDENTICALT_IS_EQUALT_IS_NOT_EQUALT_IS_SMALLER_OR_EQUALT_IS_GREATER_OR_EQUAL" + +var _LexerToken_index = [...]uint16{0, 9, 23, 29, 33, 42, 51, 59, 75, 85, 97, 110, 121, 136, 161, 187, 193, 197, 204, 214, 219, 227, 236, 248, 257, 269, 273, 281, 292, 298, 307, 314, 324, 330, 340, 347, 355, 360, 367, 376, 383, 388, 399, 407, 412, 419, 426, 433, 448, 455, 462, 473, 482, 494, 511, 525, 531, 538, 548, 557, 566, 576, 584, 590, 596, 605, 618, 628, 648, 659, 671, 686, 699, 725, 737, 759, 770, 776, 781, 795, 805, 811, 820, 834, 846, 859, 872, 884, 889, 896, 904, 910, 917, 924, 931, 939, 949, 956, 965, 976, 984, 989, 994, 1006, 1016, 1029, 1042, 1054, 1067, 1078, 1090, 1100, 1111, 1119, 1131, 1144, 1155, 1166, 1177, 1191, 1202, 1213, 1223, 1234, 1244, 1254, 1266, 1279, 1284, 1288, 1292, 1306, 1324, 1334, 1348, 1369, 1390} + +func (i LexerToken) String() string { + i -= 57346 + if i < 0 || i >= LexerToken(len(_LexerToken_index)-1) { + return "LexerToken(" + strconv.FormatInt(int64(i+57346), 10) + ")" + } + return _LexerToken_name[_LexerToken_index[i]:_LexerToken_index[i+1]] +} diff --git a/scanner/scanner.go b/scanner/scanner.go index f49e966..43cd767 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -7542,7 +7542,7 @@ yyrule2: // . c = l.Next() } lval.Token(l.createToken(tb)) - return T_INLINE_HTML + return int(T_INLINE_HTML) } yyrule3: // \<\?php([ \t]|{NEW_LINE}) { @@ -7559,7 +7559,7 @@ yyrule5: // \<\?= { l.begin(PHP) lval.Token(l.createToken(l.Token())) - return T_ECHO + return int(T_ECHO) goto yystate0 } yyrule6: // [ \t\n\r]+ @@ -7584,7 +7584,7 @@ yyrule8: // \?\>{NEW_LINE}? yyrule9: // {DNUM}|{EXPONENT_DNUM} { lval.Token(l.createToken(l.Token())) - return T_DNUMBER + return int(T_DNUMBER) goto yystate0 } yyrule10: // {BNUM} @@ -7606,10 +7606,10 @@ yyrule10: // {BNUM} } if len(tb)-i < 64 { lval.Token(l.createToken(l.Token())) - return T_LNUMBER + return int(T_LNUMBER) } else { lval.Token(l.createToken(l.Token())) - return T_DNUMBER + return int(T_DNUMBER) } goto yystate0 } @@ -7618,10 +7618,10 @@ yyrule11: // {LNUM} if len(l.Token()) < 20 { lval.Token(l.createToken(l.Token())) - return T_LNUMBER + return int(T_LNUMBER) } else { lval.Token(l.createToken(l.Token())) - return T_DNUMBER + return int(T_DNUMBER) } goto yystate0 } @@ -7645,689 +7645,689 @@ yyrule12: // {HNUM} length := len(tb) - i if length < 16 || (length == 16 && tb[i].Rune <= '7') { lval.Token(l.createToken(l.Token())) - return T_LNUMBER + return int(T_LNUMBER) } else { lval.Token(l.createToken(l.Token())) - return T_DNUMBER + return int(T_DNUMBER) } goto yystate0 } yyrule13: // abstract { lval.Token(l.createToken(l.Token())) - return T_ABSTRACT + return int(T_ABSTRACT) goto yystate0 } yyrule14: // array { lval.Token(l.createToken(l.Token())) - return T_ARRAY + return int(T_ARRAY) goto yystate0 } yyrule15: // as { lval.Token(l.createToken(l.Token())) - return T_AS + return int(T_AS) goto yystate0 } yyrule16: // break { lval.Token(l.createToken(l.Token())) - return T_BREAK + return int(T_BREAK) goto yystate0 } yyrule17: // callable { lval.Token(l.createToken(l.Token())) - return T_CALLABLE + return int(T_CALLABLE) goto yystate0 } yyrule18: // case { lval.Token(l.createToken(l.Token())) - return T_CASE + return int(T_CASE) goto yystate0 } yyrule19: // catch { lval.Token(l.createToken(l.Token())) - return T_CATCH + return int(T_CATCH) goto yystate0 } yyrule20: // class { lval.Token(l.createToken(l.Token())) - return T_CLASS + return int(T_CLASS) goto yystate0 } yyrule21: // clone { lval.Token(l.createToken(l.Token())) - return T_CLONE + return int(T_CLONE) goto yystate0 } yyrule22: // const { lval.Token(l.createToken(l.Token())) - return T_CONST + return int(T_CONST) goto yystate0 } yyrule23: // continue { lval.Token(l.createToken(l.Token())) - return T_CONTINUE + return int(T_CONTINUE) goto yystate0 } yyrule24: // declare { lval.Token(l.createToken(l.Token())) - return T_DECLARE + return int(T_DECLARE) goto yystate0 } yyrule25: // default { lval.Token(l.createToken(l.Token())) - return T_DEFAULT + return int(T_DEFAULT) goto yystate0 } yyrule26: // do { lval.Token(l.createToken(l.Token())) - return T_DO + return int(T_DO) goto yystate0 } yyrule27: // echo { lval.Token(l.createToken(l.Token())) - return T_ECHO + return int(T_ECHO) goto yystate0 } yyrule28: // else { lval.Token(l.createToken(l.Token())) - return T_ELSE + return int(T_ELSE) goto yystate0 } yyrule29: // elseif { lval.Token(l.createToken(l.Token())) - return T_ELSEIF + return int(T_ELSEIF) goto yystate0 } yyrule30: // empty { lval.Token(l.createToken(l.Token())) - return T_EMPTY + return int(T_EMPTY) goto yystate0 } yyrule31: // enddeclare { lval.Token(l.createToken(l.Token())) - return T_ENDDECLARE + return int(T_ENDDECLARE) goto yystate0 } yyrule32: // endfor { lval.Token(l.createToken(l.Token())) - return T_ENDFOR + return int(T_ENDFOR) goto yystate0 } yyrule33: // endforeach { lval.Token(l.createToken(l.Token())) - return T_ENDFOREACH + return int(T_ENDFOREACH) goto yystate0 } yyrule34: // endif { lval.Token(l.createToken(l.Token())) - return T_ENDIF + return int(T_ENDIF) goto yystate0 } yyrule35: // endswitch { lval.Token(l.createToken(l.Token())) - return T_ENDSWITCH + return int(T_ENDSWITCH) goto yystate0 } yyrule36: // endwhile { lval.Token(l.createToken(l.Token())) - return T_ENDWHILE + return int(T_ENDWHILE) goto yystate0 } yyrule37: // eval { lval.Token(l.createToken(l.Token())) - return T_EVAL + return int(T_EVAL) goto yystate0 } yyrule38: // exit|die { lval.Token(l.createToken(l.Token())) - return T_EXIT + return int(T_EXIT) goto yystate0 } yyrule39: // extends { lval.Token(l.createToken(l.Token())) - return T_EXTENDS + return int(T_EXTENDS) goto yystate0 } yyrule40: // final { lval.Token(l.createToken(l.Token())) - return T_FINAL + return int(T_FINAL) goto yystate0 } yyrule41: // finally { lval.Token(l.createToken(l.Token())) - return T_FINALLY + return int(T_FINALLY) goto yystate0 } yyrule42: // for { lval.Token(l.createToken(l.Token())) - return T_FOR + return int(T_FOR) goto yystate0 } yyrule43: // foreach { lval.Token(l.createToken(l.Token())) - return T_FOREACH + return int(T_FOREACH) goto yystate0 } yyrule44: // function|cfunction { lval.Token(l.createToken(l.Token())) - return T_FUNCTION + return int(T_FUNCTION) goto yystate0 } yyrule45: // global { lval.Token(l.createToken(l.Token())) - return T_GLOBAL + return int(T_GLOBAL) goto yystate0 } yyrule46: // goto { lval.Token(l.createToken(l.Token())) - return T_GOTO + return int(T_GOTO) goto yystate0 } yyrule47: // if { lval.Token(l.createToken(l.Token())) - return T_IF + return int(T_IF) goto yystate0 } yyrule48: // isset { lval.Token(l.createToken(l.Token())) - return T_ISSET + return int(T_ISSET) goto yystate0 } yyrule49: // implements { lval.Token(l.createToken(l.Token())) - return T_IMPLEMENTS + return int(T_IMPLEMENTS) goto yystate0 } yyrule50: // instanceof { lval.Token(l.createToken(l.Token())) - return T_INSTANCEOF + return int(T_INSTANCEOF) goto yystate0 } yyrule51: // insteadof { lval.Token(l.createToken(l.Token())) - return T_INSTEADOF + return int(T_INSTEADOF) goto yystate0 } yyrule52: // interface { lval.Token(l.createToken(l.Token())) - return T_INTERFACE + return int(T_INTERFACE) goto yystate0 } yyrule53: // list { lval.Token(l.createToken(l.Token())) - return T_LIST + return int(T_LIST) goto yystate0 } yyrule54: // namespace { lval.Token(l.createToken(l.Token())) - return T_NAMESPACE + return int(T_NAMESPACE) goto yystate0 } yyrule55: // private { lval.Token(l.createToken(l.Token())) - return T_PRIVATE + return int(T_PRIVATE) goto yystate0 } yyrule56: // public { lval.Token(l.createToken(l.Token())) - return T_PUBLIC + return int(T_PUBLIC) goto yystate0 } yyrule57: // print { lval.Token(l.createToken(l.Token())) - return T_PRINT + return int(T_PRINT) goto yystate0 } yyrule58: // protected { lval.Token(l.createToken(l.Token())) - return T_PROTECTED + return int(T_PROTECTED) goto yystate0 } yyrule59: // return { lval.Token(l.createToken(l.Token())) - return T_RETURN + return int(T_RETURN) goto yystate0 } yyrule60: // static { lval.Token(l.createToken(l.Token())) - return T_STATIC + return int(T_STATIC) goto yystate0 } yyrule61: // switch { lval.Token(l.createToken(l.Token())) - return T_SWITCH + return int(T_SWITCH) goto yystate0 } yyrule62: // throw { lval.Token(l.createToken(l.Token())) - return T_THROW + return int(T_THROW) goto yystate0 } yyrule63: // trait { lval.Token(l.createToken(l.Token())) - return T_TRAIT + return int(T_TRAIT) goto yystate0 } yyrule64: // try { lval.Token(l.createToken(l.Token())) - return T_TRY + return int(T_TRY) goto yystate0 } yyrule65: // unset { lval.Token(l.createToken(l.Token())) - return T_UNSET + return int(T_UNSET) goto yystate0 } yyrule66: // use { lval.Token(l.createToken(l.Token())) - return T_USE + return int(T_USE) goto yystate0 } yyrule67: // var { lval.Token(l.createToken(l.Token())) - return T_VAR + return int(T_VAR) goto yystate0 } yyrule68: // while { lval.Token(l.createToken(l.Token())) - return T_WHILE + return int(T_WHILE) goto yystate0 } yyrule69: // yield[ \t\n\r]+from { lval.Token(l.createToken(l.Token())) - return T_YIELD_FROM + return int(T_YIELD_FROM) goto yystate0 } yyrule70: // yield { lval.Token(l.createToken(l.Token())) - return T_YIELD + return int(T_YIELD) goto yystate0 } yyrule71: // include { lval.Token(l.createToken(l.Token())) - return T_INCLUDE + return int(T_INCLUDE) goto yystate0 } yyrule72: // include_once { lval.Token(l.createToken(l.Token())) - return T_INCLUDE_ONCE + return int(T_INCLUDE_ONCE) goto yystate0 } yyrule73: // require { lval.Token(l.createToken(l.Token())) - return T_REQUIRE + return int(T_REQUIRE) goto yystate0 } yyrule74: // require_once { lval.Token(l.createToken(l.Token())) - return T_REQUIRE_ONCE + return int(T_REQUIRE_ONCE) goto yystate0 } yyrule75: // __CLASS__ { lval.Token(l.createToken(l.Token())) - return T_CLASS_C + return int(T_CLASS_C) goto yystate0 } yyrule76: // __DIR__ { lval.Token(l.createToken(l.Token())) - return T_DIR + return int(T_DIR) goto yystate0 } yyrule77: // __FILE__ { lval.Token(l.createToken(l.Token())) - return T_FILE + return int(T_FILE) goto yystate0 } yyrule78: // __FUNCTION__ { lval.Token(l.createToken(l.Token())) - return T_FUNC_C + return int(T_FUNC_C) goto yystate0 } yyrule79: // __LINE__ { lval.Token(l.createToken(l.Token())) - return T_LINE + return int(T_LINE) goto yystate0 } yyrule80: // __NAMESPACE__ { lval.Token(l.createToken(l.Token())) - return T_NS_C + return int(T_NS_C) goto yystate0 } yyrule81: // __METHOD__ { lval.Token(l.createToken(l.Token())) - return T_METHOD_C + return int(T_METHOD_C) goto yystate0 } yyrule82: // __TRAIT__ { lval.Token(l.createToken(l.Token())) - return T_TRAIT_C + return int(T_TRAIT_C) goto yystate0 } yyrule83: // __halt_compiler { lval.Token(l.createToken(l.Token())) - return T_HALT_COMPILER + return int(T_HALT_COMPILER) goto yystate0 } yyrule84: // \([ \t]*array[ \t]*\) { lval.Token(l.createToken(l.Token())) - return T_ARRAY_CAST + return int(T_ARRAY_CAST) goto yystate0 } yyrule85: // \([ \t]*(bool|boolean)[ \t]*\) { lval.Token(l.createToken(l.Token())) - return T_BOOL_CAST + return int(T_BOOL_CAST) goto yystate0 } yyrule86: // \([ \t]*(real|double|float)[ \t]*\) { lval.Token(l.createToken(l.Token())) - return T_DOUBLE_CAST + return int(T_DOUBLE_CAST) goto yystate0 } yyrule87: // \([ \t]*(int|integer)[ \t]*\) { lval.Token(l.createToken(l.Token())) - return T_INT_CAST + return int(T_INT_CAST) goto yystate0 } yyrule88: // \([ \t]*object[ \t]*\) { lval.Token(l.createToken(l.Token())) - return T_OBJECT_CAST + return int(T_OBJECT_CAST) goto yystate0 } yyrule89: // \([ \t]*(string|binary)[ \t]*\) { lval.Token(l.createToken(l.Token())) - return T_STRING_CAST + return int(T_STRING_CAST) goto yystate0 } yyrule90: // \([ \t]*unset[ \t]*\) { lval.Token(l.createToken(l.Token())) - return T_UNSET_CAST + return int(T_UNSET_CAST) goto yystate0 } yyrule91: // new { lval.Token(l.createToken(l.Token())) - return T_NEW + return int(T_NEW) goto yystate0 } yyrule92: // and { lval.Token(l.createToken(l.Token())) - return T_LOGICAL_AND + return int(T_LOGICAL_AND) goto yystate0 } yyrule93: // or { lval.Token(l.createToken(l.Token())) - return T_LOGICAL_OR + return int(T_LOGICAL_OR) goto yystate0 } yyrule94: // xor { lval.Token(l.createToken(l.Token())) - return T_LOGICAL_XOR + return int(T_LOGICAL_XOR) goto yystate0 } yyrule95: // \\ { lval.Token(l.createToken(l.Token())) - return T_NS_SEPARATOR + return int(T_NS_SEPARATOR) goto yystate0 } yyrule96: // \.\.\. { lval.Token(l.createToken(l.Token())) - return T_ELLIPSIS + return int(T_ELLIPSIS) goto yystate0 } yyrule97: // :: { lval.Token(l.createToken(l.Token())) - return T_PAAMAYIM_NEKUDOTAYIM // T_DOUBLE_COLON + return int(T_PAAMAYIM_NEKUDOTAYIM) // T_DOUBLE_COLON goto yystate0 } yyrule98: // && { lval.Token(l.createToken(l.Token())) - return T_BOOLEAN_AND + return int(T_BOOLEAN_AND) goto yystate0 } yyrule99: // \|\| { lval.Token(l.createToken(l.Token())) - return T_BOOLEAN_OR + return int(T_BOOLEAN_OR) goto yystate0 } yyrule100: // &= { lval.Token(l.createToken(l.Token())) - return T_AND_EQUAL + return int(T_AND_EQUAL) goto yystate0 } yyrule101: // \|= { lval.Token(l.createToken(l.Token())) - return T_OR_EQUAL + return int(T_OR_EQUAL) goto yystate0 } yyrule102: // \.= { lval.Token(l.createToken(l.Token())) - return T_CONCAT_EQUAL + return int(T_CONCAT_EQUAL) goto yystate0 } yyrule103: // \*= { lval.Token(l.createToken(l.Token())) - return T_MUL_EQUAL + return int(T_MUL_EQUAL) goto yystate0 } yyrule104: // \*\*= { lval.Token(l.createToken(l.Token())) - return T_POW_EQUAL + return int(T_POW_EQUAL) goto yystate0 } yyrule105: // [/]= { lval.Token(l.createToken(l.Token())) - return T_DIV_EQUAL + return int(T_DIV_EQUAL) goto yystate0 } yyrule106: // \+= { lval.Token(l.createToken(l.Token())) - return T_PLUS_EQUAL + return int(T_PLUS_EQUAL) goto yystate0 } yyrule107: // -= { lval.Token(l.createToken(l.Token())) - return T_MINUS_EQUAL + return int(T_MINUS_EQUAL) goto yystate0 } yyrule108: // \^= { lval.Token(l.createToken(l.Token())) - return T_XOR_EQUAL + return int(T_XOR_EQUAL) goto yystate0 } yyrule109: // %= { lval.Token(l.createToken(l.Token())) - return T_MOD_EQUAL + return int(T_MOD_EQUAL) goto yystate0 } yyrule110: // -- { lval.Token(l.createToken(l.Token())) - return T_DEC + return int(T_DEC) goto yystate0 } yyrule111: // \+\+ { lval.Token(l.createToken(l.Token())) - return T_INC + return int(T_INC) goto yystate0 } yyrule112: // => { lval.Token(l.createToken(l.Token())) - return T_DOUBLE_ARROW + return int(T_DOUBLE_ARROW) goto yystate0 } yyrule113: // \<=\> { lval.Token(l.createToken(l.Token())) - return T_SPACESHIP + return int(T_SPACESHIP) goto yystate0 } yyrule114: // \!=|\<\> { lval.Token(l.createToken(l.Token())) - return T_IS_NOT_EQUAL + return int(T_IS_NOT_EQUAL) goto yystate0 } yyrule115: // \!== { lval.Token(l.createToken(l.Token())) - return T_IS_NOT_IDENTICAL + return int(T_IS_NOT_IDENTICAL) goto yystate0 } yyrule116: // == { lval.Token(l.createToken(l.Token())) - return T_IS_EQUAL + return int(T_IS_EQUAL) goto yystate0 } yyrule117: // === { lval.Token(l.createToken(l.Token())) - return T_IS_IDENTICAL + return int(T_IS_IDENTICAL) goto yystate0 } yyrule118: // \<\<= { lval.Token(l.createToken(l.Token())) - return T_SL_EQUAL + return int(T_SL_EQUAL) goto yystate0 } yyrule119: // \>\>= { lval.Token(l.createToken(l.Token())) - return T_SR_EQUAL + return int(T_SR_EQUAL) goto yystate0 } yyrule120: // \>= { lval.Token(l.createToken(l.Token())) - return T_IS_GREATER_OR_EQUAL + return int(T_IS_GREATER_OR_EQUAL) goto yystate0 } yyrule121: // \<= { lval.Token(l.createToken(l.Token())) - return T_IS_SMALLER_OR_EQUAL + return int(T_IS_SMALLER_OR_EQUAL) goto yystate0 } yyrule122: // \*\* { lval.Token(l.createToken(l.Token())) - return T_POW + return int(T_POW) goto yystate0 } yyrule123: // \<\< { lval.Token(l.createToken(l.Token())) - return T_SL + return int(T_SL) goto yystate0 } yyrule124: // \>\> { lval.Token(l.createToken(l.Token())) - return T_SR + return int(T_SR) goto yystate0 } yyrule125: // \?\? { lval.Token(l.createToken(l.Token())) - return T_COALESCE + return int(T_COALESCE) goto yystate0 } yyrule126: // (#|[/][/]) @@ -8416,20 +8416,20 @@ yyrule130: // \} yyrule131: // \${VAR_NAME} { lval.Token(l.createToken(l.Token())) - return T_VARIABLE + return int(T_VARIABLE) goto yystate0 } yyrule132: // {VAR_NAME} { lval.Token(l.createToken(l.Token())) - return T_STRING + return int(T_STRING) goto yystate0 } yyrule133: // -> { l.begin(PROPERTY) lval.Token(l.createToken(l.Token())) - return T_OBJECT_OPERATOR + return int(T_OBJECT_OPERATOR) goto yystate0 } yyrule134: // [ \t\n\r]+ @@ -8440,14 +8440,14 @@ yyrule134: // [ \t\n\r]+ yyrule135: // -> { lval.Token(l.createToken(l.Token())) - return T_OBJECT_OPERATOR + return int(T_OBJECT_OPERATOR) goto yystate0 } yyrule136: // {VAR_NAME} { l.begin(PHP) lval.Token(l.createToken(l.Token())) - return T_STRING + return int(T_STRING) goto yystate0 } yyrule137: // . @@ -8459,7 +8459,7 @@ yyrule137: // . yyrule138: // [\']([^\\\']*([\\].)*)*[\'] { lval.Token(l.createToken(l.Token())) - return T_CONSTANT_ENCAPSED_STRING + return int(T_CONSTANT_ENCAPSED_STRING) goto yystate0 } yyrule139: // ` @@ -8531,7 +8531,7 @@ yyrule141: // [b]?\<\<\<[ \t]*({VAR_NAME}|([']{VAR_NAME}['])|(["]{VAR_NAME}["])) } l.ungetChars(ungetCnt) lval.Token(l.createToken(heredocToken)) - return T_START_HEREDOC + return int(T_START_HEREDOC) } yyrule142: // .|[ \t\n\r] { @@ -8547,11 +8547,13 @@ yyrule142: // .|[ \t\n\r] if l.heredocLabel+";" == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel) + 1) + tb = tb[:len(tb)-1] break } if l.heredocLabel == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel)) + tb = tb[:len(tb)-1] break } @@ -8562,20 +8564,20 @@ yyrule142: // .|[ \t\n\r] c = l.Next() } lval.Token(l.createToken(tb)) - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } yyrule143: // {VAR_NAME}\; { l.begin(PHP) lval.Token(l.createToken(l.ungetChars(1))) - return T_END_HEREDOC + return int(T_END_HEREDOC) goto yystate0 } yyrule144: // {VAR_NAME} { l.begin(PHP) lval.Token(l.createToken(l.Token())) - return T_END_HEREDOC + return int(T_END_HEREDOC) goto yystate0 } yyrule145: // [b]?[\"] @@ -8603,7 +8605,7 @@ yyrule145: // [b]?[\"] case '"': c = l.Next() lval.Token(l.createToken(l.Token())) - return T_CONSTANT_ENCAPSED_STRING + return int(T_CONSTANT_ENCAPSED_STRING) break F case '$': @@ -8639,14 +8641,14 @@ yyrule147: // \{\$ { lval.Token(l.createToken(l.ungetChars(1))) l.pushState(PHP) - return T_CURLY_OPEN + return int(T_CURLY_OPEN) goto yystate0 } yyrule148: // \$\{ { l.pushState(STRING_VAR_NAME) lval.Token(l.createToken(l.Token())) - return T_DOLLAR_OPEN_CURLY_BRACES + return int(T_DOLLAR_OPEN_CURLY_BRACES) goto yystate0 } yyrule149: // \${VAR_NAME} @@ -8666,13 +8668,13 @@ yyrule150: // .|[ \t\n\r] if c == '{' || isValidFirstVarNameRune(rune(c)) { l.ungetChars(1) lval.Token(l.createToken(tb[:len(tb)-1])) - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } case '{': if rune(c) == '$' { l.ungetChars(1) lval.Token(l.createToken(tb[:len(tb)-1])) - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } case '\\': currentChar := l.Last @@ -8681,7 +8683,7 @@ yyrule150: // .|[ \t\n\r] } if rune(c) == '"' { lval.Token(l.createToken(l.Token())) - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } currentChar = l.Last tb = append(tb, currentChar) @@ -8703,13 +8705,13 @@ yyrule151: // .|[ \t\n\r] if c == '{' || isValidFirstVarNameRune(rune(c)) { l.ungetChars(1) lval.Token(l.createToken(tb[:len(tb)-1])) - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } case '{': if rune(c) == '$' { l.ungetChars(1) lval.Token(l.createToken(tb[:len(tb)-1])) - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } case '\\': currentChar := l.Last @@ -8718,7 +8720,7 @@ yyrule151: // .|[ \t\n\r] } if rune(c) == '`' { lval.Token(l.createToken(l.Token())) - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } currentChar = l.Last tb = append(tb, currentChar) @@ -8733,64 +8735,77 @@ yyrule152: // .|[ \t\n\r] { searchLabel := []byte{} - tb := []lex.Char{} + currentChar := l.Prev + tb := []lex.Char{currentChar} + HEREDOC_FOR: for { - if c == -1 { - break - } nls := 0 - switch c { + switch currentChar.Rune { case '\r': - nls = 1 - c := l.Next() - if c != '\n' { - nls = 0 - l.ungetChars(0) + if c == '\n' { + nls = 1 + currentChar := l.Last + tb = append(tb, currentChar) + c = l.Next() } fallthrough case '\n': if l.heredocLabel+";" == string(searchLabel) { l.begin(HEREDOC_END) - tb = l.ungetChars(len(l.heredocLabel) + 1 + nls) + l.ungetChars(len(l.heredocLabel) + 1 + nls) + + i := len(tb) - len(l.heredocLabel) - 3 - nls + if i < 1 { + break HEREDOC_FOR + } + tb = tb[:i] + lval.Token(l.createToken(tb)) - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } if l.heredocLabel == string(searchLabel) { l.begin(HEREDOC_END) - tb = l.ungetChars(len(l.heredocLabel) + nls) + l.ungetChars(len(l.heredocLabel) + nls) + + i := len(tb) - len(l.heredocLabel) - 2 - nls + if i < 1 { + break HEREDOC_FOR + } + tb = tb[:i] lval.Token(l.createToken(tb)) - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } searchLabel = []byte{} - case '$': - c = l.Next() - if rune(c) == '{' || isValidFirstVarNameRune(rune(c)) { - tb = l.ungetChars(1) - lval.Token(l.createToken(tb)) - return T_ENCAPSED_AND_WHITESPACE + if c == '{' || isValidFirstVarNameRune(rune(c)) { + l.ungetChars(1) + lval.Token(l.createToken(tb[:len(tb)-1])) + return int(T_ENCAPSED_AND_WHITESPACE) } - l.ungetChars(0) - case '{': - c = l.Next() if rune(c) == '$' { - tb = l.ungetChars(1) - lval.Token(l.createToken(tb)) - return T_ENCAPSED_AND_WHITESPACE + l.ungetChars(1) + lval.Token(l.createToken(tb[:len(tb)-1])) + return int(T_ENCAPSED_AND_WHITESPACE) } - l.ungetChars(0) case '\\': - c = l.Next() - if c == '\n' || c == '\r' { - l.ungetChars(0) + if c != '\n' && c != '\r' { + currentChar := l.Last + tb = append(tb, currentChar) + c = l.Next() } default: - searchLabel = append(searchLabel, byte(rune(c))) + searchLabel = append(searchLabel, byte(rune(currentChar.Rune))) } + if c == -1 { + break + } + + currentChar = l.Last + tb = append(tb, currentChar) c = l.Next() } goto yystate0 @@ -8798,20 +8813,20 @@ yyrule152: // .|[ \t\n\r] yyrule153: // \${VAR_NAME} { lval.Token(l.createToken(l.Token())) - return T_VARIABLE + return int(T_VARIABLE) goto yystate0 } yyrule154: // ->{VAR_NAME} { lval.Token(l.createToken(l.ungetChars(len(l.Token()) - 2))) - return T_OBJECT_OPERATOR + return int(T_OBJECT_OPERATOR) goto yystate0 } yyrule155: // {VAR_NAME} { l.popState() lval.Token(l.createToken(l.Token())) - return T_STRING + return int(T_STRING) goto yystate0 } yyrule156: // \[ @@ -8830,19 +8845,19 @@ yyrule157: // .|[ \t\n\r] yyrule158: // {LNUM}|{HNUM}|{BNUM} { lval.Token(l.createToken(l.Token())) - return T_NUM_STRING + return int(T_NUM_STRING) goto yystate0 } yyrule159: // \${VAR_NAME} { lval.Token(l.createToken(l.Token())) - return T_VARIABLE + return int(T_VARIABLE) goto yystate0 } yyrule160: // {VAR_NAME} { lval.Token(l.createToken(l.Token())) - return T_STRING + return int(T_STRING) goto yystate0 } yyrule161: // \] @@ -8858,7 +8873,7 @@ yyrule162: // [ \n\r\t\\'#] l.popState() l.popState() lval.Token(l.createToken(l.Token())) - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) goto yystate0 } yyrule163: // {OPERATORS} @@ -8878,7 +8893,7 @@ yyrule165: // {VAR_NAME}[\[\}] l.popState() l.pushState(PHP) lval.Token(l.createToken(l.ungetChars(1))) - return T_STRING_VARNAME + return int(T_STRING_VARNAME) goto yystate0 } yyrule166: // . diff --git a/scanner/scanner.l b/scanner/scanner.l index 53fc614..0d81b83 100644 --- a/scanner/scanner.l +++ b/scanner/scanner.l @@ -82,17 +82,17 @@ NEW_LINE (\r|\n|\r\n) } lval.Token(l.createToken(tb)) - return T_INLINE_HTML + return int(T_INLINE_HTML) \<\?php([ \t]|{NEW_LINE}) l.begin(PHP);l.ungetChars(len(l.Token())-5) \<\? l.begin(PHP); -\<\?= l.begin(PHP);lval.Token(l.createToken(l.Token())); return T_ECHO; +\<\?= l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_ECHO); [ \t\n\r]+ l.addWhiteSpace(l.Token()) [;][ \t\n\r]*\?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';'); \?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';'); -{DNUM}|{EXPONENT_DNUM} lval.Token(l.createToken(l.Token())); return T_DNUMBER +{DNUM}|{EXPONENT_DNUM} lval.Token(l.createToken(l.Token())); return int(T_DNUMBER) {BNUM} tb := l.Token() i:=2 @@ -106,15 +106,15 @@ NEW_LINE (\r|\n|\r\n) } } if len(tb) - i < 64 { - lval.Token(l.createToken(l.Token())); return T_LNUMBER + lval.Token(l.createToken(l.Token())); return int(T_LNUMBER) } else { - lval.Token(l.createToken(l.Token())); return T_DNUMBER + lval.Token(l.createToken(l.Token())); return int(T_DNUMBER) } {LNUM} if len(l.Token()) < 20 { - lval.Token(l.createToken(l.Token())); return T_LNUMBER + lval.Token(l.createToken(l.Token())); return int(T_LNUMBER) } else { - lval.Token(l.createToken(l.Token())); return T_DNUMBER + lval.Token(l.createToken(l.Token())); return int(T_DNUMBER) } {HNUM} tb := l.Token() @@ -130,124 +130,124 @@ NEW_LINE (\r|\n|\r\n) } length := len(tb) - i if length < 16 || (length == 16 && tb[i].Rune <= '7') { - lval.Token(l.createToken(l.Token())); return T_LNUMBER + lval.Token(l.createToken(l.Token())); return int(T_LNUMBER) } else { - lval.Token(l.createToken(l.Token())); return T_DNUMBER + lval.Token(l.createToken(l.Token())); return int(T_DNUMBER) } -abstract lval.Token(l.createToken(l.Token())); return T_ABSTRACT -array lval.Token(l.createToken(l.Token())); return T_ARRAY -as lval.Token(l.createToken(l.Token())); return T_AS -break lval.Token(l.createToken(l.Token())); return T_BREAK -callable lval.Token(l.createToken(l.Token())); return T_CALLABLE -case lval.Token(l.createToken(l.Token())); return T_CASE -catch lval.Token(l.createToken(l.Token())); return T_CATCH -class lval.Token(l.createToken(l.Token())); return T_CLASS -clone lval.Token(l.createToken(l.Token())); return T_CLONE -const lval.Token(l.createToken(l.Token())); return T_CONST -continue lval.Token(l.createToken(l.Token())); return T_CONTINUE -declare lval.Token(l.createToken(l.Token())); return T_DECLARE -default lval.Token(l.createToken(l.Token())); return T_DEFAULT -do lval.Token(l.createToken(l.Token())); return T_DO -echo lval.Token(l.createToken(l.Token())); return T_ECHO -else lval.Token(l.createToken(l.Token())); return T_ELSE -elseif lval.Token(l.createToken(l.Token())); return T_ELSEIF -empty lval.Token(l.createToken(l.Token())); return T_EMPTY -enddeclare lval.Token(l.createToken(l.Token())); return T_ENDDECLARE -endfor lval.Token(l.createToken(l.Token())); return T_ENDFOR -endforeach lval.Token(l.createToken(l.Token())); return T_ENDFOREACH -endif lval.Token(l.createToken(l.Token())); return T_ENDIF -endswitch lval.Token(l.createToken(l.Token())); return T_ENDSWITCH -endwhile lval.Token(l.createToken(l.Token())); return T_ENDWHILE -eval lval.Token(l.createToken(l.Token())); return T_EVAL -exit|die lval.Token(l.createToken(l.Token())); return T_EXIT -extends lval.Token(l.createToken(l.Token())); return T_EXTENDS -final lval.Token(l.createToken(l.Token())); return T_FINAL -finally lval.Token(l.createToken(l.Token())); return T_FINALLY -for lval.Token(l.createToken(l.Token())); return T_FOR -foreach lval.Token(l.createToken(l.Token())); return T_FOREACH -function|cfunction lval.Token(l.createToken(l.Token())); return T_FUNCTION -global lval.Token(l.createToken(l.Token())); return T_GLOBAL -goto lval.Token(l.createToken(l.Token())); return T_GOTO -if lval.Token(l.createToken(l.Token())); return T_IF -isset lval.Token(l.createToken(l.Token())); return T_ISSET -implements lval.Token(l.createToken(l.Token())); return T_IMPLEMENTS -instanceof lval.Token(l.createToken(l.Token())); return T_INSTANCEOF -insteadof lval.Token(l.createToken(l.Token())); return T_INSTEADOF -interface lval.Token(l.createToken(l.Token())); return T_INTERFACE -list lval.Token(l.createToken(l.Token())); return T_LIST -namespace lval.Token(l.createToken(l.Token())); return T_NAMESPACE -private lval.Token(l.createToken(l.Token())); return T_PRIVATE -public lval.Token(l.createToken(l.Token())); return T_PUBLIC -print lval.Token(l.createToken(l.Token())); return T_PRINT -protected lval.Token(l.createToken(l.Token())); return T_PROTECTED -return lval.Token(l.createToken(l.Token())); return T_RETURN -static lval.Token(l.createToken(l.Token())); return T_STATIC -switch lval.Token(l.createToken(l.Token())); return T_SWITCH -throw lval.Token(l.createToken(l.Token())); return T_THROW -trait lval.Token(l.createToken(l.Token())); return T_TRAIT -try lval.Token(l.createToken(l.Token())); return T_TRY -unset lval.Token(l.createToken(l.Token())); return T_UNSET -use lval.Token(l.createToken(l.Token())); return T_USE -var lval.Token(l.createToken(l.Token())); return T_VAR -while lval.Token(l.createToken(l.Token())); return T_WHILE -yield[ \t\n\r]+from lval.Token(l.createToken(l.Token())); return T_YIELD_FROM -yield lval.Token(l.createToken(l.Token())); return T_YIELD -include lval.Token(l.createToken(l.Token())); return T_INCLUDE -include_once lval.Token(l.createToken(l.Token())); return T_INCLUDE_ONCE -require lval.Token(l.createToken(l.Token())); return T_REQUIRE -require_once lval.Token(l.createToken(l.Token())); return T_REQUIRE_ONCE -__CLASS__ lval.Token(l.createToken(l.Token())); return T_CLASS_C -__DIR__ lval.Token(l.createToken(l.Token())); return T_DIR -__FILE__ lval.Token(l.createToken(l.Token())); return T_FILE -__FUNCTION__ lval.Token(l.createToken(l.Token())); return T_FUNC_C -__LINE__ lval.Token(l.createToken(l.Token())); return T_LINE -__NAMESPACE__ lval.Token(l.createToken(l.Token())); return T_NS_C -__METHOD__ lval.Token(l.createToken(l.Token())); return T_METHOD_C -__TRAIT__ lval.Token(l.createToken(l.Token())); return T_TRAIT_C -__halt_compiler lval.Token(l.createToken(l.Token())); return T_HALT_COMPILER -\([ \t]*array[ \t]*\) lval.Token(l.createToken(l.Token())); return T_ARRAY_CAST -\([ \t]*(bool|boolean)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_BOOL_CAST -\([ \t]*(real|double|float)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_DOUBLE_CAST -\([ \t]*(int|integer)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_INT_CAST -\([ \t]*object[ \t]*\) lval.Token(l.createToken(l.Token())); return T_OBJECT_CAST -\([ \t]*(string|binary)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_STRING_CAST -\([ \t]*unset[ \t]*\) lval.Token(l.createToken(l.Token())); return T_UNSET_CAST -new lval.Token(l.createToken(l.Token())); return T_NEW -and lval.Token(l.createToken(l.Token())); return T_LOGICAL_AND -or lval.Token(l.createToken(l.Token())); return T_LOGICAL_OR -xor lval.Token(l.createToken(l.Token())); return T_LOGICAL_XOR -\\ lval.Token(l.createToken(l.Token())); return T_NS_SEPARATOR -\.\.\. lval.Token(l.createToken(l.Token())); return T_ELLIPSIS -:: lval.Token(l.createToken(l.Token())); return T_PAAMAYIM_NEKUDOTAYIM // T_DOUBLE_COLON -&& lval.Token(l.createToken(l.Token())); return T_BOOLEAN_AND -\|\| lval.Token(l.createToken(l.Token())); return T_BOOLEAN_OR -&= lval.Token(l.createToken(l.Token())); return T_AND_EQUAL -\|= lval.Token(l.createToken(l.Token())); return T_OR_EQUAL -\.= lval.Token(l.createToken(l.Token())); return T_CONCAT_EQUAL -\*= lval.Token(l.createToken(l.Token())); return T_MUL_EQUAL -\*\*= lval.Token(l.createToken(l.Token())); return T_POW_EQUAL -[/]= lval.Token(l.createToken(l.Token())); return T_DIV_EQUAL -\+= lval.Token(l.createToken(l.Token())); return T_PLUS_EQUAL --= lval.Token(l.createToken(l.Token())); return T_MINUS_EQUAL -\^= lval.Token(l.createToken(l.Token())); return T_XOR_EQUAL -%= lval.Token(l.createToken(l.Token())); return T_MOD_EQUAL --- lval.Token(l.createToken(l.Token())); return T_DEC -\+\+ lval.Token(l.createToken(l.Token())); return T_INC -=> lval.Token(l.createToken(l.Token())); return T_DOUBLE_ARROW -\<=\> lval.Token(l.createToken(l.Token())); return T_SPACESHIP -\!=|\<\> lval.Token(l.createToken(l.Token())); return T_IS_NOT_EQUAL -\!== lval.Token(l.createToken(l.Token())); return T_IS_NOT_IDENTICAL -== lval.Token(l.createToken(l.Token())); return T_IS_EQUAL -=== lval.Token(l.createToken(l.Token())); return T_IS_IDENTICAL -\<\<= lval.Token(l.createToken(l.Token())); return T_SL_EQUAL -\>\>= lval.Token(l.createToken(l.Token())); return T_SR_EQUAL -\>= lval.Token(l.createToken(l.Token())); return T_IS_GREATER_OR_EQUAL -\<= lval.Token(l.createToken(l.Token())); return T_IS_SMALLER_OR_EQUAL -\*\* lval.Token(l.createToken(l.Token())); return T_POW -\<\< lval.Token(l.createToken(l.Token())); return T_SL -\>\> lval.Token(l.createToken(l.Token())); return T_SR -\?\? lval.Token(l.createToken(l.Token())); return T_COALESCE +abstract lval.Token(l.createToken(l.Token())); return int(T_ABSTRACT) +array lval.Token(l.createToken(l.Token())); return int(T_ARRAY) +as lval.Token(l.createToken(l.Token())); return int(T_AS) +break lval.Token(l.createToken(l.Token())); return int(T_BREAK) +callable lval.Token(l.createToken(l.Token())); return int(T_CALLABLE) +case lval.Token(l.createToken(l.Token())); return int(T_CASE) +catch lval.Token(l.createToken(l.Token())); return int(T_CATCH) +class lval.Token(l.createToken(l.Token())); return int(T_CLASS) +clone lval.Token(l.createToken(l.Token())); return int(T_CLONE) +const lval.Token(l.createToken(l.Token())); return int(T_CONST) +continue lval.Token(l.createToken(l.Token())); return int(T_CONTINUE) +declare lval.Token(l.createToken(l.Token())); return int(T_DECLARE) +default lval.Token(l.createToken(l.Token())); return int(T_DEFAULT) +do lval.Token(l.createToken(l.Token())); return int(T_DO) +echo lval.Token(l.createToken(l.Token())); return int(T_ECHO) +else lval.Token(l.createToken(l.Token())); return int(T_ELSE) +elseif lval.Token(l.createToken(l.Token())); return int(T_ELSEIF) +empty lval.Token(l.createToken(l.Token())); return int(T_EMPTY) +enddeclare lval.Token(l.createToken(l.Token())); return int(T_ENDDECLARE) +endfor lval.Token(l.createToken(l.Token())); return int(T_ENDFOR) +endforeach lval.Token(l.createToken(l.Token())); return int(T_ENDFOREACH) +endif lval.Token(l.createToken(l.Token())); return int(T_ENDIF) +endswitch lval.Token(l.createToken(l.Token())); return int(T_ENDSWITCH) +endwhile lval.Token(l.createToken(l.Token())); return int(T_ENDWHILE) +eval lval.Token(l.createToken(l.Token())); return int(T_EVAL) +exit|die lval.Token(l.createToken(l.Token())); return int(T_EXIT) +extends lval.Token(l.createToken(l.Token())); return int(T_EXTENDS) +final lval.Token(l.createToken(l.Token())); return int(T_FINAL) +finally lval.Token(l.createToken(l.Token())); return int(T_FINALLY) +for lval.Token(l.createToken(l.Token())); return int(T_FOR) +foreach lval.Token(l.createToken(l.Token())); return int(T_FOREACH) +function|cfunction lval.Token(l.createToken(l.Token())); return int(T_FUNCTION) +global lval.Token(l.createToken(l.Token())); return int(T_GLOBAL) +goto lval.Token(l.createToken(l.Token())); return int(T_GOTO) +if lval.Token(l.createToken(l.Token())); return int(T_IF) +isset lval.Token(l.createToken(l.Token())); return int(T_ISSET) +implements lval.Token(l.createToken(l.Token())); return int(T_IMPLEMENTS) +instanceof lval.Token(l.createToken(l.Token())); return int(T_INSTANCEOF) +insteadof lval.Token(l.createToken(l.Token())); return int(T_INSTEADOF) +interface lval.Token(l.createToken(l.Token())); return int(T_INTERFACE) +list lval.Token(l.createToken(l.Token())); return int(T_LIST) +namespace lval.Token(l.createToken(l.Token())); return int(T_NAMESPACE) +private lval.Token(l.createToken(l.Token())); return int(T_PRIVATE) +public lval.Token(l.createToken(l.Token())); return int(T_PUBLIC) +print lval.Token(l.createToken(l.Token())); return int(T_PRINT) +protected lval.Token(l.createToken(l.Token())); return int(T_PROTECTED) +return lval.Token(l.createToken(l.Token())); return int(T_RETURN) +static lval.Token(l.createToken(l.Token())); return int(T_STATIC) +switch lval.Token(l.createToken(l.Token())); return int(T_SWITCH) +throw lval.Token(l.createToken(l.Token())); return int(T_THROW) +trait lval.Token(l.createToken(l.Token())); return int(T_TRAIT) +try lval.Token(l.createToken(l.Token())); return int(T_TRY) +unset lval.Token(l.createToken(l.Token())); return int(T_UNSET) +use lval.Token(l.createToken(l.Token())); return int(T_USE) +var lval.Token(l.createToken(l.Token())); return int(T_VAR) +while lval.Token(l.createToken(l.Token())); return int(T_WHILE) +yield[ \t\n\r]+from lval.Token(l.createToken(l.Token())); return int(T_YIELD_FROM) +yield lval.Token(l.createToken(l.Token())); return int(T_YIELD) +include lval.Token(l.createToken(l.Token())); return int(T_INCLUDE) +include_once lval.Token(l.createToken(l.Token())); return int(T_INCLUDE_ONCE) +require lval.Token(l.createToken(l.Token())); return int(T_REQUIRE) +require_once lval.Token(l.createToken(l.Token())); return int(T_REQUIRE_ONCE) +__CLASS__ lval.Token(l.createToken(l.Token())); return int(T_CLASS_C) +__DIR__ lval.Token(l.createToken(l.Token())); return int(T_DIR) +__FILE__ lval.Token(l.createToken(l.Token())); return int(T_FILE) +__FUNCTION__ lval.Token(l.createToken(l.Token())); return int(T_FUNC_C) +__LINE__ lval.Token(l.createToken(l.Token())); return int(T_LINE) +__NAMESPACE__ lval.Token(l.createToken(l.Token())); return int(T_NS_C) +__METHOD__ lval.Token(l.createToken(l.Token())); return int(T_METHOD_C) +__TRAIT__ lval.Token(l.createToken(l.Token())); return int(T_TRAIT_C) +__halt_compiler lval.Token(l.createToken(l.Token())); return int(T_HALT_COMPILER) +\([ \t]*array[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_ARRAY_CAST) +\([ \t]*(bool|boolean)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_BOOL_CAST) +\([ \t]*(real|double|float)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_DOUBLE_CAST) +\([ \t]*(int|integer)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_INT_CAST) +\([ \t]*object[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_OBJECT_CAST) +\([ \t]*(string|binary)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_STRING_CAST) +\([ \t]*unset[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_UNSET_CAST) +new lval.Token(l.createToken(l.Token())); return int(T_NEW) +and lval.Token(l.createToken(l.Token())); return int(T_LOGICAL_AND) +or lval.Token(l.createToken(l.Token())); return int(T_LOGICAL_OR) +xor lval.Token(l.createToken(l.Token())); return int(T_LOGICAL_XOR) +\\ lval.Token(l.createToken(l.Token())); return int(T_NS_SEPARATOR) +\.\.\. lval.Token(l.createToken(l.Token())); return int(T_ELLIPSIS) +:: lval.Token(l.createToken(l.Token())); return int(T_PAAMAYIM_NEKUDOTAYIM) // T_DOUBLE_COLON +&& lval.Token(l.createToken(l.Token())); return int(T_BOOLEAN_AND) +\|\| lval.Token(l.createToken(l.Token())); return int(T_BOOLEAN_OR) +&= lval.Token(l.createToken(l.Token())); return int(T_AND_EQUAL) +\|= lval.Token(l.createToken(l.Token())); return int(T_OR_EQUAL) +\.= lval.Token(l.createToken(l.Token())); return int(T_CONCAT_EQUAL) +\*= lval.Token(l.createToken(l.Token())); return int(T_MUL_EQUAL) +\*\*= lval.Token(l.createToken(l.Token())); return int(T_POW_EQUAL) +[/]= lval.Token(l.createToken(l.Token())); return int(T_DIV_EQUAL) +\+= lval.Token(l.createToken(l.Token())); return int(T_PLUS_EQUAL) +-= lval.Token(l.createToken(l.Token())); return int(T_MINUS_EQUAL) +\^= lval.Token(l.createToken(l.Token())); return int(T_XOR_EQUAL) +%= lval.Token(l.createToken(l.Token())); return int(T_MOD_EQUAL) +-- lval.Token(l.createToken(l.Token())); return int(T_DEC) +\+\+ lval.Token(l.createToken(l.Token())); return int(T_INC) +=> lval.Token(l.createToken(l.Token())); return int(T_DOUBLE_ARROW) +\<=\> lval.Token(l.createToken(l.Token())); return int(T_SPACESHIP) +\!=|\<\> lval.Token(l.createToken(l.Token())); return int(T_IS_NOT_EQUAL) +\!== lval.Token(l.createToken(l.Token())); return int(T_IS_NOT_IDENTICAL) +== lval.Token(l.createToken(l.Token())); return int(T_IS_EQUAL) +=== lval.Token(l.createToken(l.Token())); return int(T_IS_IDENTICAL) +\<\<= lval.Token(l.createToken(l.Token())); return int(T_SL_EQUAL) +\>\>= lval.Token(l.createToken(l.Token())); return int(T_SR_EQUAL) +\>= lval.Token(l.createToken(l.Token())); return int(T_IS_GREATER_OR_EQUAL) +\<= lval.Token(l.createToken(l.Token())); return int(T_IS_SMALLER_OR_EQUAL) +\*\* lval.Token(l.createToken(l.Token())); return int(T_POW) +\<\< lval.Token(l.createToken(l.Token())); return int(T_SL) +\>\> lval.Token(l.createToken(l.Token())); return int(T_SR) +\?\? lval.Token(l.createToken(l.Token())); return int(T_COALESCE) (#|[/][/]) tb := l.Token() @@ -319,16 +319,16 @@ NEW_LINE (\r|\n|\r\n) \{ l.pushState(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) \} l.popState(); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])); l.PhpDocComment = "" -\${VAR_NAME} lval.Token(l.createToken(l.Token())); return T_VARIABLE -{VAR_NAME} lval.Token(l.createToken(l.Token())); return T_STRING +\${VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_VARIABLE) +{VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_STRING) --> l.begin(PROPERTY);lval.Token(l.createToken(l.Token())); return T_OBJECT_OPERATOR; +-> l.begin(PROPERTY);lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR); [ \t\n\r]+ l.addWhiteSpace(l.Token()) --> lval.Token(l.createToken(l.Token())); return T_OBJECT_OPERATOR; -{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return T_STRING; +-> lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR); +{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_STRING); . l.ungetChars(1);l.begin(PHP) -[\']([^\\\']*([\\].)*)*[\'] lval.Token(l.createToken(l.Token())); return T_CONSTANT_ENCAPSED_STRING; +[\']([^\\\']*([\\].)*)*[\'] lval.Token(l.createToken(l.Token())); return int(T_CONSTANT_ENCAPSED_STRING); ` l.begin(BACKQUOTE); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) ` l.begin(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) @@ -394,7 +394,7 @@ NEW_LINE (\r|\n|\r\n) l.ungetChars(ungetCnt) lval.Token(l.createToken(heredocToken)); - return T_START_HEREDOC + return int(T_START_HEREDOC) .|[ \t\n\r] searchLabel := []byte{} @@ -409,12 +409,14 @@ NEW_LINE (\r|\n|\r\n) if l.heredocLabel + ";" == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel)+1) + tb = tb[:len(tb)-1] break; } if l.heredocLabel == string(searchLabel) { l.begin(HEREDOC_END) tb = l.ungetChars(len(l.heredocLabel)) + tb = tb[:len(tb)-1] break; } @@ -427,10 +429,10 @@ NEW_LINE (\r|\n|\r\n) } lval.Token(l.createToken(tb) ) - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) -{VAR_NAME}\; l.begin(PHP);lval.Token(l.createToken(l.ungetChars(1))); return T_END_HEREDOC -{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return T_END_HEREDOC +{VAR_NAME}\; l.begin(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_END_HEREDOC) +{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_END_HEREDOC) [b]?[\"] binPrefix := l.Token()[0].Rune == 'b' @@ -453,7 +455,7 @@ NEW_LINE (\r|\n|\r\n) switch c { case '"' : c = l.Next(); - lval.Token(l.createToken(l.Token())); return T_CONSTANT_ENCAPSED_STRING + lval.Token(l.createToken(l.Token())); return int(T_CONSTANT_ENCAPSED_STRING) break F; case '$': @@ -480,8 +482,8 @@ NEW_LINE (\r|\n|\r\n) } \" l.popState(); lval.Token(l.createToken(l.Token())); return Rune2Class(l.Token()[0].Rune) -\{\$ lval.Token(l.createToken(l.ungetChars(1))); l.pushState(PHP); return T_CURLY_OPEN -\$\{ l.pushState(STRING_VAR_NAME); lval.Token(l.createToken(l.Token())); return T_DOLLAR_OPEN_CURLY_BRACES +\{\$ lval.Token(l.createToken(l.ungetChars(1))); l.pushState(PHP); return int(T_CURLY_OPEN) +\$\{ l.pushState(STRING_VAR_NAME); lval.Token(l.createToken(l.Token())); return int(T_DOLLAR_OPEN_CURLY_BRACES) \${VAR_NAME} l.ungetChars(len(l.Token()));l.pushState(STRING_VAR) .|[ \t\n\r] currentChar := l.Prev @@ -492,14 +494,14 @@ NEW_LINE (\r|\n|\r\n) if c == '{' || isValidFirstVarNameRune(rune(c)) { l.ungetChars(1) lval.Token(l.createToken(tb[:len(tb)-1])); - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } case '{': if rune(c) == '$' { l.ungetChars(1) lval.Token(l.createToken(tb[:len(tb)-1])); - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } case '\\': @@ -510,7 +512,7 @@ NEW_LINE (\r|\n|\r\n) if rune(c) == '"' { lval.Token(l.createToken(l.Token())); - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } currentChar = l.Last @@ -532,14 +534,14 @@ NEW_LINE (\r|\n|\r\n) if c == '{' || isValidFirstVarNameRune(rune(c)) { l.ungetChars(1) lval.Token(l.createToken(tb[:len(tb)-1])); - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } case '{': if rune(c) == '$' { l.ungetChars(1) lval.Token(l.createToken(tb[:len(tb)-1])); - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } case '\\': @@ -550,7 +552,7 @@ NEW_LINE (\r|\n|\r\n) if rune(c) == '`' { lval.Token(l.createToken(l.Token())); - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } currentChar = l.Last @@ -564,90 +566,105 @@ NEW_LINE (\r|\n|\r\n) .|[ \t\n\r] searchLabel := []byte{} - tb := []lex.Char{} + currentChar := l.Prev + tb := []lex.Char{currentChar} - for { - if c == -1 { - break; - } - + HEREDOC_FOR:for { nls := 0 - switch c { + switch currentChar.Rune { case '\r': - nls = 1 - c := l.Next() - if c != '\n' { - nls = 0 - l.ungetChars(0) + if c == '\n' { + nls = 1 + currentChar := l.Last + tb = append(tb, currentChar) + c = l.Next(); } fallthrough case '\n': - if l.heredocLabel + ";" == string(searchLabel) { + if l.heredocLabel + ";" == string(searchLabel) { l.begin(HEREDOC_END) - tb = l.ungetChars(len(l.heredocLabel)+1+nls) + l.ungetChars(len(l.heredocLabel)+1+nls) + + i := len(tb) - len(l.heredocLabel) - 3 - nls + if i < 1 { + break HEREDOC_FOR; + } + tb = tb[:i] + lval.Token(l.createToken(tb)); - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } if l.heredocLabel == string(searchLabel) { l.begin(HEREDOC_END) - tb = l.ungetChars(len(l.heredocLabel)+nls) + l.ungetChars(len(l.heredocLabel)+nls) + + i := len(tb) - len(l.heredocLabel) - 2 - nls + if i < 1 { + break HEREDOC_FOR; + } + tb = tb[:i] + lval.Token(l.createToken(tb)); - return T_ENCAPSED_AND_WHITESPACE + return int(T_ENCAPSED_AND_WHITESPACE) } searchLabel = []byte{} - + case '$': - c = l.Next(); - if rune(c) == '{' || isValidFirstVarNameRune(rune(c)) { - tb = l.ungetChars(1) - lval.Token(l.createToken(tb)); - return T_ENCAPSED_AND_WHITESPACE - } - l.ungetChars(0) - + if c == '{' || isValidFirstVarNameRune(rune(c)) { + l.ungetChars(1) + lval.Token(l.createToken(tb[:len(tb)-1])); + return int(T_ENCAPSED_AND_WHITESPACE) + } + case '{': - c = l.Next(); if rune(c) == '$' { - tb = l.ungetChars(1) - lval.Token(l.createToken(tb)); - return T_ENCAPSED_AND_WHITESPACE - } - l.ungetChars(0) + l.ungetChars(1) + lval.Token(l.createToken(tb[:len(tb)-1])); + return int(T_ENCAPSED_AND_WHITESPACE) + } case '\\': - c = l.Next(); - if c == '\n' || c == '\r' { - l.ungetChars(0) + if c != '\n' && c != '\r' { + currentChar := l.Last + tb = append(tb, currentChar) + c = l.Next(); } default: - searchLabel = append(searchLabel, byte(rune(c))) + searchLabel = append(searchLabel, byte(rune(currentChar.Rune))) } + if c == -1 { + break; + } + + currentChar = l.Last + tb = append(tb, currentChar) c = l.Next() + } -\${VAR_NAME} lval.Token(l.createToken(l.Token())); return T_VARIABLE -->{VAR_NAME} lval.Token(l.createToken(l.ungetChars(len(l.Token())-2))); return T_OBJECT_OPERATOR -{VAR_NAME} l.popState();lval.Token(l.createToken(l.Token())); return T_STRING +\${VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_VARIABLE) +->{VAR_NAME} lval.Token(l.createToken(l.ungetChars(len(l.Token())-2))); return int(T_OBJECT_OPERATOR) +{VAR_NAME} l.popState();lval.Token(l.createToken(l.Token())); return int(T_STRING) \[ l.pushState(STRING_VAR_INDEX);lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) .|[ \t\n\r] l.ungetChars(1);l.popState() -{LNUM}|{HNUM}|{BNUM} lval.Token(l.createToken(l.Token())); return T_NUM_STRING -\${VAR_NAME} lval.Token(l.createToken(l.Token())); return T_VARIABLE -{VAR_NAME} lval.Token(l.createToken(l.Token())); return T_STRING +{LNUM}|{HNUM}|{BNUM} lval.Token(l.createToken(l.Token())); return int(T_NUM_STRING) +\${VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_VARIABLE) +{VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_STRING) \] l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) -[ \n\r\t\\'#] l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return T_ENCAPSED_AND_WHITESPACE +[ \n\r\t\\'#] l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return int(T_ENCAPSED_AND_WHITESPACE) {OPERATORS} lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) . lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) -{VAR_NAME}[\[\}] l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return T_STRING_VARNAME +{VAR_NAME}[\[\}] l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_STRING_VARNAME) . l.ungetChars(1);l.popState();l.pushState(PHP) %% diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 408d0ff..c640944 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -230,194 +230,194 @@ func TestTokens(t *testing.T) { ` - expected := []int{ - scanner.T_INLINE_HTML, - scanner.Rune2Class(';'), - scanner.T_ECHO, - scanner.Rune2Class(';'), + expected := []string{ + scanner.T_INLINE_HTML.String(), + scanner.LexerToken(scanner.Rune2Class(';')).String(), + scanner.T_ECHO.String(), + scanner.LexerToken(scanner.Rune2Class(';')).String(), - scanner.T_DNUMBER, - scanner.T_DNUMBER, - scanner.T_DNUMBER, - scanner.T_DNUMBER, + scanner.T_DNUMBER.String(), + scanner.T_DNUMBER.String(), + scanner.T_DNUMBER.String(), + scanner.T_DNUMBER.String(), - scanner.T_LNUMBER, - scanner.T_DNUMBER, + scanner.T_LNUMBER.String(), + scanner.T_DNUMBER.String(), - scanner.T_LNUMBER, - scanner.T_DNUMBER, + scanner.T_LNUMBER.String(), + scanner.T_DNUMBER.String(), - scanner.T_LNUMBER, - scanner.T_DNUMBER, + scanner.T_LNUMBER.String(), + scanner.T_DNUMBER.String(), - scanner.T_ABSTRACT, - scanner.T_ARRAY, - scanner.T_AS, - scanner.T_BREAK, - scanner.T_CALLABLE, - scanner.T_CASE, - scanner.T_CATCH, - scanner.T_CLASS, - scanner.T_CLONE, - scanner.T_CONST, - scanner.T_CONTINUE, - scanner.T_DECLARE, - scanner.T_DEFAULT, - scanner.T_DO, - scanner.T_ECHO, - scanner.T_ELSE, - scanner.T_ELSEIF, - scanner.T_EMPTY, - scanner.T_ENDDECLARE, - scanner.T_ENDFOR, - scanner.T_ENDFOREACH, - scanner.T_ENDIF, - scanner.T_ENDSWITCH, - scanner.T_ENDWHILE, - scanner.T_EVAL, - scanner.T_EXIT, - scanner.T_EXTENDS, - scanner.T_FINAL, - scanner.T_FINALLY, - scanner.T_FOR, - scanner.T_FOREACH, - scanner.T_FUNCTION, - scanner.T_FUNCTION, - scanner.T_GLOBAL, - scanner.T_GOTO, - scanner.T_IF, - scanner.T_ISSET, - scanner.T_IMPLEMENTS, - scanner.T_INSTANCEOF, - scanner.T_INSTEADOF, - scanner.T_INTERFACE, - scanner.T_LIST, - scanner.T_NAMESPACE, - scanner.T_PRIVATE, - scanner.T_PUBLIC, - scanner.T_PRINT, - scanner.T_PROTECTED, - scanner.T_RETURN, - scanner.T_STATIC, - scanner.T_SWITCH, - scanner.T_THROW, - scanner.T_TRAIT, - scanner.T_TRY, - scanner.T_UNSET, - scanner.T_USE, - scanner.T_VAR, - scanner.T_WHILE, - scanner.T_YIELD_FROM, - scanner.T_YIELD, - scanner.T_INCLUDE, - scanner.T_INCLUDE_ONCE, - scanner.T_REQUIRE, - scanner.T_REQUIRE_ONCE, + scanner.T_ABSTRACT.String(), + scanner.T_ARRAY.String(), + scanner.T_AS.String(), + scanner.T_BREAK.String(), + scanner.T_CALLABLE.String(), + scanner.T_CASE.String(), + scanner.T_CATCH.String(), + scanner.T_CLASS.String(), + scanner.T_CLONE.String(), + scanner.T_CONST.String(), + scanner.T_CONTINUE.String(), + scanner.T_DECLARE.String(), + scanner.T_DEFAULT.String(), + scanner.T_DO.String(), + scanner.T_ECHO.String(), + scanner.T_ELSE.String(), + scanner.T_ELSEIF.String(), + scanner.T_EMPTY.String(), + scanner.T_ENDDECLARE.String(), + scanner.T_ENDFOR.String(), + scanner.T_ENDFOREACH.String(), + scanner.T_ENDIF.String(), + scanner.T_ENDSWITCH.String(), + scanner.T_ENDWHILE.String(), + scanner.T_EVAL.String(), + scanner.T_EXIT.String(), + scanner.T_EXTENDS.String(), + scanner.T_FINAL.String(), + scanner.T_FINALLY.String(), + scanner.T_FOR.String(), + scanner.T_FOREACH.String(), + scanner.T_FUNCTION.String(), + scanner.T_FUNCTION.String(), + scanner.T_GLOBAL.String(), + scanner.T_GOTO.String(), + scanner.T_IF.String(), + scanner.T_ISSET.String(), + scanner.T_IMPLEMENTS.String(), + scanner.T_INSTANCEOF.String(), + scanner.T_INSTEADOF.String(), + scanner.T_INTERFACE.String(), + scanner.T_LIST.String(), + scanner.T_NAMESPACE.String(), + scanner.T_PRIVATE.String(), + scanner.T_PUBLIC.String(), + scanner.T_PRINT.String(), + scanner.T_PROTECTED.String(), + scanner.T_RETURN.String(), + scanner.T_STATIC.String(), + scanner.T_SWITCH.String(), + scanner.T_THROW.String(), + scanner.T_TRAIT.String(), + scanner.T_TRY.String(), + scanner.T_UNSET.String(), + scanner.T_USE.String(), + scanner.T_VAR.String(), + scanner.T_WHILE.String(), + scanner.T_YIELD_FROM.String(), + scanner.T_YIELD.String(), + scanner.T_INCLUDE.String(), + scanner.T_INCLUDE_ONCE.String(), + scanner.T_REQUIRE.String(), + scanner.T_REQUIRE_ONCE.String(), - scanner.T_CLASS_C, - scanner.T_DIR, - scanner.T_FILE, - scanner.T_FUNC_C, - scanner.T_LINE, - scanner.T_NS_C, - scanner.T_METHOD_C, - scanner.T_TRAIT_C, - scanner.T_HALT_COMPILER, + scanner.T_CLASS_C.String(), + scanner.T_DIR.String(), + scanner.T_FILE.String(), + scanner.T_FUNC_C.String(), + scanner.T_LINE.String(), + scanner.T_NS_C.String(), + scanner.T_METHOD_C.String(), + scanner.T_TRAIT_C.String(), + scanner.T_HALT_COMPILER.String(), - scanner.T_NEW, - scanner.T_LOGICAL_AND, - scanner.T_LOGICAL_OR, - scanner.T_LOGICAL_XOR, + scanner.T_NEW.String(), + scanner.T_LOGICAL_AND.String(), + scanner.T_LOGICAL_OR.String(), + scanner.T_LOGICAL_XOR.String(), - scanner.T_NS_SEPARATOR, - scanner.T_ELLIPSIS, - scanner.T_PAAMAYIM_NEKUDOTAYIM, - scanner.T_BOOLEAN_AND, - scanner.T_BOOLEAN_OR, - scanner.T_AND_EQUAL, - scanner.T_OR_EQUAL, - scanner.T_CONCAT_EQUAL, - scanner.T_MUL_EQUAL, - scanner.T_POW_EQUAL, - scanner.T_DIV_EQUAL, - scanner.T_PLUS_EQUAL, - scanner.T_MINUS_EQUAL, - scanner.T_XOR_EQUAL, - scanner.T_MOD_EQUAL, - scanner.T_DEC, - scanner.T_INC, - scanner.T_DOUBLE_ARROW, - scanner.T_SPACESHIP, - scanner.T_IS_NOT_EQUAL, - scanner.T_IS_NOT_EQUAL, - scanner.T_IS_NOT_IDENTICAL, - scanner.T_IS_EQUAL, - scanner.T_IS_IDENTICAL, - scanner.T_SL_EQUAL, - scanner.T_SR_EQUAL, - scanner.T_IS_GREATER_OR_EQUAL, - scanner.T_IS_SMALLER_OR_EQUAL, - scanner.T_POW, - scanner.T_SL, - scanner.T_SR, - scanner.T_COALESCE, + scanner.T_NS_SEPARATOR.String(), + scanner.T_ELLIPSIS.String(), + scanner.T_PAAMAYIM_NEKUDOTAYIM.String(), + scanner.T_BOOLEAN_AND.String(), + scanner.T_BOOLEAN_OR.String(), + scanner.T_AND_EQUAL.String(), + scanner.T_OR_EQUAL.String(), + scanner.T_CONCAT_EQUAL.String(), + scanner.T_MUL_EQUAL.String(), + scanner.T_POW_EQUAL.String(), + scanner.T_DIV_EQUAL.String(), + scanner.T_PLUS_EQUAL.String(), + scanner.T_MINUS_EQUAL.String(), + scanner.T_XOR_EQUAL.String(), + scanner.T_MOD_EQUAL.String(), + scanner.T_DEC.String(), + scanner.T_INC.String(), + scanner.T_DOUBLE_ARROW.String(), + scanner.T_SPACESHIP.String(), + scanner.T_IS_NOT_EQUAL.String(), + scanner.T_IS_NOT_EQUAL.String(), + scanner.T_IS_NOT_IDENTICAL.String(), + scanner.T_IS_EQUAL.String(), + scanner.T_IS_IDENTICAL.String(), + scanner.T_SL_EQUAL.String(), + scanner.T_SR_EQUAL.String(), + scanner.T_IS_GREATER_OR_EQUAL.String(), + scanner.T_IS_SMALLER_OR_EQUAL.String(), + scanner.T_POW.String(), + scanner.T_SL.String(), + scanner.T_SR.String(), + scanner.T_COALESCE.String(), - scanner.Rune2Class(';'), - scanner.Rune2Class(':'), - scanner.Rune2Class(','), - scanner.Rune2Class('.'), - scanner.Rune2Class('['), - scanner.Rune2Class(']'), - scanner.Rune2Class('('), - scanner.Rune2Class(')'), - scanner.Rune2Class('|'), - scanner.Rune2Class('/'), - scanner.Rune2Class('^'), - scanner.Rune2Class('&'), - scanner.Rune2Class('+'), - scanner.Rune2Class('-'), - scanner.Rune2Class('*'), - scanner.Rune2Class('='), - scanner.Rune2Class('%'), - scanner.Rune2Class('!'), - scanner.Rune2Class('~'), - scanner.Rune2Class('$'), - scanner.Rune2Class('<'), - scanner.Rune2Class('>'), - scanner.Rune2Class('?'), - scanner.Rune2Class('@'), - scanner.Rune2Class('{'), - scanner.Rune2Class('}'), + scanner.LexerToken(scanner.Rune2Class(';')).String(), + scanner.LexerToken(scanner.Rune2Class(':')).String(), + scanner.LexerToken(scanner.Rune2Class(',')).String(), + scanner.LexerToken(scanner.Rune2Class('.')).String(), + scanner.LexerToken(scanner.Rune2Class('[')).String(), + scanner.LexerToken(scanner.Rune2Class(']')).String(), + scanner.LexerToken(scanner.Rune2Class('(')).String(), + scanner.LexerToken(scanner.Rune2Class(')')).String(), + scanner.LexerToken(scanner.Rune2Class('|')).String(), + scanner.LexerToken(scanner.Rune2Class('/')).String(), + scanner.LexerToken(scanner.Rune2Class('^')).String(), + scanner.LexerToken(scanner.Rune2Class('&')).String(), + scanner.LexerToken(scanner.Rune2Class('+')).String(), + scanner.LexerToken(scanner.Rune2Class('-')).String(), + scanner.LexerToken(scanner.Rune2Class('*')).String(), + scanner.LexerToken(scanner.Rune2Class('=')).String(), + scanner.LexerToken(scanner.Rune2Class('%')).String(), + scanner.LexerToken(scanner.Rune2Class('!')).String(), + scanner.LexerToken(scanner.Rune2Class('~')).String(), + scanner.LexerToken(scanner.Rune2Class('$')).String(), + scanner.LexerToken(scanner.Rune2Class('<')).String(), + scanner.LexerToken(scanner.Rune2Class('>')).String(), + scanner.LexerToken(scanner.Rune2Class('?')).String(), + scanner.LexerToken(scanner.Rune2Class('@')).String(), + scanner.LexerToken(scanner.Rune2Class('{')).String(), + scanner.LexerToken(scanner.Rune2Class('}')).String(), - scanner.T_VARIABLE, - scanner.T_STRING, + scanner.T_VARIABLE.String(), + scanner.T_STRING.String(), - scanner.T_OBJECT_OPERATOR, - scanner.T_OBJECT_OPERATOR, - scanner.T_STRING, + scanner.T_OBJECT_OPERATOR.String(), + scanner.T_OBJECT_OPERATOR.String(), + scanner.T_STRING.String(), - scanner.T_CONSTANT_ENCAPSED_STRING, - scanner.T_CONSTANT_ENCAPSED_STRING, - scanner.T_CONSTANT_ENCAPSED_STRING, + scanner.T_CONSTANT_ENCAPSED_STRING.String(), + scanner.T_CONSTANT_ENCAPSED_STRING.String(), + scanner.T_CONSTANT_ENCAPSED_STRING.String(), - scanner.T_ARRAY_CAST, - scanner.T_BOOL_CAST, - scanner.T_BOOL_CAST, - scanner.T_DOUBLE_CAST, - scanner.T_DOUBLE_CAST, - scanner.T_DOUBLE_CAST, - scanner.T_INT_CAST, - scanner.T_INT_CAST, - scanner.T_OBJECT_CAST, - scanner.T_STRING_CAST, - scanner.T_STRING_CAST, - scanner.T_UNSET_CAST, + scanner.T_ARRAY_CAST.String(), + scanner.T_BOOL_CAST.String(), + scanner.T_BOOL_CAST.String(), + scanner.T_DOUBLE_CAST.String(), + scanner.T_DOUBLE_CAST.String(), + scanner.T_DOUBLE_CAST.String(), + scanner.T_INT_CAST.String(), + scanner.T_INT_CAST.String(), + scanner.T_OBJECT_CAST.String(), + scanner.T_STRING_CAST.String(), + scanner.T_STRING_CAST.String(), + scanner.T_UNSET_CAST.String(), } lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer.WithMeta = true lv := &lval{} - actual := []int{} + actual := []string{} for { token := lexer.Lex(lv) @@ -425,7 +425,7 @@ func TestTokens(t *testing.T) { break } - actual = append(actual, token) + actual = append(actual, scanner.LexerToken(token).String()) } assertEqual(t, expected, actual) @@ -448,65 +448,65 @@ func TestTeplateStringTokens(t *testing.T) { "$0$foo" ` - expected := []int{ - scanner.Rune2Class('"'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.Rune2Class('"'), + expected := []string{ + scanner.LexerToken(scanner.Rune2Class('"')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('"')).String(), - scanner.Rune2Class('"'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.T_CURLY_OPEN, - scanner.T_VARIABLE, - scanner.Rune2Class('}'), - scanner.Rune2Class('"'), + scanner.LexerToken(scanner.Rune2Class('"')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.T_CURLY_OPEN.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('}')).String(), + scanner.LexerToken(scanner.Rune2Class('"')).String(), - scanner.Rune2Class('"'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_CURLY_OPEN, - scanner.T_VARIABLE, - scanner.Rune2Class('}'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_DOLLAR_OPEN_CURLY_BRACES, - scanner.T_STRING_VARNAME, - scanner.Rune2Class('}'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.Rune2Class('"'), + scanner.LexerToken(scanner.Rune2Class('"')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_CURLY_OPEN.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('}')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_DOLLAR_OPEN_CURLY_BRACES.String(), + scanner.T_STRING_VARNAME.String(), + scanner.LexerToken(scanner.Rune2Class('}')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.LexerToken(scanner.Rune2Class('"')).String(), - scanner.Rune2Class('"'), - scanner.T_CURLY_OPEN, - scanner.T_VARIABLE, - scanner.Rune2Class('}'), - scanner.Rune2Class('"'), + scanner.LexerToken(scanner.Rune2Class('"')).String(), + scanner.T_CURLY_OPEN.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('}')).String(), + scanner.LexerToken(scanner.Rune2Class('"')).String(), - scanner.Rune2Class('"'), - scanner.T_VARIABLE, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.Rune2Class('"'), + scanner.LexerToken(scanner.Rune2Class('"')).String(), + scanner.T_VARIABLE.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.LexerToken(scanner.Rune2Class('"')).String(), - scanner.Rune2Class('"'), - scanner.T_VARIABLE, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.Rune2Class('"'), + scanner.LexerToken(scanner.Rune2Class('"')).String(), + scanner.T_VARIABLE.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.LexerToken(scanner.Rune2Class('"')).String(), - scanner.Rune2Class('"'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.Rune2Class('"'), + scanner.LexerToken(scanner.Rune2Class('"')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('"')).String(), - scanner.Rune2Class('"'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.Rune2Class('"'), + scanner.LexerToken(scanner.Rune2Class('"')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('"')).String(), } lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer.WithMeta = true lv := &lval{} - actual := []int{} + actual := []string{} for { token := lexer.Lex(lv) @@ -514,7 +514,7 @@ func TestTeplateStringTokens(t *testing.T) { break } - actual = append(actual, token) + actual = append(actual, scanner.LexerToken(token).String()) } assertEqual(t, expected, actual) @@ -534,65 +534,65 @@ func TestBackquoteStringTokens(t *testing.T) { ` + "`$0$foo`" + ` ` - expected := []int{ - scanner.Rune2Class('`'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.Rune2Class('`'), + expected := []string{ + scanner.LexerToken(scanner.Rune2Class('`')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('`')).String(), - scanner.Rune2Class('`'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.T_CURLY_OPEN, - scanner.T_VARIABLE, - scanner.Rune2Class('}'), - scanner.Rune2Class('`'), + scanner.LexerToken(scanner.Rune2Class('`')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.T_CURLY_OPEN.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('}')).String(), + scanner.LexerToken(scanner.Rune2Class('`')).String(), - scanner.Rune2Class('`'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_CURLY_OPEN, - scanner.T_VARIABLE, - scanner.Rune2Class('}'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_DOLLAR_OPEN_CURLY_BRACES, - scanner.T_STRING_VARNAME, - scanner.Rune2Class('}'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.Rune2Class('`'), + scanner.LexerToken(scanner.Rune2Class('`')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_CURLY_OPEN.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('}')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_DOLLAR_OPEN_CURLY_BRACES.String(), + scanner.T_STRING_VARNAME.String(), + scanner.LexerToken(scanner.Rune2Class('}')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.LexerToken(scanner.Rune2Class('`')).String(), - scanner.Rune2Class('`'), - scanner.T_CURLY_OPEN, - scanner.T_VARIABLE, - scanner.Rune2Class('}'), - scanner.Rune2Class('`'), + scanner.LexerToken(scanner.Rune2Class('`')).String(), + scanner.T_CURLY_OPEN.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('}')).String(), + scanner.LexerToken(scanner.Rune2Class('`')).String(), - scanner.Rune2Class('`'), - scanner.T_VARIABLE, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.Rune2Class('`'), + scanner.LexerToken(scanner.Rune2Class('`')).String(), + scanner.T_VARIABLE.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.LexerToken(scanner.Rune2Class('`')).String(), - scanner.Rune2Class('`'), - scanner.T_VARIABLE, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.Rune2Class('`'), + scanner.LexerToken(scanner.Rune2Class('`')).String(), + scanner.T_VARIABLE.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.LexerToken(scanner.Rune2Class('`')).String(), - scanner.Rune2Class('`'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.Rune2Class('`'), + scanner.LexerToken(scanner.Rune2Class('`')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('`')).String(), - scanner.Rune2Class('`'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.Rune2Class('`'), + scanner.LexerToken(scanner.Rune2Class('`')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('`')).String(), } lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer.WithMeta = true lv := &lval{} - actual := []int{} + actual := []string{} for { token := lexer.Lex(lv) @@ -600,7 +600,7 @@ func TestBackquoteStringTokens(t *testing.T) { break } - actual = append(actual, token) + actual = append(actual, scanner.LexerToken(token).String()) } assertEqual(t, expected, actual) @@ -630,63 +630,64 @@ CAT; CAT; ` - expected := []int{ - scanner.T_START_HEREDOC, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_END_HEREDOC, - scanner.Rune2Class(';'), + expected := []string{ + scanner.T_START_HEREDOC.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_END_HEREDOC.String(), + scanner.LexerToken(scanner.Rune2Class(';')).String(), - scanner.T_START_HEREDOC, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_END_HEREDOC, - scanner.Rune2Class(';'), + scanner.T_START_HEREDOC.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_END_HEREDOC.String(), + scanner.LexerToken(scanner.Rune2Class(';')).String(), - scanner.T_START_HEREDOC, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.T_OBJECT_OPERATOR, - scanner.T_STRING, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.Rune2Class('['), - scanner.T_NUM_STRING, - scanner.Rune2Class(']'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.Rune2Class('['), - scanner.T_NUM_STRING, - scanner.Rune2Class(']'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.Rune2Class('['), - scanner.T_NUM_STRING, - scanner.Rune2Class(']'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.Rune2Class('['), - scanner.T_STRING, - scanner.Rune2Class(']'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.Rune2Class('['), - scanner.T_VARIABLE, - scanner.Rune2Class(']'), - scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_CURLY_OPEN, - scanner.T_VARIABLE, - scanner.Rune2Class('}'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_DOLLAR_OPEN_CURLY_BRACES, - scanner.T_STRING_VARNAME, - scanner.Rune2Class('}'), - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_END_HEREDOC, - scanner.Rune2Class(';'), + scanner.T_START_HEREDOC.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.T_OBJECT_OPERATOR.String(), + scanner.T_STRING.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('[')).String(), + scanner.T_NUM_STRING.String(), + scanner.LexerToken(scanner.Rune2Class(']')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('[')).String(), + scanner.T_NUM_STRING.String(), + scanner.LexerToken(scanner.Rune2Class(']')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('[')).String(), + scanner.T_NUM_STRING.String(), + scanner.LexerToken(scanner.Rune2Class(']')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('[')).String(), + scanner.T_STRING.String(), + scanner.LexerToken(scanner.Rune2Class(']')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('[')).String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class(']')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_CURLY_OPEN.String(), + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class('}')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_DOLLAR_OPEN_CURLY_BRACES.String(), + scanner.T_STRING_VARNAME.String(), + scanner.LexerToken(scanner.Rune2Class('}')).String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_END_HEREDOC.String(), + scanner.LexerToken(scanner.Rune2Class(';')).String(), } lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer.WithMeta = true lv := &lval{} - actual := []int{} + actual := []string{} for { token := lexer.Lex(lv) @@ -694,7 +695,7 @@ CAT; break } - actual = append(actual, token) + actual = append(actual, scanner.LexerToken(token).String()) } assertEqual(t, expected, actual) @@ -723,44 +724,42 @@ $foo$bar\ CAT ` - expected := []int{ - scanner.T_START_HEREDOC, - scanner.T_VARIABLE, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_END_HEREDOC, - scanner.Rune2Class(';'), + expected := []string{ + scanner.T_START_HEREDOC.String(), + scanner.T_VARIABLE.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_END_HEREDOC.String(), + scanner.LexerToken(scanner.Rune2Class(';')).String(), - scanner.T_START_HEREDOC, - scanner.T_VARIABLE, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_END_HEREDOC, - scanner.Rune2Class(';'), + scanner.T_START_HEREDOC.String(), + scanner.T_VARIABLE.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_END_HEREDOC.String(), + scanner.LexerToken(scanner.Rune2Class(';')).String(), - scanner.T_START_HEREDOC, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_END_HEREDOC, - scanner.Rune2Class(';'), + scanner.T_START_HEREDOC.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.T_END_HEREDOC.String(), + scanner.LexerToken(scanner.Rune2Class(';')).String(), - scanner.T_START_HEREDOC, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_VARIABLE, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_END_HEREDOC, - scanner.Rune2Class(';'), + scanner.T_START_HEREDOC.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_VARIABLE.String(), + scanner.T_END_HEREDOC.String(), + scanner.LexerToken(scanner.Rune2Class(';')).String(), - scanner.T_START_HEREDOC, - scanner.T_VARIABLE, - scanner.T_VARIABLE, - scanner.T_ENCAPSED_AND_WHITESPACE, - scanner.T_END_HEREDOC, + scanner.T_START_HEREDOC.String(), + scanner.T_VARIABLE.String(), + scanner.T_VARIABLE.String(), + scanner.T_ENCAPSED_AND_WHITESPACE.String(), + scanner.T_END_HEREDOC.String(), } lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer.WithMeta = true lv := &lval{} - actual := []int{} + actual := []string{} for { token := lexer.Lex(lv) @@ -768,7 +767,46 @@ CAT break } - actual = append(actual, token) + actual = append(actual, scanner.LexerToken(token).String()) + } + + assertEqual(t, expected, actual) +} + +func TestHereDocTokens3(t *testing.T) { + src := ` test ` - expected := []int{ - scanner.T_VARIABLE, - scanner.Rune2Class(';'), - scanner.T_INLINE_HTML, + expected := []string{ + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class(';')).String(), + scanner.T_INLINE_HTML.String(), - scanner.T_VARIABLE, - scanner.Rune2Class(';'), - scanner.T_INLINE_HTML, + scanner.T_VARIABLE.String(), + scanner.LexerToken(scanner.Rune2Class(';')).String(), + scanner.T_INLINE_HTML.String(), } lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer.WithMeta = true lv := &lval{} - actual := []int{} + actual := []string{} for { token := lexer.Lex(lv) @@ -801,7 +839,7 @@ func TestInlineHtmlNopTokens(t *testing.T) { break } - actual = append(actual, token) + actual = append(actual, scanner.LexerToken(token).String()) } assertEqual(t, expected, actual) @@ -810,12 +848,12 @@ func TestInlineHtmlNopTokens(t *testing.T) { func TestStringTokensAfterVariable(t *testing.T) { src := `