maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed bug when heredoc starts by \; removing \n from a heredoc value end

Browse Source
This commit is contained in:
z7zmey 2018-07-14 18:00:48 +03:00
parent 4da7b36056
commit 867095823d
11 changed files with 848 additions and 791 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
node/scalar/t_heredoc_test.go
View File

@ -70,15 +70,6 @@ LBL;
Value: "var", Value: "var",
}, },
}, },
&scalar.EncapsedStringPart{
Position: &position.Position{
StartLine: 2,
EndLine: 2,
StartPos: 20,
EndPos: 20,
},
Value: "\n",
},
}, },
}, },
}, },
@ -152,15 +143,6 @@ LBL;
Value: "var", Value: "var",
}, },
}, },
&scalar.EncapsedStringPart{
Position: &position.Position{
StartLine: 2,
EndLine: 2,
StartPos: 22,
EndPos: 22,
},
Value: "\n",
},
}, },
}, },
}, },
@ -213,9 +195,9 @@ LBL;
StartLine: 2, StartLine: 2,
EndLine: 2, EndLine: 2,
StartPos: 13, StartPos: 13,
EndPos: 22, EndPos: 21,
}, },
Value: "test $var\n", Value: "test $var",
}, },
}, },
}, },
@ -313,9 +295,9 @@ CAD;
StartLine: 2, StartLine: 2,
EndLine: 2, EndLine: 2,
StartPos: 11, StartPos: 11,
EndPos: 17, EndPos: 16,
}, },
Value: "\thello\n", Value: "\thello",
}, },
}, },
}, },

21
php5/php5_test.go
View File

@ -18597,9 +18597,9 @@ CAD;
StartLine: 5, StartLine: 5,
EndLine: 5, EndLine: 5,
StartPos: 27, StartPos: 27,
EndPos: 33, EndPos: 32,
}, },
Value: "\thello\n", Value: "\thello",
}, },
}, },
}, },
@ -18625,9 +18625,9 @@ CAD;
StartLine: 8, StartLine: 8,
EndLine: 8, EndLine: 8,
StartPos: 50, StartPos: 50,
EndPos: 56, EndPos: 55,
}, },
Value: "\thello\n", Value: "\thello",
}, },
}, },
}, },
@ -18674,15 +18674,6 @@ CAD;
Value: "world", Value: "world",
}, },
}, },
&scalar.EncapsedStringPart{
Position: &position.Position{
StartLine: 11,
EndLine: 11,
StartPos: 86,
EndPos: 86,
},
Value: "\n",
},
}, },
}, },
}, },
@ -18707,9 +18698,9 @@ CAD;
StartLine: 14, StartLine: 14,
EndLine: 14, EndLine: 14,
StartPos: 103, StartPos: 103,
EndPos: 116, EndPos: 115,
}, },
Value: "\thello $world\n", Value: "\thello $world",
}, },
}, },
}, },

21
php7/php7_test.go
View File

@ -16312,9 +16312,9 @@ CAD;
StartLine: 5, StartLine: 5,
EndLine: 5, EndLine: 5,
StartPos: 27, StartPos: 27,
EndPos: 33, EndPos: 32,
}, },
Value: "\thello\n", Value: "\thello",
}, },
}, },
}, },
@ -16340,9 +16340,9 @@ CAD;
StartLine: 8, StartLine: 8,
EndLine: 8, EndLine: 8,
StartPos: 50, StartPos: 50,
EndPos: 56, EndPos: 55,
}, },
Value: "\thello\n", Value: "\thello",
}, },
}, },
}, },
@ -16389,15 +16389,6 @@ CAD;
Value: "world", Value: "world",
}, },
}, },
&scalar.EncapsedStringPart{
Position: &position.Position{
StartLine: 11,
EndLine: 11,
StartPos: 86,
EndPos: 86,
},
Value: "\n",
},
}, },
}, },
}, },
@ -16422,9 +16413,9 @@ CAD;
StartLine: 14, StartLine: 14,
EndLine: 14, EndLine: 14,
StartPos: 103, StartPos: 103,
EndPos: 116, EndPos: 115,
}, },
Value: "\thello $world\n", Value: "\thello $world",
}, },
}, },
}, },

1
printer/printer.go
View File

@ -665,6 +665,7 @@ func (p *Printer) printScalarHeredoc(n node.Node) {
} }
} }
io.WriteString(p.w, "\n")
io.WriteString(p.w, strings.Trim(nn.Label, "\"'")) io.WriteString(p.w, strings.Trim(nn.Label, "\"'"))
p.printMeta(nn, meta.NodeEnd) p.printMeta(nn, meta.NodeEnd)

4
printer/printer_test.go
View File

@ -575,7 +575,7 @@ func TestPrinterPrintScalarHeredoc(t *testing.T) {
Parts: []node.Node{ Parts: []node.Node{
&scalar.EncapsedStringPart{Value: "hello "}, &scalar.EncapsedStringPart{Value: "hello "},
&expr.Variable{VarName: &node.Identifier{Value: "var"}}, &expr.Variable{VarName: &node.Identifier{Value: "var"}},
&scalar.EncapsedStringPart{Value: " world\n"}, &scalar.EncapsedStringPart{Value: " world"},
}, },
}) })
@ -602,7 +602,7 @@ func TestPrinterPrintScalarNowdoc(t *testing.T) {
}, },
Label: "'LBL'", Label: "'LBL'",
Parts: []node.Node{ Parts: []node.Node{
&scalar.EncapsedStringPart{Value: "hello world\n"}, &scalar.EncapsedStringPart{Value: "hello world"},
}, },
}) })

2
scanner/lexer.go
View File

@ -124,6 +124,8 @@ func (l *Lexer) createToken(chars []lex.Char) *Token {
token.Meta = l.Meta token.Meta = l.Meta
token.Value = l.tokenString(chars) token.Value = l.tokenString(chars)
// fmt.Println(l.tokenString(chars))
token.StartLine = l.File.Line(firstChar.Pos()) token.StartLine = l.File.Line(firstChar.Pos())
token.EndLine = l.File.Line(lastChar.Pos()) token.EndLine = l.File.Line(lastChar.Pos())
token.StartPos = int(firstChar.Pos()) token.StartPos = int(firstChar.Pos())

5
scanner/lexer_tokens.go
View File

@ -1,7 +1,10 @@
package scanner package scanner
type LexerToken int
//go:generate stringer -type=LexerToken -output ./lexer_tokens_string.go
const ( const (
T_INCLUDE int = iota + 57346 T_INCLUDE LexerToken = iota + 57346
T_INCLUDE_ONCE T_INCLUDE_ONCE
T_EXIT T_EXIT
T_IF T_IF

17
scanner/lexer_tokens_string.go Normal file
View File

@ -0,0 +1,17 @@
// Code generated by "stringer -type=LexerToken -output ./lexer_tokens_string.go"; DO NOT EDIT.
package scanner
import "strconv"
const _LexerToken_name = "T_INCLUDET_INCLUDE_ONCET_EXITT_IFT_LNUMBERT_DNUMBERT_STRINGT_STRING_VARNAMET_VARIABLET_NUM_STRINGT_INLINE_HTMLT_CHARACTERT_BAD_CHARACTERT_ENCAPSED_AND_WHITESPACET_CONSTANT_ENCAPSED_STRINGT_ECHOT_DOT_WHILET_ENDWHILET_FORT_ENDFORT_FOREACHT_ENDFOREACHT_DECLARET_ENDDECLARET_AST_SWITCHT_ENDSWITCHT_CASET_DEFAULTT_BREAKT_CONTINUET_GOTOT_FUNCTIONT_CONSTT_RETURNT_TRYT_CATCHT_FINALLYT_THROWT_USET_INSTEADOFT_GLOBALT_VART_UNSETT_ISSETT_EMPTYT_HALT_COMPILERT_CLASST_TRAITT_INTERFACET_EXTENDST_IMPLEMENTST_OBJECT_OPERATORT_DOUBLE_ARROWT_LISTT_ARRAYT_CALLABLET_CLASS_CT_TRAIT_CT_METHOD_CT_FUNC_CT_LINET_FILET_COMMENTT_DOC_COMMENTT_OPEN_TAGT_OPEN_TAG_WITH_ECHOT_CLOSE_TAGT_WHITESPACET_START_HEREDOCT_END_HEREDOCT_DOLLAR_OPEN_CURLY_BRACEST_CURLY_OPENT_PAAMAYIM_NEKUDOTAYIMT_NAMESPACET_NS_CT_DIRT_NS_SEPARATORT_ELLIPSIST_EVALT_REQUIRET_REQUIRE_ONCET_LOGICAL_ORT_LOGICAL_XORT_LOGICAL_ANDT_INSTANCEOFT_NEWT_CLONET_ELSEIFT_ELSET_ENDIFT_PRINTT_YIELDT_STATICT_ABSTRACTT_FINALT_PRIVATET_PROTECTEDT_PUBLICT_INCT_DECT_YIELD_FROMT_INT_CASTT_DOUBLE_CASTT_STRING_CASTT_ARRAY_CASTT_OBJECT_CASTT_BOOL_CASTT_UNSET_CASTT_COALESCET_SPACESHIPT_NOELSET_PLUS_EQUALT_MINUS_EQUALT_MUL_EQUALT_POW_EQUALT_DIV_EQUALT_CONCAT_EQUALT_MOD_EQUALT_AND_EQUALT_OR_EQUALT_XOR_EQUALT_SL_EQUALT_SR_EQUALT_BOOLEAN_ORT_BOOLEAN_ANDT_POWT_SLT_SRT_IS_IDENTICALT_IS_NOT_IDENTICALT_IS_EQUALT_IS_NOT_EQUALT_IS_SMALLER_OR_EQUALT_IS_GREATER_OR_EQUAL"
var _LexerToken_index = [...]uint16{0, 9, 23, 29, 33, 42, 51, 59, 75, 85, 97, 110, 121, 136, 161, 187, 193, 197, 204, 214, 219, 227, 236, 248, 257, 269, 273, 281, 292, 298, 307, 314, 324, 330, 340, 347, 355, 360, 367, 376, 383, 388, 399, 407, 412, 419, 426, 433, 448, 455, 462, 473, 482, 494, 511, 525, 531, 538, 548, 557, 566, 576, 584, 590, 596, 605, 618, 628, 648, 659, 671, 686, 699, 725, 737, 759, 770, 776, 781, 795, 805, 811, 820, 834, 846, 859, 872, 884, 889, 896, 904, 910, 917, 924, 931, 939, 949, 956, 965, 976, 984, 989, 994, 1006, 1016, 1029, 1042, 1054, 1067, 1078, 1090, 1100, 1111, 1119, 1131, 1144, 1155, 1166, 1177, 1191, 1202, 1213, 1223, 1234, 1244, 1254, 1266, 1279, 1284, 1288, 1292, 1306, 1324, 1334, 1348, 1369, 1390}
func (i LexerToken) String() string {
i -= 57346
if i < 0 || i >= LexerToken(len(_LexerToken_index)-1) {
return "LexerToken(" + strconv.FormatInt(int64(i+57346), 10) + ")"
}
return _LexerToken_name[_LexerToken_index[i]:_LexerToken_index[i+1]]
}

375
scanner/scanner.go
View File

File diff suppressed because it is too large Load Diff

377
scanner/scanner.l
View File

@ -82,17 +82,17 @@ NEW_LINE (\r|\n|\r\n)
} }
lval.Token(l.createToken(tb)) lval.Token(l.createToken(tb))
return T_INLINE_HTML return int(T_INLINE_HTML)
<INITIAL>\<\?php([ \t]|{NEW_LINE}) l.begin(PHP);l.ungetChars(len(l.Token())-5) <INITIAL>\<\?php([ \t]|{NEW_LINE}) l.begin(PHP);l.ungetChars(len(l.Token())-5)
<INITIAL>\<\? l.begin(PHP); <INITIAL>\<\? l.begin(PHP);
<INITIAL>\<\?= l.begin(PHP);lval.Token(l.createToken(l.Token())); return T_ECHO; <INITIAL>\<\?= l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_ECHO);
<PHP>[ \t\n\r]+ l.addWhiteSpace(l.Token()) <PHP>[ \t\n\r]+ l.addWhiteSpace(l.Token())
<PHP>[;][ \t\n\r]*\?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';'); <PHP>[;][ \t\n\r]*\?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
<PHP>\?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';'); <PHP>\?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
<PHP>{DNUM}|{EXPONENT_DNUM} lval.Token(l.createToken(l.Token())); return T_DNUMBER <PHP>{DNUM}|{EXPONENT_DNUM} lval.Token(l.createToken(l.Token())); return int(T_DNUMBER)
<PHP>{BNUM} <PHP>{BNUM}
tb := l.Token() tb := l.Token()
i:=2 i:=2
@ -106,15 +106,15 @@ NEW_LINE (\r|\n|\r\n)
} }
} }
if len(tb) - i < 64 { if len(tb) - i < 64 {
lval.Token(l.createToken(l.Token())); return T_LNUMBER lval.Token(l.createToken(l.Token())); return int(T_LNUMBER)
} else { } else {
lval.Token(l.createToken(l.Token())); return T_DNUMBER lval.Token(l.createToken(l.Token())); return int(T_DNUMBER)
} }
<PHP>{LNUM} <PHP>{LNUM}
if len(l.Token()) < 20 { if len(l.Token()) < 20 {
lval.Token(l.createToken(l.Token())); return T_LNUMBER lval.Token(l.createToken(l.Token())); return int(T_LNUMBER)
} else { } else {
lval.Token(l.createToken(l.Token())); return T_DNUMBER lval.Token(l.createToken(l.Token())); return int(T_DNUMBER)
} }
<PHP>{HNUM} <PHP>{HNUM}
tb := l.Token() tb := l.Token()
@ -130,124 +130,124 @@ NEW_LINE (\r|\n|\r\n)
} }
length := len(tb) - i length := len(tb) - i
if length < 16 || (length == 16 && tb[i].Rune <= '7') { if length < 16 || (length == 16 && tb[i].Rune <= '7') {
lval.Token(l.createToken(l.Token())); return T_LNUMBER lval.Token(l.createToken(l.Token())); return int(T_LNUMBER)
} else { } else {
lval.Token(l.createToken(l.Token())); return T_DNUMBER lval.Token(l.createToken(l.Token())); return int(T_DNUMBER)
} }
<PHP>abstract lval.Token(l.createToken(l.Token())); return T_ABSTRACT <PHP>abstract lval.Token(l.createToken(l.Token())); return int(T_ABSTRACT)
<PHP>array lval.Token(l.createToken(l.Token())); return T_ARRAY <PHP>array lval.Token(l.createToken(l.Token())); return int(T_ARRAY)
<PHP>as lval.Token(l.createToken(l.Token())); return T_AS <PHP>as lval.Token(l.createToken(l.Token())); return int(T_AS)
<PHP>break lval.Token(l.createToken(l.Token())); return T_BREAK <PHP>break lval.Token(l.createToken(l.Token())); return int(T_BREAK)
<PHP>callable lval.Token(l.createToken(l.Token())); return T_CALLABLE <PHP>callable lval.Token(l.createToken(l.Token())); return int(T_CALLABLE)
<PHP>case lval.Token(l.createToken(l.Token())); return T_CASE <PHP>case lval.Token(l.createToken(l.Token())); return int(T_CASE)
<PHP>catch lval.Token(l.createToken(l.Token())); return T_CATCH <PHP>catch lval.Token(l.createToken(l.Token())); return int(T_CATCH)
<PHP>class lval.Token(l.createToken(l.Token())); return T_CLASS <PHP>class lval.Token(l.createToken(l.Token())); return int(T_CLASS)
<PHP>clone lval.Token(l.createToken(l.Token())); return T_CLONE <PHP>clone lval.Token(l.createToken(l.Token())); return int(T_CLONE)
<PHP>const lval.Token(l.createToken(l.Token())); return T_CONST <PHP>const lval.Token(l.createToken(l.Token())); return int(T_CONST)
<PHP>continue lval.Token(l.createToken(l.Token())); return T_CONTINUE <PHP>continue lval.Token(l.createToken(l.Token())); return int(T_CONTINUE)
<PHP>declare lval.Token(l.createToken(l.Token())); return T_DECLARE <PHP>declare lval.Token(l.createToken(l.Token())); return int(T_DECLARE)
<PHP>default lval.Token(l.createToken(l.Token())); return T_DEFAULT <PHP>default lval.Token(l.createToken(l.Token())); return int(T_DEFAULT)
<PHP>do lval.Token(l.createToken(l.Token())); return T_DO <PHP>do lval.Token(l.createToken(l.Token())); return int(T_DO)
<PHP>echo lval.Token(l.createToken(l.Token())); return T_ECHO <PHP>echo lval.Token(l.createToken(l.Token())); return int(T_ECHO)
<PHP>else lval.Token(l.createToken(l.Token())); return T_ELSE <PHP>else lval.Token(l.createToken(l.Token())); return int(T_ELSE)
<PHP>elseif lval.Token(l.createToken(l.Token())); return T_ELSEIF <PHP>elseif lval.Token(l.createToken(l.Token())); return int(T_ELSEIF)
<PHP>empty lval.Token(l.createToken(l.Token())); return T_EMPTY <PHP>empty lval.Token(l.createToken(l.Token())); return int(T_EMPTY)
<PHP>enddeclare lval.Token(l.createToken(l.Token())); return T_ENDDECLARE <PHP>enddeclare lval.Token(l.createToken(l.Token())); return int(T_ENDDECLARE)
<PHP>endfor lval.Token(l.createToken(l.Token())); return T_ENDFOR <PHP>endfor lval.Token(l.createToken(l.Token())); return int(T_ENDFOR)
<PHP>endforeach lval.Token(l.createToken(l.Token())); return T_ENDFOREACH <PHP>endforeach lval.Token(l.createToken(l.Token())); return int(T_ENDFOREACH)
<PHP>endif lval.Token(l.createToken(l.Token())); return T_ENDIF <PHP>endif lval.Token(l.createToken(l.Token())); return int(T_ENDIF)
<PHP>endswitch lval.Token(l.createToken(l.Token())); return T_ENDSWITCH <PHP>endswitch lval.Token(l.createToken(l.Token())); return int(T_ENDSWITCH)
<PHP>endwhile lval.Token(l.createToken(l.Token())); return T_ENDWHILE <PHP>endwhile lval.Token(l.createToken(l.Token())); return int(T_ENDWHILE)
<PHP>eval lval.Token(l.createToken(l.Token())); return T_EVAL <PHP>eval lval.Token(l.createToken(l.Token())); return int(T_EVAL)
<PHP>exit|die lval.Token(l.createToken(l.Token())); return T_EXIT <PHP>exit|die lval.Token(l.createToken(l.Token())); return int(T_EXIT)
<PHP>extends lval.Token(l.createToken(l.Token())); return T_EXTENDS <PHP>extends lval.Token(l.createToken(l.Token())); return int(T_EXTENDS)
<PHP>final lval.Token(l.createToken(l.Token())); return T_FINAL <PHP>final lval.Token(l.createToken(l.Token())); return int(T_FINAL)
<PHP>finally lval.Token(l.createToken(l.Token())); return T_FINALLY <PHP>finally lval.Token(l.createToken(l.Token())); return int(T_FINALLY)
<PHP>for lval.Token(l.createToken(l.Token())); return T_FOR <PHP>for lval.Token(l.createToken(l.Token())); return int(T_FOR)
<PHP>foreach lval.Token(l.createToken(l.Token())); return T_FOREACH <PHP>foreach lval.Token(l.createToken(l.Token())); return int(T_FOREACH)
<PHP>function|cfunction lval.Token(l.createToken(l.Token())); return T_FUNCTION <PHP>function|cfunction lval.Token(l.createToken(l.Token())); return int(T_FUNCTION)
<PHP>global lval.Token(l.createToken(l.Token())); return T_GLOBAL <PHP>global lval.Token(l.createToken(l.Token())); return int(T_GLOBAL)
<PHP>goto lval.Token(l.createToken(l.Token())); return T_GOTO <PHP>goto lval.Token(l.createToken(l.Token())); return int(T_GOTO)
<PHP>if lval.Token(l.createToken(l.Token())); return T_IF <PHP>if lval.Token(l.createToken(l.Token())); return int(T_IF)
<PHP>isset lval.Token(l.createToken(l.Token())); return T_ISSET <PHP>isset lval.Token(l.createToken(l.Token())); return int(T_ISSET)
<PHP>implements lval.Token(l.createToken(l.Token())); return T_IMPLEMENTS <PHP>implements lval.Token(l.createToken(l.Token())); return int(T_IMPLEMENTS)
<PHP>instanceof lval.Token(l.createToken(l.Token())); return T_INSTANCEOF <PHP>instanceof lval.Token(l.createToken(l.Token())); return int(T_INSTANCEOF)
<PHP>insteadof lval.Token(l.createToken(l.Token())); return T_INSTEADOF <PHP>insteadof lval.Token(l.createToken(l.Token())); return int(T_INSTEADOF)
<PHP>interface lval.Token(l.createToken(l.Token())); return T_INTERFACE <PHP>interface lval.Token(l.createToken(l.Token())); return int(T_INTERFACE)
<PHP>list lval.Token(l.createToken(l.Token())); return T_LIST <PHP>list lval.Token(l.createToken(l.Token())); return int(T_LIST)
<PHP>namespace lval.Token(l.createToken(l.Token())); return T_NAMESPACE <PHP>namespace lval.Token(l.createToken(l.Token())); return int(T_NAMESPACE)
<PHP>private lval.Token(l.createToken(l.Token())); return T_PRIVATE <PHP>private lval.Token(l.createToken(l.Token())); return int(T_PRIVATE)
<PHP>public lval.Token(l.createToken(l.Token())); return T_PUBLIC <PHP>public lval.Token(l.createToken(l.Token())); return int(T_PUBLIC)
<PHP>print lval.Token(l.createToken(l.Token())); return T_PRINT <PHP>print lval.Token(l.createToken(l.Token())); return int(T_PRINT)
<PHP>protected lval.Token(l.createToken(l.Token())); return T_PROTECTED <PHP>protected lval.Token(l.createToken(l.Token())); return int(T_PROTECTED)
<PHP>return lval.Token(l.createToken(l.Token())); return T_RETURN <PHP>return lval.Token(l.createToken(l.Token())); return int(T_RETURN)
<PHP>static lval.Token(l.createToken(l.Token())); return T_STATIC <PHP>static lval.Token(l.createToken(l.Token())); return int(T_STATIC)
<PHP>switch lval.Token(l.createToken(l.Token())); return T_SWITCH <PHP>switch lval.Token(l.createToken(l.Token())); return int(T_SWITCH)
<PHP>throw lval.Token(l.createToken(l.Token())); return T_THROW <PHP>throw lval.Token(l.createToken(l.Token())); return int(T_THROW)
<PHP>trait lval.Token(l.createToken(l.Token())); return T_TRAIT <PHP>trait lval.Token(l.createToken(l.Token())); return int(T_TRAIT)
<PHP>try lval.Token(l.createToken(l.Token())); return T_TRY <PHP>try lval.Token(l.createToken(l.Token())); return int(T_TRY)
<PHP>unset lval.Token(l.createToken(l.Token())); return T_UNSET <PHP>unset lval.Token(l.createToken(l.Token())); return int(T_UNSET)
<PHP>use lval.Token(l.createToken(l.Token())); return T_USE <PHP>use lval.Token(l.createToken(l.Token())); return int(T_USE)
<PHP>var lval.Token(l.createToken(l.Token())); return T_VAR <PHP>var lval.Token(l.createToken(l.Token())); return int(T_VAR)
<PHP>while lval.Token(l.createToken(l.Token())); return T_WHILE <PHP>while lval.Token(l.createToken(l.Token())); return int(T_WHILE)
<PHP>yield[ \t\n\r]+from lval.Token(l.createToken(l.Token())); return T_YIELD_FROM <PHP>yield[ \t\n\r]+from lval.Token(l.createToken(l.Token())); return int(T_YIELD_FROM)
<PHP>yield lval.Token(l.createToken(l.Token())); return T_YIELD <PHP>yield lval.Token(l.createToken(l.Token())); return int(T_YIELD)
<PHP>include lval.Token(l.createToken(l.Token())); return T_INCLUDE <PHP>include lval.Token(l.createToken(l.Token())); return int(T_INCLUDE)
<PHP>include_once lval.Token(l.createToken(l.Token())); return T_INCLUDE_ONCE <PHP>include_once lval.Token(l.createToken(l.Token())); return int(T_INCLUDE_ONCE)
<PHP>require lval.Token(l.createToken(l.Token())); return T_REQUIRE <PHP>require lval.Token(l.createToken(l.Token())); return int(T_REQUIRE)
<PHP>require_once lval.Token(l.createToken(l.Token())); return T_REQUIRE_ONCE <PHP>require_once lval.Token(l.createToken(l.Token())); return int(T_REQUIRE_ONCE)
<PHP>__CLASS__ lval.Token(l.createToken(l.Token())); return T_CLASS_C <PHP>__CLASS__ lval.Token(l.createToken(l.Token())); return int(T_CLASS_C)
<PHP>__DIR__ lval.Token(l.createToken(l.Token())); return T_DIR <PHP>__DIR__ lval.Token(l.createToken(l.Token())); return int(T_DIR)
<PHP>__FILE__ lval.Token(l.createToken(l.Token())); return T_FILE <PHP>__FILE__ lval.Token(l.createToken(l.Token())); return int(T_FILE)
<PHP>__FUNCTION__ lval.Token(l.createToken(l.Token())); return T_FUNC_C <PHP>__FUNCTION__ lval.Token(l.createToken(l.Token())); return int(T_FUNC_C)
<PHP>__LINE__ lval.Token(l.createToken(l.Token())); return T_LINE <PHP>__LINE__ lval.Token(l.createToken(l.Token())); return int(T_LINE)
<PHP>__NAMESPACE__ lval.Token(l.createToken(l.Token())); return T_NS_C <PHP>__NAMESPACE__ lval.Token(l.createToken(l.Token())); return int(T_NS_C)
<PHP>__METHOD__ lval.Token(l.createToken(l.Token())); return T_METHOD_C <PHP>__METHOD__ lval.Token(l.createToken(l.Token())); return int(T_METHOD_C)
<PHP>__TRAIT__ lval.Token(l.createToken(l.Token())); return T_TRAIT_C <PHP>__TRAIT__ lval.Token(l.createToken(l.Token())); return int(T_TRAIT_C)
<PHP>__halt_compiler lval.Token(l.createToken(l.Token())); return T_HALT_COMPILER <PHP>__halt_compiler lval.Token(l.createToken(l.Token())); return int(T_HALT_COMPILER)
<PHP>\([ \t]*array[ \t]*\) lval.Token(l.createToken(l.Token())); return T_ARRAY_CAST <PHP>\([ \t]*array[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_ARRAY_CAST)
<PHP>\([ \t]*(bool|boolean)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_BOOL_CAST <PHP>\([ \t]*(bool|boolean)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_BOOL_CAST)
<PHP>\([ \t]*(real|double|float)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_DOUBLE_CAST <PHP>\([ \t]*(real|double|float)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_DOUBLE_CAST)
<PHP>\([ \t]*(int|integer)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_INT_CAST <PHP>\([ \t]*(int|integer)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_INT_CAST)
<PHP>\([ \t]*object[ \t]*\) lval.Token(l.createToken(l.Token())); return T_OBJECT_CAST <PHP>\([ \t]*object[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_OBJECT_CAST)
<PHP>\([ \t]*(string|binary)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_STRING_CAST <PHP>\([ \t]*(string|binary)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_STRING_CAST)
<PHP>\([ \t]*unset[ \t]*\) lval.Token(l.createToken(l.Token())); return T_UNSET_CAST <PHP>\([ \t]*unset[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_UNSET_CAST)
<PHP>new lval.Token(l.createToken(l.Token())); return T_NEW <PHP>new lval.Token(l.createToken(l.Token())); return int(T_NEW)
<PHP>and lval.Token(l.createToken(l.Token())); return T_LOGICAL_AND <PHP>and lval.Token(l.createToken(l.Token())); return int(T_LOGICAL_AND)
<PHP>or lval.Token(l.createToken(l.Token())); return T_LOGICAL_OR <PHP>or lval.Token(l.createToken(l.Token())); return int(T_LOGICAL_OR)
<PHP>xor lval.Token(l.createToken(l.Token())); return T_LOGICAL_XOR <PHP>xor lval.Token(l.createToken(l.Token())); return int(T_LOGICAL_XOR)
<PHP>\\ lval.Token(l.createToken(l.Token())); return T_NS_SEPARATOR <PHP>\\ lval.Token(l.createToken(l.Token())); return int(T_NS_SEPARATOR)
<PHP>\.\.\. lval.Token(l.createToken(l.Token())); return T_ELLIPSIS <PHP>\.\.\. lval.Token(l.createToken(l.Token())); return int(T_ELLIPSIS)
<PHP>:: lval.Token(l.createToken(l.Token())); return T_PAAMAYIM_NEKUDOTAYIM // T_DOUBLE_COLON <PHP>:: lval.Token(l.createToken(l.Token())); return int(T_PAAMAYIM_NEKUDOTAYIM) // T_DOUBLE_COLON
<PHP>&& lval.Token(l.createToken(l.Token())); return T_BOOLEAN_AND <PHP>&& lval.Token(l.createToken(l.Token())); return int(T_BOOLEAN_AND)
<PHP>\|\| lval.Token(l.createToken(l.Token())); return T_BOOLEAN_OR <PHP>\|\| lval.Token(l.createToken(l.Token())); return int(T_BOOLEAN_OR)
<PHP>&= lval.Token(l.createToken(l.Token())); return T_AND_EQUAL <PHP>&= lval.Token(l.createToken(l.Token())); return int(T_AND_EQUAL)
<PHP>\|= lval.Token(l.createToken(l.Token())); return T_OR_EQUAL <PHP>\|= lval.Token(l.createToken(l.Token())); return int(T_OR_EQUAL)
<PHP>\.= lval.Token(l.createToken(l.Token())); return T_CONCAT_EQUAL <PHP>\.= lval.Token(l.createToken(l.Token())); return int(T_CONCAT_EQUAL)
<PHP>\*= lval.Token(l.createToken(l.Token())); return T_MUL_EQUAL <PHP>\*= lval.Token(l.createToken(l.Token())); return int(T_MUL_EQUAL)
<PHP>\*\*= lval.Token(l.createToken(l.Token())); return T_POW_EQUAL <PHP>\*\*= lval.Token(l.createToken(l.Token())); return int(T_POW_EQUAL)
<PHP>[/]= lval.Token(l.createToken(l.Token())); return T_DIV_EQUAL <PHP>[/]= lval.Token(l.createToken(l.Token())); return int(T_DIV_EQUAL)
<PHP>\+= lval.Token(l.createToken(l.Token())); return T_PLUS_EQUAL <PHP>\+= lval.Token(l.createToken(l.Token())); return int(T_PLUS_EQUAL)
<PHP>-= lval.Token(l.createToken(l.Token())); return T_MINUS_EQUAL <PHP>-= lval.Token(l.createToken(l.Token())); return int(T_MINUS_EQUAL)
<PHP>\^= lval.Token(l.createToken(l.Token())); return T_XOR_EQUAL <PHP>\^= lval.Token(l.createToken(l.Token())); return int(T_XOR_EQUAL)
<PHP>%= lval.Token(l.createToken(l.Token())); return T_MOD_EQUAL <PHP>%= lval.Token(l.createToken(l.Token())); return int(T_MOD_EQUAL)
<PHP>-- lval.Token(l.createToken(l.Token())); return T_DEC <PHP>-- lval.Token(l.createToken(l.Token())); return int(T_DEC)
<PHP>\+\+ lval.Token(l.createToken(l.Token())); return T_INC <PHP>\+\+ lval.Token(l.createToken(l.Token())); return int(T_INC)
<PHP>=> lval.Token(l.createToken(l.Token())); return T_DOUBLE_ARROW <PHP>=> lval.Token(l.createToken(l.Token())); return int(T_DOUBLE_ARROW)
<PHP>\<=\> lval.Token(l.createToken(l.Token())); return T_SPACESHIP <PHP>\<=\> lval.Token(l.createToken(l.Token())); return int(T_SPACESHIP)
<PHP>\!=|\<\> lval.Token(l.createToken(l.Token())); return T_IS_NOT_EQUAL <PHP>\!=|\<\> lval.Token(l.createToken(l.Token())); return int(T_IS_NOT_EQUAL)
<PHP>\!== lval.Token(l.createToken(l.Token())); return T_IS_NOT_IDENTICAL <PHP>\!== lval.Token(l.createToken(l.Token())); return int(T_IS_NOT_IDENTICAL)
<PHP>== lval.Token(l.createToken(l.Token())); return T_IS_EQUAL <PHP>== lval.Token(l.createToken(l.Token())); return int(T_IS_EQUAL)
<PHP>=== lval.Token(l.createToken(l.Token())); return T_IS_IDENTICAL <PHP>=== lval.Token(l.createToken(l.Token())); return int(T_IS_IDENTICAL)
<PHP>\<\<= lval.Token(l.createToken(l.Token())); return T_SL_EQUAL <PHP>\<\<= lval.Token(l.createToken(l.Token())); return int(T_SL_EQUAL)
<PHP>\>\>= lval.Token(l.createToken(l.Token())); return T_SR_EQUAL <PHP>\>\>= lval.Token(l.createToken(l.Token())); return int(T_SR_EQUAL)
<PHP>\>= lval.Token(l.createToken(l.Token())); return T_IS_GREATER_OR_EQUAL <PHP>\>= lval.Token(l.createToken(l.Token())); return int(T_IS_GREATER_OR_EQUAL)
<PHP>\<= lval.Token(l.createToken(l.Token())); return T_IS_SMALLER_OR_EQUAL <PHP>\<= lval.Token(l.createToken(l.Token())); return int(T_IS_SMALLER_OR_EQUAL)
<PHP>\*\* lval.Token(l.createToken(l.Token())); return T_POW <PHP>\*\* lval.Token(l.createToken(l.Token())); return int(T_POW)
<PHP>\<\< lval.Token(l.createToken(l.Token())); return T_SL <PHP>\<\< lval.Token(l.createToken(l.Token())); return int(T_SL)
<PHP>\>\> lval.Token(l.createToken(l.Token())); return T_SR <PHP>\>\> lval.Token(l.createToken(l.Token())); return int(T_SR)
<PHP>\?\? lval.Token(l.createToken(l.Token())); return T_COALESCE <PHP>\?\? lval.Token(l.createToken(l.Token())); return int(T_COALESCE)
<PHP>(#|[/][/]) <PHP>(#|[/][/])
tb := l.Token() tb := l.Token()
@ -319,16 +319,16 @@ NEW_LINE (\r|\n|\r\n)
<PHP>\{ l.pushState(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) <PHP>\{ l.pushState(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<PHP>\} l.popState(); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])); l.PhpDocComment = "" <PHP>\} l.popState(); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])); l.PhpDocComment = ""
<PHP>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return T_VARIABLE <PHP>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_VARIABLE)
<PHP>{VAR_NAME} lval.Token(l.createToken(l.Token())); return T_STRING <PHP>{VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_STRING)
<PHP>-> l.begin(PROPERTY);lval.Token(l.createToken(l.Token())); return T_OBJECT_OPERATOR; <PHP>-> l.begin(PROPERTY);lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR);
<PROPERTY>[ \t\n\r]+ l.addWhiteSpace(l.Token()) <PROPERTY>[ \t\n\r]+ l.addWhiteSpace(l.Token())
<PROPERTY>-> lval.Token(l.createToken(l.Token())); return T_OBJECT_OPERATOR; <PROPERTY>-> lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR);
<PROPERTY>{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return T_STRING; <PROPERTY>{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_STRING);
<PROPERTY>. l.ungetChars(1);l.begin(PHP) <PROPERTY>. l.ungetChars(1);l.begin(PHP)
<PHP>[\']([^\\\']*([\\].)*)*[\'] lval.Token(l.createToken(l.Token())); return T_CONSTANT_ENCAPSED_STRING; <PHP>[\']([^\\\']*([\\].)*)*[\'] lval.Token(l.createToken(l.Token())); return int(T_CONSTANT_ENCAPSED_STRING);
<PHP>` l.begin(BACKQUOTE); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) <PHP>` l.begin(BACKQUOTE); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<BACKQUOTE>` l.begin(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) <BACKQUOTE>` l.begin(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
@ -394,7 +394,7 @@ NEW_LINE (\r|\n|\r\n)
l.ungetChars(ungetCnt) l.ungetChars(ungetCnt)
lval.Token(l.createToken(heredocToken)); lval.Token(l.createToken(heredocToken));
return T_START_HEREDOC return int(T_START_HEREDOC)
<NOWDOC>.|[ \t\n\r] <NOWDOC>.|[ \t\n\r]
searchLabel := []byte{} searchLabel := []byte{}
@ -409,12 +409,14 @@ NEW_LINE (\r|\n|\r\n)
if l.heredocLabel + ";" == string(searchLabel) { if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+1) tb = l.ungetChars(len(l.heredocLabel)+1)
tb = tb[:len(tb)-1]
break; break;
} }
if l.heredocLabel == string(searchLabel) { if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)) tb = l.ungetChars(len(l.heredocLabel))
tb = tb[:len(tb)-1]
break; break;
} }
@ -427,10 +429,10 @@ NEW_LINE (\r|\n|\r\n)
} }
lval.Token(l.createToken(tb) ) lval.Token(l.createToken(tb) )
return T_ENCAPSED_AND_WHITESPACE return int(T_ENCAPSED_AND_WHITESPACE)
<HEREDOC_END>{VAR_NAME}\; l.begin(PHP);lval.Token(l.createToken(l.ungetChars(1))); return T_END_HEREDOC <HEREDOC_END>{VAR_NAME}\; l.begin(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_END_HEREDOC)
<HEREDOC_END>{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return T_END_HEREDOC <HEREDOC_END>{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_END_HEREDOC)
<PHP>[b]?[\"] <PHP>[b]?[\"]
binPrefix := l.Token()[0].Rune == 'b' binPrefix := l.Token()[0].Rune == 'b'
@ -453,7 +455,7 @@ NEW_LINE (\r|\n|\r\n)
switch c { switch c {
case '"' : case '"' :
c = l.Next(); c = l.Next();
lval.Token(l.createToken(l.Token())); return T_CONSTANT_ENCAPSED_STRING lval.Token(l.createToken(l.Token())); return int(T_CONSTANT_ENCAPSED_STRING)
break F; break F;
case '$': case '$':
@ -480,8 +482,8 @@ NEW_LINE (\r|\n|\r\n)
} }
<STRING>\" l.popState(); lval.Token(l.createToken(l.Token())); return Rune2Class(l.Token()[0].Rune) <STRING>\" l.popState(); lval.Token(l.createToken(l.Token())); return Rune2Class(l.Token()[0].Rune)
<STRING,HEREDOC,BACKQUOTE>\{\$ lval.Token(l.createToken(l.ungetChars(1))); l.pushState(PHP); return T_CURLY_OPEN <STRING,HEREDOC,BACKQUOTE>\{\$ lval.Token(l.createToken(l.ungetChars(1))); l.pushState(PHP); return int(T_CURLY_OPEN)
<STRING,HEREDOC,BACKQUOTE>\$\{ l.pushState(STRING_VAR_NAME); lval.Token(l.createToken(l.Token())); return T_DOLLAR_OPEN_CURLY_BRACES <STRING,HEREDOC,BACKQUOTE>\$\{ l.pushState(STRING_VAR_NAME); lval.Token(l.createToken(l.Token())); return int(T_DOLLAR_OPEN_CURLY_BRACES)
<STRING,HEREDOC,BACKQUOTE>\${VAR_NAME} l.ungetChars(len(l.Token()));l.pushState(STRING_VAR) <STRING,HEREDOC,BACKQUOTE>\${VAR_NAME} l.ungetChars(len(l.Token()));l.pushState(STRING_VAR)
<STRING>.|[ \t\n\r] <STRING>.|[ \t\n\r]
currentChar := l.Prev currentChar := l.Prev
@ -492,14 +494,14 @@ NEW_LINE (\r|\n|\r\n)
if c == '{' || isValidFirstVarNameRune(rune(c)) { if c == '{' || isValidFirstVarNameRune(rune(c)) {
l.ungetChars(1) l.ungetChars(1)
lval.Token(l.createToken(tb[:len(tb)-1])); lval.Token(l.createToken(tb[:len(tb)-1]));
return T_ENCAPSED_AND_WHITESPACE return int(T_ENCAPSED_AND_WHITESPACE)
} }
case '{': case '{':
if rune(c) == '$' { if rune(c) == '$' {
l.ungetChars(1) l.ungetChars(1)
lval.Token(l.createToken(tb[:len(tb)-1])); lval.Token(l.createToken(tb[:len(tb)-1]));
return T_ENCAPSED_AND_WHITESPACE return int(T_ENCAPSED_AND_WHITESPACE)
} }
case '\\': case '\\':
@ -510,7 +512,7 @@ NEW_LINE (\r|\n|\r\n)
if rune(c) == '"' { if rune(c) == '"' {
lval.Token(l.createToken(l.Token())); lval.Token(l.createToken(l.Token()));
return T_ENCAPSED_AND_WHITESPACE return int(T_ENCAPSED_AND_WHITESPACE)
} }
currentChar = l.Last currentChar = l.Last
@ -532,14 +534,14 @@ NEW_LINE (\r|\n|\r\n)
if c == '{' || isValidFirstVarNameRune(rune(c)) { if c == '{' || isValidFirstVarNameRune(rune(c)) {
l.ungetChars(1) l.ungetChars(1)
lval.Token(l.createToken(tb[:len(tb)-1])); lval.Token(l.createToken(tb[:len(tb)-1]));
return T_ENCAPSED_AND_WHITESPACE return int(T_ENCAPSED_AND_WHITESPACE)
} }
case '{': case '{':
if rune(c) == '$' { if rune(c) == '$' {
l.ungetChars(1) l.ungetChars(1)
lval.Token(l.createToken(tb[:len(tb)-1])); lval.Token(l.createToken(tb[:len(tb)-1]));
return T_ENCAPSED_AND_WHITESPACE return int(T_ENCAPSED_AND_WHITESPACE)
} }
case '\\': case '\\':
@ -550,7 +552,7 @@ NEW_LINE (\r|\n|\r\n)
if rune(c) == '`' { if rune(c) == '`' {
lval.Token(l.createToken(l.Token())); lval.Token(l.createToken(l.Token()));
return T_ENCAPSED_AND_WHITESPACE return int(T_ENCAPSED_AND_WHITESPACE)
} }
currentChar = l.Last currentChar = l.Last
@ -564,23 +566,20 @@ NEW_LINE (\r|\n|\r\n)
<HEREDOC>.|[ \t\n\r] <HEREDOC>.|[ \t\n\r]
searchLabel := []byte{} searchLabel := []byte{}
tb := []lex.Char{} currentChar := l.Prev
tb := []lex.Char{currentChar}
for {
if c == -1 {
break;
}
HEREDOC_FOR:for {
nls := 0 nls := 0
switch c { switch currentChar.Rune {
case '\r': case '\r':
nls = 1
c := l.Next()
if c != '\n' { if c == '\n' {
nls = 0 nls = 1
l.ungetChars(0) currentChar := l.Last
tb = append(tb, currentChar)
c = l.Next();
} }
fallthrough fallthrough
@ -588,66 +587,84 @@ NEW_LINE (\r|\n|\r\n)
case '\n': case '\n':
if l.heredocLabel + ";" == string(searchLabel) { if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+1+nls) l.ungetChars(len(l.heredocLabel)+1+nls)
i := len(tb) - len(l.heredocLabel) - 3 - nls
if i < 1 {
break HEREDOC_FOR;
}
tb = tb[:i]
lval.Token(l.createToken(tb)); lval.Token(l.createToken(tb));
return T_ENCAPSED_AND_WHITESPACE return int(T_ENCAPSED_AND_WHITESPACE)
} }
if l.heredocLabel == string(searchLabel) { if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END) l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+nls) l.ungetChars(len(l.heredocLabel)+nls)
i := len(tb) - len(l.heredocLabel) - 2 - nls
if i < 1 {
break HEREDOC_FOR;
}
tb = tb[:i]
lval.Token(l.createToken(tb)); lval.Token(l.createToken(tb));
return T_ENCAPSED_AND_WHITESPACE return int(T_ENCAPSED_AND_WHITESPACE)
} }
searchLabel = []byte{} searchLabel = []byte{}
case '$': case '$':
c = l.Next(); if c == '{' || isValidFirstVarNameRune(rune(c)) {
if rune(c) == '{' || isValidFirstVarNameRune(rune(c)) { l.ungetChars(1)
tb = l.ungetChars(1) lval.Token(l.createToken(tb[:len(tb)-1]));
lval.Token(l.createToken(tb)); return int(T_ENCAPSED_AND_WHITESPACE)
return T_ENCAPSED_AND_WHITESPACE
} }
l.ungetChars(0)
case '{': case '{':
c = l.Next();
if rune(c) == '$' { if rune(c) == '$' {
tb = l.ungetChars(1) l.ungetChars(1)
lval.Token(l.createToken(tb)); lval.Token(l.createToken(tb[:len(tb)-1]));
return T_ENCAPSED_AND_WHITESPACE return int(T_ENCAPSED_AND_WHITESPACE)
} }
l.ungetChars(0)
case '\\': case '\\':
c = l.Next(); if c != '\n' && c != '\r' {
if c == '\n' || c == '\r' { currentChar := l.Last
l.ungetChars(0) tb = append(tb, currentChar)
c = l.Next();
} }
default: default:
searchLabel = append(searchLabel, byte(rune(c))) searchLabel = append(searchLabel, byte(rune(currentChar.Rune)))
} }
if c == -1 {
break;
}
currentChar = l.Last
tb = append(tb, currentChar)
c = l.Next() c = l.Next()
} }
<STRING_VAR>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return T_VARIABLE <STRING_VAR>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_VARIABLE)
<STRING_VAR>->{VAR_NAME} lval.Token(l.createToken(l.ungetChars(len(l.Token())-2))); return T_OBJECT_OPERATOR <STRING_VAR>->{VAR_NAME} lval.Token(l.createToken(l.ungetChars(len(l.Token())-2))); return int(T_OBJECT_OPERATOR)
<STRING_VAR>{VAR_NAME} l.popState();lval.Token(l.createToken(l.Token())); return T_STRING <STRING_VAR>{VAR_NAME} l.popState();lval.Token(l.createToken(l.Token())); return int(T_STRING)
<STRING_VAR>\[ l.pushState(STRING_VAR_INDEX);lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) <STRING_VAR>\[ l.pushState(STRING_VAR_INDEX);lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR>.|[ \t\n\r] l.ungetChars(1);l.popState() <STRING_VAR>.|[ \t\n\r] l.ungetChars(1);l.popState()
<STRING_VAR_INDEX>{LNUM}|{HNUM}|{BNUM} lval.Token(l.createToken(l.Token())); return T_NUM_STRING <STRING_VAR_INDEX>{LNUM}|{HNUM}|{BNUM} lval.Token(l.createToken(l.Token())); return int(T_NUM_STRING)
<STRING_VAR_INDEX>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return T_VARIABLE <STRING_VAR_INDEX>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_VARIABLE)
<STRING_VAR_INDEX>{VAR_NAME} lval.Token(l.createToken(l.Token())); return T_STRING <STRING_VAR_INDEX>{VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_STRING)
<STRING_VAR_INDEX>\] l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) <STRING_VAR_INDEX>\] l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_INDEX>[ \n\r\t\\'#] l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return T_ENCAPSED_AND_WHITESPACE <STRING_VAR_INDEX>[ \n\r\t\\'#] l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return int(T_ENCAPSED_AND_WHITESPACE)
<STRING_VAR_INDEX>{OPERATORS} lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) <STRING_VAR_INDEX>{OPERATORS} lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_INDEX>. lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])) <STRING_VAR_INDEX>. lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_NAME>{VAR_NAME}[\[\}] l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return T_STRING_VARNAME <STRING_VAR_NAME>{VAR_NAME}[\[\}] l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_STRING_VARNAME)
<STRING_VAR_NAME>. l.ungetChars(1);l.popState();l.pushState(PHP) <STRING_VAR_NAME>. l.ungetChars(1);l.popState();l.pushState(PHP)
%% %%

780
scanner/scanner_test.go
View File

@ -230,194 +230,194 @@ func TestTokens(t *testing.T) {
` `
expected := []int{ expected := []string{
scanner.T_INLINE_HTML, scanner.T_INLINE_HTML.String(),
scanner.Rune2Class(';'), scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_ECHO, scanner.T_ECHO.String(),
scanner.Rune2Class(';'), scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_DNUMBER, scanner.T_DNUMBER.String(),
scanner.T_DNUMBER, scanner.T_DNUMBER.String(),
scanner.T_DNUMBER, scanner.T_DNUMBER.String(),
scanner.T_DNUMBER, scanner.T_DNUMBER.String(),
scanner.T_LNUMBER, scanner.T_LNUMBER.String(),
scanner.T_DNUMBER, scanner.T_DNUMBER.String(),
scanner.T_LNUMBER, scanner.T_LNUMBER.String(),
scanner.T_DNUMBER, scanner.T_DNUMBER.String(),
scanner.T_LNUMBER, scanner.T_LNUMBER.String(),
scanner.T_DNUMBER, scanner.T_DNUMBER.String(),
scanner.T_ABSTRACT, scanner.T_ABSTRACT.String(),
scanner.T_ARRAY, scanner.T_ARRAY.String(),
scanner.T_AS, scanner.T_AS.String(),
scanner.T_BREAK, scanner.T_BREAK.String(),
scanner.T_CALLABLE, scanner.T_CALLABLE.String(),
scanner.T_CASE, scanner.T_CASE.String(),
scanner.T_CATCH, scanner.T_CATCH.String(),
scanner.T_CLASS, scanner.T_CLASS.String(),
scanner.T_CLONE, scanner.T_CLONE.String(),
scanner.T_CONST, scanner.T_CONST.String(),
scanner.T_CONTINUE, scanner.T_CONTINUE.String(),
scanner.T_DECLARE, scanner.T_DECLARE.String(),
scanner.T_DEFAULT, scanner.T_DEFAULT.String(),
scanner.T_DO, scanner.T_DO.String(),
scanner.T_ECHO, scanner.T_ECHO.String(),
scanner.T_ELSE, scanner.T_ELSE.String(),
scanner.T_ELSEIF, scanner.T_ELSEIF.String(),
scanner.T_EMPTY, scanner.T_EMPTY.String(),
scanner.T_ENDDECLARE, scanner.T_ENDDECLARE.String(),
scanner.T_ENDFOR, scanner.T_ENDFOR.String(),
scanner.T_ENDFOREACH, scanner.T_ENDFOREACH.String(),
scanner.T_ENDIF, scanner.T_ENDIF.String(),
scanner.T_ENDSWITCH, scanner.T_ENDSWITCH.String(),
scanner.T_ENDWHILE, scanner.T_ENDWHILE.String(),
scanner.T_EVAL, scanner.T_EVAL.String(),
scanner.T_EXIT, scanner.T_EXIT.String(),
scanner.T_EXTENDS, scanner.T_EXTENDS.String(),
scanner.T_FINAL, scanner.T_FINAL.String(),
scanner.T_FINALLY, scanner.T_FINALLY.String(),
scanner.T_FOR, scanner.T_FOR.String(),
scanner.T_FOREACH, scanner.T_FOREACH.String(),
scanner.T_FUNCTION, scanner.T_FUNCTION.String(),
scanner.T_FUNCTION, scanner.T_FUNCTION.String(),
scanner.T_GLOBAL, scanner.T_GLOBAL.String(),
scanner.T_GOTO, scanner.T_GOTO.String(),
scanner.T_IF, scanner.T_IF.String(),
scanner.T_ISSET, scanner.T_ISSET.String(),
scanner.T_IMPLEMENTS, scanner.T_IMPLEMENTS.String(),
scanner.T_INSTANCEOF, scanner.T_INSTANCEOF.String(),
scanner.T_INSTEADOF, scanner.T_INSTEADOF.String(),
scanner.T_INTERFACE, scanner.T_INTERFACE.String(),
scanner.T_LIST, scanner.T_LIST.String(),
scanner.T_NAMESPACE, scanner.T_NAMESPACE.String(),
scanner.T_PRIVATE, scanner.T_PRIVATE.String(),
scanner.T_PUBLIC, scanner.T_PUBLIC.String(),
scanner.T_PRINT, scanner.T_PRINT.String(),
scanner.T_PROTECTED, scanner.T_PROTECTED.String(),
scanner.T_RETURN, scanner.T_RETURN.String(),
scanner.T_STATIC, scanner.T_STATIC.String(),
scanner.T_SWITCH, scanner.T_SWITCH.String(),
scanner.T_THROW, scanner.T_THROW.String(),
scanner.T_TRAIT, scanner.T_TRAIT.String(),
scanner.T_TRY, scanner.T_TRY.String(),
scanner.T_UNSET, scanner.T_UNSET.String(),
scanner.T_USE, scanner.T_USE.String(),
scanner.T_VAR, scanner.T_VAR.String(),
scanner.T_WHILE, scanner.T_WHILE.String(),
scanner.T_YIELD_FROM, scanner.T_YIELD_FROM.String(),
scanner.T_YIELD, scanner.T_YIELD.String(),
scanner.T_INCLUDE, scanner.T_INCLUDE.String(),
scanner.T_INCLUDE_ONCE, scanner.T_INCLUDE_ONCE.String(),
scanner.T_REQUIRE, scanner.T_REQUIRE.String(),
scanner.T_REQUIRE_ONCE, scanner.T_REQUIRE_ONCE.String(),
scanner.T_CLASS_C, scanner.T_CLASS_C.String(),
scanner.T_DIR, scanner.T_DIR.String(),
scanner.T_FILE, scanner.T_FILE.String(),
scanner.T_FUNC_C, scanner.T_FUNC_C.String(),
scanner.T_LINE, scanner.T_LINE.String(),
scanner.T_NS_C, scanner.T_NS_C.String(),
scanner.T_METHOD_C, scanner.T_METHOD_C.String(),
scanner.T_TRAIT_C, scanner.T_TRAIT_C.String(),
scanner.T_HALT_COMPILER, scanner.T_HALT_COMPILER.String(),
scanner.T_NEW, scanner.T_NEW.String(),
scanner.T_LOGICAL_AND, scanner.T_LOGICAL_AND.String(),
scanner.T_LOGICAL_OR, scanner.T_LOGICAL_OR.String(),
scanner.T_LOGICAL_XOR, scanner.T_LOGICAL_XOR.String(),
scanner.T_NS_SEPARATOR, scanner.T_NS_SEPARATOR.String(),
scanner.T_ELLIPSIS, scanner.T_ELLIPSIS.String(),
scanner.T_PAAMAYIM_NEKUDOTAYIM, scanner.T_PAAMAYIM_NEKUDOTAYIM.String(),
scanner.T_BOOLEAN_AND, scanner.T_BOOLEAN_AND.String(),
scanner.T_BOOLEAN_OR, scanner.T_BOOLEAN_OR.String(),
scanner.T_AND_EQUAL, scanner.T_AND_EQUAL.String(),
scanner.T_OR_EQUAL, scanner.T_OR_EQUAL.String(),
scanner.T_CONCAT_EQUAL, scanner.T_CONCAT_EQUAL.String(),
scanner.T_MUL_EQUAL, scanner.T_MUL_EQUAL.String(),
scanner.T_POW_EQUAL, scanner.T_POW_EQUAL.String(),
scanner.T_DIV_EQUAL, scanner.T_DIV_EQUAL.String(),
scanner.T_PLUS_EQUAL, scanner.T_PLUS_EQUAL.String(),
scanner.T_MINUS_EQUAL, scanner.T_MINUS_EQUAL.String(),
scanner.T_XOR_EQUAL, scanner.T_XOR_EQUAL.String(),
scanner.T_MOD_EQUAL, scanner.T_MOD_EQUAL.String(),
scanner.T_DEC, scanner.T_DEC.String(),
scanner.T_INC, scanner.T_INC.String(),
scanner.T_DOUBLE_ARROW, scanner.T_DOUBLE_ARROW.String(),
scanner.T_SPACESHIP, scanner.T_SPACESHIP.String(),
scanner.T_IS_NOT_EQUAL, scanner.T_IS_NOT_EQUAL.String(),
scanner.T_IS_NOT_EQUAL, scanner.T_IS_NOT_EQUAL.String(),
scanner.T_IS_NOT_IDENTICAL, scanner.T_IS_NOT_IDENTICAL.String(),
scanner.T_IS_EQUAL, scanner.T_IS_EQUAL.String(),
scanner.T_IS_IDENTICAL, scanner.T_IS_IDENTICAL.String(),
scanner.T_SL_EQUAL, scanner.T_SL_EQUAL.String(),
scanner.T_SR_EQUAL, scanner.T_SR_EQUAL.String(),
scanner.T_IS_GREATER_OR_EQUAL, scanner.T_IS_GREATER_OR_EQUAL.String(),
scanner.T_IS_SMALLER_OR_EQUAL, scanner.T_IS_SMALLER_OR_EQUAL.String(),
scanner.T_POW, scanner.T_POW.String(),
scanner.T_SL, scanner.T_SL.String(),
scanner.T_SR, scanner.T_SR.String(),
scanner.T_COALESCE, scanner.T_COALESCE.String(),
scanner.Rune2Class(';'), scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.Rune2Class(':'), scanner.LexerToken(scanner.Rune2Class(':')).String(),
scanner.Rune2Class(','), scanner.LexerToken(scanner.Rune2Class(',')).String(),
scanner.Rune2Class('.'), scanner.LexerToken(scanner.Rune2Class('.')).String(),
scanner.Rune2Class('['), scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.Rune2Class(']'), scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.Rune2Class('('), scanner.LexerToken(scanner.Rune2Class('(')).String(),
scanner.Rune2Class(')'), scanner.LexerToken(scanner.Rune2Class(')')).String(),
scanner.Rune2Class('|'), scanner.LexerToken(scanner.Rune2Class('|')).String(),
scanner.Rune2Class('/'), scanner.LexerToken(scanner.Rune2Class('/')).String(),
scanner.Rune2Class('^'), scanner.LexerToken(scanner.Rune2Class('^')).String(),
scanner.Rune2Class('&'), scanner.LexerToken(scanner.Rune2Class('&')).String(),
scanner.Rune2Class('+'), scanner.LexerToken(scanner.Rune2Class('+')).String(),
scanner.Rune2Class('-'), scanner.LexerToken(scanner.Rune2Class('-')).String(),
scanner.Rune2Class('*'), scanner.LexerToken(scanner.Rune2Class('*')).String(),
scanner.Rune2Class('='), scanner.LexerToken(scanner.Rune2Class('=')).String(),
scanner.Rune2Class('%'), scanner.LexerToken(scanner.Rune2Class('%')).String(),
scanner.Rune2Class('!'), scanner.LexerToken(scanner.Rune2Class('!')).String(),
scanner.Rune2Class('~'), scanner.LexerToken(scanner.Rune2Class('~')).String(),
scanner.Rune2Class('$'), scanner.LexerToken(scanner.Rune2Class('$')).String(),
scanner.Rune2Class('<'), scanner.LexerToken(scanner.Rune2Class('<')).String(),
scanner.Rune2Class('>'), scanner.LexerToken(scanner.Rune2Class('>')).String(),
scanner.Rune2Class('?'), scanner.LexerToken(scanner.Rune2Class('?')).String(),
scanner.Rune2Class('@'), scanner.LexerToken(scanner.Rune2Class('@')).String(),
scanner.Rune2Class('{'), scanner.LexerToken(scanner.Rune2Class('{')).String(),
scanner.Rune2Class('}'), scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_STRING, scanner.T_STRING.String(),
scanner.T_OBJECT_OPERATOR, scanner.T_OBJECT_OPERATOR.String(),
scanner.T_OBJECT_OPERATOR, scanner.T_OBJECT_OPERATOR.String(),
scanner.T_STRING, scanner.T_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING, scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING, scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING, scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_ARRAY_CAST, scanner.T_ARRAY_CAST.String(),
scanner.T_BOOL_CAST, scanner.T_BOOL_CAST.String(),
scanner.T_BOOL_CAST, scanner.T_BOOL_CAST.String(),
scanner.T_DOUBLE_CAST, scanner.T_DOUBLE_CAST.String(),
scanner.T_DOUBLE_CAST, scanner.T_DOUBLE_CAST.String(),
scanner.T_DOUBLE_CAST, scanner.T_DOUBLE_CAST.String(),
scanner.T_INT_CAST, scanner.T_INT_CAST.String(),
scanner.T_INT_CAST, scanner.T_INT_CAST.String(),
scanner.T_OBJECT_CAST, scanner.T_OBJECT_CAST.String(),
scanner.T_STRING_CAST, scanner.T_STRING_CAST.String(),
scanner.T_STRING_CAST, scanner.T_STRING_CAST.String(),
scanner.T_UNSET_CAST, scanner.T_UNSET_CAST.String(),
} }
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true lexer.WithMeta = true
lv := &lval{} lv := &lval{}
actual := []int{} actual := []string{}
for { for {
token := lexer.Lex(lv) token := lexer.Lex(lv)
@ -425,7 +425,7 @@ func TestTokens(t *testing.T) {
break break
} }
actual = append(actual, token) actual = append(actual, scanner.LexerToken(token).String())
} }
assertEqual(t, expected, actual) assertEqual(t, expected, actual)
@ -448,65 +448,65 @@ func TestTeplateStringTokens(t *testing.T) {
"$0$foo" "$0$foo"
` `
expected := []int{ expected := []string{
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_CURLY_OPEN, scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('}'), scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_CURLY_OPEN, scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('}'), scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_DOLLAR_OPEN_CURLY_BRACES, scanner.T_DOLLAR_OPEN_CURLY_BRACES.String(),
scanner.T_STRING_VARNAME, scanner.T_STRING_VARNAME.String(),
scanner.Rune2Class('}'), scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_CURLY_OPEN, scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('}'), scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
} }
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true lexer.WithMeta = true
lv := &lval{} lv := &lval{}
actual := []int{} actual := []string{}
for { for {
token := lexer.Lex(lv) token := lexer.Lex(lv)
@ -514,7 +514,7 @@ func TestTeplateStringTokens(t *testing.T) {
break break
} }
actual = append(actual, token) actual = append(actual, scanner.LexerToken(token).String())
} }
assertEqual(t, expected, actual) assertEqual(t, expected, actual)
@ -534,65 +534,65 @@ func TestBackquoteStringTokens(t *testing.T) {
` + "`$0$foo`" + ` ` + "`$0$foo`" + `
` `
expected := []int{ expected := []string{
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_CURLY_OPEN, scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('}'), scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_CURLY_OPEN, scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('}'), scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_DOLLAR_OPEN_CURLY_BRACES, scanner.T_DOLLAR_OPEN_CURLY_BRACES.String(),
scanner.T_STRING_VARNAME, scanner.T_STRING_VARNAME.String(),
scanner.Rune2Class('}'), scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_CURLY_OPEN, scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('}'), scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('`'), scanner.LexerToken(scanner.Rune2Class('`')).String(),
} }
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true lexer.WithMeta = true
lv := &lval{} lv := &lval{}
actual := []int{} actual := []string{}
for { for {
token := lexer.Lex(lv) token := lexer.Lex(lv)
@ -600,7 +600,7 @@ func TestBackquoteStringTokens(t *testing.T) {
break break
} }
actual = append(actual, token) actual = append(actual, scanner.LexerToken(token).String())
} }
assertEqual(t, expected, actual) assertEqual(t, expected, actual)
@ -630,63 +630,64 @@ CAT;
CAT; CAT;
` `
expected := []int{ expected := []string{
scanner.T_START_HEREDOC, scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_END_HEREDOC, scanner.T_END_HEREDOC.String(),
scanner.Rune2Class(';'), scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_START_HEREDOC, scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_END_HEREDOC, scanner.T_END_HEREDOC.String(),
scanner.Rune2Class(';'), scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_START_HEREDOC, scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_OBJECT_OPERATOR, scanner.T_OBJECT_OPERATOR.String(),
scanner.T_STRING, scanner.T_STRING.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('['), scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_NUM_STRING, scanner.T_NUM_STRING.String(),
scanner.Rune2Class(']'), scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('['), scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_NUM_STRING, scanner.T_NUM_STRING.String(),
scanner.Rune2Class(']'), scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('['), scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_NUM_STRING, scanner.T_NUM_STRING.String(),
scanner.Rune2Class(']'), scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('['), scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_STRING, scanner.T_STRING.String(),
scanner.Rune2Class(']'), scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('['), scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class(']'), scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_CURLY_OPEN, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_CURLY_OPEN.String(),
scanner.Rune2Class('}'), scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_DOLLAR_OPEN_CURLY_BRACES, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_STRING_VARNAME, scanner.T_DOLLAR_OPEN_CURLY_BRACES.String(),
scanner.Rune2Class('}'), scanner.T_STRING_VARNAME.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_END_HEREDOC, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.Rune2Class(';'), scanner.T_END_HEREDOC.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
} }
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true lexer.WithMeta = true
lv := &lval{} lv := &lval{}
actual := []int{} actual := []string{}
for { for {
token := lexer.Lex(lv) token := lexer.Lex(lv)
@ -694,7 +695,7 @@ CAT;
break break
} }
actual = append(actual, token) actual = append(actual, scanner.LexerToken(token).String())
} }
assertEqual(t, expected, actual) assertEqual(t, expected, actual)
@ -723,44 +724,42 @@ $foo$bar\
CAT CAT
` `
expected := []int{ expected := []string{
scanner.T_START_HEREDOC, scanner.T_START_HEREDOC.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_END_HEREDOC, scanner.T_END_HEREDOC.String(),
scanner.Rune2Class(';'), scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_START_HEREDOC, scanner.T_START_HEREDOC.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_END_HEREDOC, scanner.T_END_HEREDOC.String(),
scanner.Rune2Class(';'), scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_START_HEREDOC, scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_END_HEREDOC.String(),
scanner.T_END_HEREDOC, scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC, scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_END_HEREDOC.String(),
scanner.T_END_HEREDOC, scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC, scanner.T_START_HEREDOC.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_END_HEREDOC, scanner.T_END_HEREDOC.String(),
} }
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true lexer.WithMeta = true
lv := &lval{} lv := &lval{}
actual := []int{} actual := []string{}
for { for {
token := lexer.Lex(lv) token := lexer.Lex(lv)
@ -768,7 +767,46 @@ CAT
break break
} }
actual = append(actual, token) actual = append(actual, scanner.LexerToken(token).String())
}
assertEqual(t, expected, actual)
}
func TestHereDocTokens3(t *testing.T) {
src := `<?php
<<<"CAT"
\\{$a['b']}
CAT;
`
expected := []string{
scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_END_HEREDOC.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []string{}
for {
token := lexer.Lex(lv)
if token < 0 {
break
}
actual = append(actual, scanner.LexerToken(token).String())
} }
assertEqual(t, expected, actual) assertEqual(t, expected, actual)
@ -780,20 +818,20 @@ func TestInlineHtmlNopTokens(t *testing.T) {
$a ?> test $a ?> test
` `
expected := []int{ expected := []string{
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class(';'), scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_INLINE_HTML, scanner.T_INLINE_HTML.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class(';'), scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_INLINE_HTML, scanner.T_INLINE_HTML.String(),
} }
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true lexer.WithMeta = true
lv := &lval{} lv := &lval{}
actual := []int{} actual := []string{}
for { for {
token := lexer.Lex(lv) token := lexer.Lex(lv)
@ -801,7 +839,7 @@ func TestInlineHtmlNopTokens(t *testing.T) {
break break
} }
actual = append(actual, token) actual = append(actual, scanner.LexerToken(token).String())
} }
assertEqual(t, expected, actual) assertEqual(t, expected, actual)
@ -810,12 +848,12 @@ func TestInlineHtmlNopTokens(t *testing.T) {
func TestStringTokensAfterVariable(t *testing.T) { func TestStringTokensAfterVariable(t *testing.T) {
src := `<?php "test \"$var\""` src := `<?php "test \"$var\""`
expected := []int{ expected := []string{
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.Rune2Class('"'), scanner.LexerToken(scanner.Rune2Class('"')).String(),
} }
expectedTokens := []string{ expectedTokens := []string{
@ -828,7 +866,7 @@ func TestStringTokensAfterVariable(t *testing.T) {
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lv := &lval{} lv := &lval{}
actual := []int{} actual := []string{}
actualTokens := []string{} actualTokens := []string{}
for { for {
@ -838,7 +876,7 @@ func TestStringTokensAfterVariable(t *testing.T) {
} }
actualTokens = append(actualTokens, lv.Tkn.Value) actualTokens = append(actualTokens, lv.Tkn.Value)
actual = append(actual, token) actual = append(actual, scanner.LexerToken(token).String())
} }
assertEqual(t, expected, actual) assertEqual(t, expected, actual)
@ -848,10 +886,10 @@ func TestStringTokensAfterVariable(t *testing.T) {
func TestSlashAfterVariable(t *testing.T) { func TestSlashAfterVariable(t *testing.T) {
src := `<?php $foo/3` src := `<?php $foo/3`
expected := []int{ expected := []string{
scanner.T_VARIABLE, scanner.T_VARIABLE.String(),
scanner.Rune2Class('/'), scanner.LexerToken(scanner.Rune2Class('/')).String(),
scanner.T_LNUMBER, scanner.T_LNUMBER.String(),
} }
expectedTokens := []string{ expectedTokens := []string{
@ -862,7 +900,7 @@ func TestSlashAfterVariable(t *testing.T) {
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php") lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lv := &lval{} lv := &lval{}
actual := []int{} actual := []string{}
actualTokens := []string{} actualTokens := []string{}
for { for {
@ -872,7 +910,7 @@ func TestSlashAfterVariable(t *testing.T) {
} }
actualTokens = append(actualTokens, lv.Tkn.Value) actualTokens = append(actualTokens, lv.Tkn.Value)
actual = append(actual, token) actual = append(actual, scanner.LexerToken(token).String())
} }
assertEqual(t, expected, actual) assertEqual(t, expected, actual)