maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed bug when heredoc starts by \; removing \n from a heredoc value end

Browse Source
This commit is contained in:
z7zmey 2018-07-14 18:00:48 +03:00
parent 4da7b36056
commit 867095823d
11 changed files with 848 additions and 791 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
node/scalar/t_heredoc_test.go
View File

@ -70,15 +70,6 @@ LBL;
Value: "var",
},
},
&scalar.EncapsedStringPart{
Position: &position.Position{
StartLine: 2,
EndLine: 2,
StartPos: 20,
EndPos: 20,
},
Value: "\n",
},
},
},
},
@ -152,15 +143,6 @@ LBL;
Value: "var",
},
},
&scalar.EncapsedStringPart{
Position: &position.Position{
StartLine: 2,
EndLine: 2,
StartPos: 22,
EndPos: 22,
},
Value: "\n",
},
},
},
},
@ -213,9 +195,9 @@ LBL;
StartLine: 2,
EndLine: 2,
StartPos: 13,
EndPos: 22,
EndPos: 21,
},
Value: "test $var\n",
Value: "test $var",
},
},
},
@ -313,9 +295,9 @@ CAD;
StartLine: 2,
EndLine: 2,
StartPos: 11,
EndPos: 17,
EndPos: 16,
},
Value: "\thello\n",
Value: "\thello",
},
},
},

21
php5/php5_test.go
View File

@ -18597,9 +18597,9 @@ CAD;
StartLine: 5,
EndLine: 5,
StartPos: 27,
EndPos: 33,
EndPos: 32,
},
Value: "\thello\n",
Value: "\thello",
},
},
},
@ -18625,9 +18625,9 @@ CAD;
StartLine: 8,
EndLine: 8,
StartPos: 50,
EndPos: 56,
EndPos: 55,
},
Value: "\thello\n",
Value: "\thello",
},
},
},
@ -18674,15 +18674,6 @@ CAD;
Value: "world",
},
},
&scalar.EncapsedStringPart{
Position: &position.Position{
StartLine: 11,
EndLine: 11,
StartPos: 86,
EndPos: 86,
},
Value: "\n",
},
},
},
},
@ -18707,9 +18698,9 @@ CAD;
StartLine: 14,
EndLine: 14,
StartPos: 103,
EndPos: 116,
EndPos: 115,
},
Value: "\thello $world\n",
Value: "\thello $world",
},
},
},

21
php7/php7_test.go
View File

@ -16312,9 +16312,9 @@ CAD;
StartLine: 5,
EndLine: 5,
StartPos: 27,
EndPos: 33,
EndPos: 32,
},
Value: "\thello\n",
Value: "\thello",
},
},
},
@ -16340,9 +16340,9 @@ CAD;
StartLine: 8,
EndLine: 8,
StartPos: 50,
EndPos: 56,
EndPos: 55,
},
Value: "\thello\n",
Value: "\thello",
},
},
},
@ -16389,15 +16389,6 @@ CAD;
Value: "world",
},
},
&scalar.EncapsedStringPart{
Position: &position.Position{
StartLine: 11,
EndLine: 11,
StartPos: 86,
EndPos: 86,
},
Value: "\n",
},
},
},
},
@ -16422,9 +16413,9 @@ CAD;
StartLine: 14,
EndLine: 14,
StartPos: 103,
EndPos: 116,
EndPos: 115,
},
Value: "\thello $world\n",
Value: "\thello $world",
},
},
},

1
printer/printer.go
View File

@ -665,6 +665,7 @@ func (p *Printer) printScalarHeredoc(n node.Node) {
}
}
io.WriteString(p.w, "\n")
io.WriteString(p.w, strings.Trim(nn.Label, "\"'"))
p.printMeta(nn, meta.NodeEnd)

4
printer/printer_test.go
View File

@ -575,7 +575,7 @@ func TestPrinterPrintScalarHeredoc(t *testing.T) {
Parts: []node.Node{
&scalar.EncapsedStringPart{Value: "hello "},
&expr.Variable{VarName: &node.Identifier{Value: "var"}},
&scalar.EncapsedStringPart{Value: " world\n"},
&scalar.EncapsedStringPart{Value: " world"},
},
})
@ -602,7 +602,7 @@ func TestPrinterPrintScalarNowdoc(t *testing.T) {
},
Label: "'LBL'",
Parts: []node.Node{
&scalar.EncapsedStringPart{Value: "hello world\n"},
&scalar.EncapsedStringPart{Value: "hello world"},
},
})

2
scanner/lexer.go
View File

@ -124,6 +124,8 @@ func (l *Lexer) createToken(chars []lex.Char) *Token {
token.Meta = l.Meta
token.Value = l.tokenString(chars)
// fmt.Println(l.tokenString(chars))
token.StartLine = l.File.Line(firstChar.Pos())
token.EndLine = l.File.Line(lastChar.Pos())
token.StartPos = int(firstChar.Pos())

5
scanner/lexer_tokens.go
View File

@ -1,7 +1,10 @@
package scanner
type LexerToken int
//go:generate stringer -type=LexerToken -output ./lexer_tokens_string.go
const (
T_INCLUDE int = iota + 57346
T_INCLUDE LexerToken = iota + 57346
T_INCLUDE_ONCE
T_EXIT
T_IF

17
scanner/lexer_tokens_string.go Normal file
View File

@ -0,0 +1,17 @@
// Code generated by "stringer -type=LexerToken -output ./lexer_tokens_string.go"; DO NOT EDIT.
package scanner
import "strconv"
const _LexerToken_name = "T_INCLUDET_INCLUDE_ONCET_EXITT_IFT_LNUMBERT_DNUMBERT_STRINGT_STRING_VARNAMET_VARIABLET_NUM_STRINGT_INLINE_HTMLT_CHARACTERT_BAD_CHARACTERT_ENCAPSED_AND_WHITESPACET_CONSTANT_ENCAPSED_STRINGT_ECHOT_DOT_WHILET_ENDWHILET_FORT_ENDFORT_FOREACHT_ENDFOREACHT_DECLARET_ENDDECLARET_AST_SWITCHT_ENDSWITCHT_CASET_DEFAULTT_BREAKT_CONTINUET_GOTOT_FUNCTIONT_CONSTT_RETURNT_TRYT_CATCHT_FINALLYT_THROWT_USET_INSTEADOFT_GLOBALT_VART_UNSETT_ISSETT_EMPTYT_HALT_COMPILERT_CLASST_TRAITT_INTERFACET_EXTENDST_IMPLEMENTST_OBJECT_OPERATORT_DOUBLE_ARROWT_LISTT_ARRAYT_CALLABLET_CLASS_CT_TRAIT_CT_METHOD_CT_FUNC_CT_LINET_FILET_COMMENTT_DOC_COMMENTT_OPEN_TAGT_OPEN_TAG_WITH_ECHOT_CLOSE_TAGT_WHITESPACET_START_HEREDOCT_END_HEREDOCT_DOLLAR_OPEN_CURLY_BRACEST_CURLY_OPENT_PAAMAYIM_NEKUDOTAYIMT_NAMESPACET_NS_CT_DIRT_NS_SEPARATORT_ELLIPSIST_EVALT_REQUIRET_REQUIRE_ONCET_LOGICAL_ORT_LOGICAL_XORT_LOGICAL_ANDT_INSTANCEOFT_NEWT_CLONET_ELSEIFT_ELSET_ENDIFT_PRINTT_YIELDT_STATICT_ABSTRACTT_FINALT_PRIVATET_PROTECTEDT_PUBLICT_INCT_DECT_YIELD_FROMT_INT_CASTT_DOUBLE_CASTT_STRING_CASTT_ARRAY_CASTT_OBJECT_CASTT_BOOL_CASTT_UNSET_CASTT_COALESCET_SPACESHIPT_NOELSET_PLUS_EQUALT_MINUS_EQUALT_MUL_EQUALT_POW_EQUALT_DIV_EQUALT_CONCAT_EQUALT_MOD_EQUALT_AND_EQUALT_OR_EQUALT_XOR_EQUALT_SL_EQUALT_SR_EQUALT_BOOLEAN_ORT_BOOLEAN_ANDT_POWT_SLT_SRT_IS_IDENTICALT_IS_NOT_IDENTICALT_IS_EQUALT_IS_NOT_EQUALT_IS_SMALLER_OR_EQUALT_IS_GREATER_OR_EQUAL"
var _LexerToken_index = [...]uint16{0, 9, 23, 29, 33, 42, 51, 59, 75, 85, 97, 110, 121, 136, 161, 187, 193, 197, 204, 214, 219, 227, 236, 248, 257, 269, 273, 281, 292, 298, 307, 314, 324, 330, 340, 347, 355, 360, 367, 376, 383, 388, 399, 407, 412, 419, 426, 433, 448, 455, 462, 473, 482, 494, 511, 525, 531, 538, 548, 557, 566, 576, 584, 590, 596, 605, 618, 628, 648, 659, 671, 686, 699, 725, 737, 759, 770, 776, 781, 795, 805, 811, 820, 834, 846, 859, 872, 884, 889, 896, 904, 910, 917, 924, 931, 939, 949, 956, 965, 976, 984, 989, 994, 1006, 1016, 1029, 1042, 1054, 1067, 1078, 1090, 1100, 1111, 1119, 1131, 1144, 1155, 1166, 1177, 1191, 1202, 1213, 1223, 1234, 1244, 1254, 1266, 1279, 1284, 1288, 1292, 1306, 1324, 1334, 1348, 1369, 1390}
func (i LexerToken) String() string {
i -= 57346
if i < 0 || i >= LexerToken(len(_LexerToken_index)-1) {
return "LexerToken(" + strconv.FormatInt(int64(i+57346), 10) + ")"
}
return _LexerToken_name[_LexerToken_index[i]:_LexerToken_index[i+1]]
}

375
scanner/scanner.go
View File

File diff suppressed because it is too large Load Diff

377
scanner/scanner.l
View File

@ -82,17 +82,17 @@ NEW_LINE (\r|\n|\r\n)
}
lval.Token(l.createToken(tb))
return T_INLINE_HTML
return int(T_INLINE_HTML)
<INITIAL>\<\?php([ \t]|{NEW_LINE}) l.begin(PHP);l.ungetChars(len(l.Token())-5)
<INITIAL>\<\? l.begin(PHP);
<INITIAL>\<\?= l.begin(PHP);lval.Token(l.createToken(l.Token())); return T_ECHO;
<INITIAL>\<\?= l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_ECHO);
<PHP>[ \t\n\r]+ l.addWhiteSpace(l.Token())
<PHP>[;][ \t\n\r]*\?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
<PHP>\?\>{NEW_LINE}? l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
<PHP>{DNUM}|{EXPONENT_DNUM} lval.Token(l.createToken(l.Token())); return T_DNUMBER
<PHP>{DNUM}|{EXPONENT_DNUM} lval.Token(l.createToken(l.Token())); return int(T_DNUMBER)
<PHP>{BNUM}
tb := l.Token()
i:=2
@ -106,15 +106,15 @@ NEW_LINE (\r|\n|\r\n)
}
}
if len(tb) - i < 64 {
lval.Token(l.createToken(l.Token())); return T_LNUMBER
lval.Token(l.createToken(l.Token())); return int(T_LNUMBER)
} else {
lval.Token(l.createToken(l.Token())); return T_DNUMBER
lval.Token(l.createToken(l.Token())); return int(T_DNUMBER)
}
<PHP>{LNUM}
if len(l.Token()) < 20 {
lval.Token(l.createToken(l.Token())); return T_LNUMBER
lval.Token(l.createToken(l.Token())); return int(T_LNUMBER)
} else {
lval.Token(l.createToken(l.Token())); return T_DNUMBER
lval.Token(l.createToken(l.Token())); return int(T_DNUMBER)
}
<PHP>{HNUM}
tb := l.Token()
@ -130,124 +130,124 @@ NEW_LINE (\r|\n|\r\n)
}
length := len(tb) - i
if length < 16 || (length == 16 && tb[i].Rune <= '7') {
lval.Token(l.createToken(l.Token())); return T_LNUMBER
lval.Token(l.createToken(l.Token())); return int(T_LNUMBER)
} else {
lval.Token(l.createToken(l.Token())); return T_DNUMBER
lval.Token(l.createToken(l.Token())); return int(T_DNUMBER)
}
<PHP>abstract lval.Token(l.createToken(l.Token())); return T_ABSTRACT
<PHP>array lval.Token(l.createToken(l.Token())); return T_ARRAY
<PHP>as lval.Token(l.createToken(l.Token())); return T_AS
<PHP>break lval.Token(l.createToken(l.Token())); return T_BREAK
<PHP>callable lval.Token(l.createToken(l.Token())); return T_CALLABLE
<PHP>case lval.Token(l.createToken(l.Token())); return T_CASE
<PHP>catch lval.Token(l.createToken(l.Token())); return T_CATCH
<PHP>class lval.Token(l.createToken(l.Token())); return T_CLASS
<PHP>clone lval.Token(l.createToken(l.Token())); return T_CLONE
<PHP>const lval.Token(l.createToken(l.Token())); return T_CONST
<PHP>continue lval.Token(l.createToken(l.Token())); return T_CONTINUE
<PHP>declare lval.Token(l.createToken(l.Token())); return T_DECLARE
<PHP>default lval.Token(l.createToken(l.Token())); return T_DEFAULT
<PHP>do lval.Token(l.createToken(l.Token())); return T_DO
<PHP>echo lval.Token(l.createToken(l.Token())); return T_ECHO
<PHP>else lval.Token(l.createToken(l.Token())); return T_ELSE
<PHP>elseif lval.Token(l.createToken(l.Token())); return T_ELSEIF
<PHP>empty lval.Token(l.createToken(l.Token())); return T_EMPTY
<PHP>enddeclare lval.Token(l.createToken(l.Token())); return T_ENDDECLARE
<PHP>endfor lval.Token(l.createToken(l.Token())); return T_ENDFOR
<PHP>endforeach lval.Token(l.createToken(l.Token())); return T_ENDFOREACH
<PHP>endif lval.Token(l.createToken(l.Token())); return T_ENDIF
<PHP>endswitch lval.Token(l.createToken(l.Token())); return T_ENDSWITCH
<PHP>endwhile lval.Token(l.createToken(l.Token())); return T_ENDWHILE
<PHP>eval lval.Token(l.createToken(l.Token())); return T_EVAL
<PHP>exit|die lval.Token(l.createToken(l.Token())); return T_EXIT
<PHP>extends lval.Token(l.createToken(l.Token())); return T_EXTENDS
<PHP>final lval.Token(l.createToken(l.Token())); return T_FINAL
<PHP>finally lval.Token(l.createToken(l.Token())); return T_FINALLY
<PHP>for lval.Token(l.createToken(l.Token())); return T_FOR
<PHP>foreach lval.Token(l.createToken(l.Token())); return T_FOREACH
<PHP>function|cfunction lval.Token(l.createToken(l.Token())); return T_FUNCTION
<PHP>global lval.Token(l.createToken(l.Token())); return T_GLOBAL
<PHP>goto lval.Token(l.createToken(l.Token())); return T_GOTO
<PHP>if lval.Token(l.createToken(l.Token())); return T_IF
<PHP>isset lval.Token(l.createToken(l.Token())); return T_ISSET
<PHP>implements lval.Token(l.createToken(l.Token())); return T_IMPLEMENTS
<PHP>instanceof lval.Token(l.createToken(l.Token())); return T_INSTANCEOF
<PHP>insteadof lval.Token(l.createToken(l.Token())); return T_INSTEADOF
<PHP>interface lval.Token(l.createToken(l.Token())); return T_INTERFACE
<PHP>list lval.Token(l.createToken(l.Token())); return T_LIST
<PHP>namespace lval.Token(l.createToken(l.Token())); return T_NAMESPACE
<PHP>private lval.Token(l.createToken(l.Token())); return T_PRIVATE
<PHP>public lval.Token(l.createToken(l.Token())); return T_PUBLIC
<PHP>print lval.Token(l.createToken(l.Token())); return T_PRINT
<PHP>protected lval.Token(l.createToken(l.Token())); return T_PROTECTED
<PHP>return lval.Token(l.createToken(l.Token())); return T_RETURN
<PHP>static lval.Token(l.createToken(l.Token())); return T_STATIC
<PHP>switch lval.Token(l.createToken(l.Token())); return T_SWITCH
<PHP>throw lval.Token(l.createToken(l.Token())); return T_THROW
<PHP>trait lval.Token(l.createToken(l.Token())); return T_TRAIT
<PHP>try lval.Token(l.createToken(l.Token())); return T_TRY
<PHP>unset lval.Token(l.createToken(l.Token())); return T_UNSET
<PHP>use lval.Token(l.createToken(l.Token())); return T_USE
<PHP>var lval.Token(l.createToken(l.Token())); return T_VAR
<PHP>while lval.Token(l.createToken(l.Token())); return T_WHILE
<PHP>yield[ \t\n\r]+from lval.Token(l.createToken(l.Token())); return T_YIELD_FROM
<PHP>yield lval.Token(l.createToken(l.Token())); return T_YIELD
<PHP>include lval.Token(l.createToken(l.Token())); return T_INCLUDE
<PHP>include_once lval.Token(l.createToken(l.Token())); return T_INCLUDE_ONCE
<PHP>require lval.Token(l.createToken(l.Token())); return T_REQUIRE
<PHP>require_once lval.Token(l.createToken(l.Token())); return T_REQUIRE_ONCE
<PHP>__CLASS__ lval.Token(l.createToken(l.Token())); return T_CLASS_C
<PHP>__DIR__ lval.Token(l.createToken(l.Token())); return T_DIR
<PHP>__FILE__ lval.Token(l.createToken(l.Token())); return T_FILE
<PHP>__FUNCTION__ lval.Token(l.createToken(l.Token())); return T_FUNC_C
<PHP>__LINE__ lval.Token(l.createToken(l.Token())); return T_LINE
<PHP>__NAMESPACE__ lval.Token(l.createToken(l.Token())); return T_NS_C
<PHP>__METHOD__ lval.Token(l.createToken(l.Token())); return T_METHOD_C
<PHP>__TRAIT__ lval.Token(l.createToken(l.Token())); return T_TRAIT_C
<PHP>__halt_compiler lval.Token(l.createToken(l.Token())); return T_HALT_COMPILER
<PHP>\([ \t]*array[ \t]*\) lval.Token(l.createToken(l.Token())); return T_ARRAY_CAST
<PHP>\([ \t]*(bool|boolean)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_BOOL_CAST
<PHP>\([ \t]*(real|double|float)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_DOUBLE_CAST
<PHP>\([ \t]*(int|integer)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_INT_CAST
<PHP>\([ \t]*object[ \t]*\) lval.Token(l.createToken(l.Token())); return T_OBJECT_CAST
<PHP>\([ \t]*(string|binary)[ \t]*\) lval.Token(l.createToken(l.Token())); return T_STRING_CAST
<PHP>\([ \t]*unset[ \t]*\) lval.Token(l.createToken(l.Token())); return T_UNSET_CAST
<PHP>new lval.Token(l.createToken(l.Token())); return T_NEW
<PHP>and lval.Token(l.createToken(l.Token())); return T_LOGICAL_AND
<PHP>or lval.Token(l.createToken(l.Token())); return T_LOGICAL_OR
<PHP>xor lval.Token(l.createToken(l.Token())); return T_LOGICAL_XOR
<PHP>\\ lval.Token(l.createToken(l.Token())); return T_NS_SEPARATOR
<PHP>\.\.\. lval.Token(l.createToken(l.Token())); return T_ELLIPSIS
<PHP>:: lval.Token(l.createToken(l.Token())); return T_PAAMAYIM_NEKUDOTAYIM // T_DOUBLE_COLON
<PHP>&& lval.Token(l.createToken(l.Token())); return T_BOOLEAN_AND
<PHP>\|\| lval.Token(l.createToken(l.Token())); return T_BOOLEAN_OR
<PHP>&= lval.Token(l.createToken(l.Token())); return T_AND_EQUAL
<PHP>\|= lval.Token(l.createToken(l.Token())); return T_OR_EQUAL
<PHP>\.= lval.Token(l.createToken(l.Token())); return T_CONCAT_EQUAL
<PHP>\*= lval.Token(l.createToken(l.Token())); return T_MUL_EQUAL
<PHP>\*\*= lval.Token(l.createToken(l.Token())); return T_POW_EQUAL
<PHP>[/]= lval.Token(l.createToken(l.Token())); return T_DIV_EQUAL
<PHP>\+= lval.Token(l.createToken(l.Token())); return T_PLUS_EQUAL
<PHP>-= lval.Token(l.createToken(l.Token())); return T_MINUS_EQUAL
<PHP>\^= lval.Token(l.createToken(l.Token())); return T_XOR_EQUAL
<PHP>%= lval.Token(l.createToken(l.Token())); return T_MOD_EQUAL
<PHP>-- lval.Token(l.createToken(l.Token())); return T_DEC
<PHP>\+\+ lval.Token(l.createToken(l.Token())); return T_INC
<PHP>=> lval.Token(l.createToken(l.Token())); return T_DOUBLE_ARROW
<PHP>\<=\> lval.Token(l.createToken(l.Token())); return T_SPACESHIP
<PHP>\!=|\<\> lval.Token(l.createToken(l.Token())); return T_IS_NOT_EQUAL
<PHP>\!== lval.Token(l.createToken(l.Token())); return T_IS_NOT_IDENTICAL
<PHP>== lval.Token(l.createToken(l.Token())); return T_IS_EQUAL
<PHP>=== lval.Token(l.createToken(l.Token())); return T_IS_IDENTICAL
<PHP>\<\<= lval.Token(l.createToken(l.Token())); return T_SL_EQUAL
<PHP>\>\>= lval.Token(l.createToken(l.Token())); return T_SR_EQUAL
<PHP>\>= lval.Token(l.createToken(l.Token())); return T_IS_GREATER_OR_EQUAL
<PHP>\<= lval.Token(l.createToken(l.Token())); return T_IS_SMALLER_OR_EQUAL
<PHP>\*\* lval.Token(l.createToken(l.Token())); return T_POW
<PHP>\<\< lval.Token(l.createToken(l.Token())); return T_SL
<PHP>\>\> lval.Token(l.createToken(l.Token())); return T_SR
<PHP>\?\? lval.Token(l.createToken(l.Token())); return T_COALESCE
<PHP>abstract lval.Token(l.createToken(l.Token())); return int(T_ABSTRACT)
<PHP>array lval.Token(l.createToken(l.Token())); return int(T_ARRAY)
<PHP>as lval.Token(l.createToken(l.Token())); return int(T_AS)
<PHP>break lval.Token(l.createToken(l.Token())); return int(T_BREAK)
<PHP>callable lval.Token(l.createToken(l.Token())); return int(T_CALLABLE)
<PHP>case lval.Token(l.createToken(l.Token())); return int(T_CASE)
<PHP>catch lval.Token(l.createToken(l.Token())); return int(T_CATCH)
<PHP>class lval.Token(l.createToken(l.Token())); return int(T_CLASS)
<PHP>clone lval.Token(l.createToken(l.Token())); return int(T_CLONE)
<PHP>const lval.Token(l.createToken(l.Token())); return int(T_CONST)
<PHP>continue lval.Token(l.createToken(l.Token())); return int(T_CONTINUE)
<PHP>declare lval.Token(l.createToken(l.Token())); return int(T_DECLARE)
<PHP>default lval.Token(l.createToken(l.Token())); return int(T_DEFAULT)
<PHP>do lval.Token(l.createToken(l.Token())); return int(T_DO)
<PHP>echo lval.Token(l.createToken(l.Token())); return int(T_ECHO)
<PHP>else lval.Token(l.createToken(l.Token())); return int(T_ELSE)
<PHP>elseif lval.Token(l.createToken(l.Token())); return int(T_ELSEIF)
<PHP>empty lval.Token(l.createToken(l.Token())); return int(T_EMPTY)
<PHP>enddeclare lval.Token(l.createToken(l.Token())); return int(T_ENDDECLARE)
<PHP>endfor lval.Token(l.createToken(l.Token())); return int(T_ENDFOR)
<PHP>endforeach lval.Token(l.createToken(l.Token())); return int(T_ENDFOREACH)
<PHP>endif lval.Token(l.createToken(l.Token())); return int(T_ENDIF)
<PHP>endswitch lval.Token(l.createToken(l.Token())); return int(T_ENDSWITCH)
<PHP>endwhile lval.Token(l.createToken(l.Token())); return int(T_ENDWHILE)
<PHP>eval lval.Token(l.createToken(l.Token())); return int(T_EVAL)
<PHP>exit|die lval.Token(l.createToken(l.Token())); return int(T_EXIT)
<PHP>extends lval.Token(l.createToken(l.Token())); return int(T_EXTENDS)
<PHP>final lval.Token(l.createToken(l.Token())); return int(T_FINAL)
<PHP>finally lval.Token(l.createToken(l.Token())); return int(T_FINALLY)
<PHP>for lval.Token(l.createToken(l.Token())); return int(T_FOR)
<PHP>foreach lval.Token(l.createToken(l.Token())); return int(T_FOREACH)
<PHP>function|cfunction lval.Token(l.createToken(l.Token())); return int(T_FUNCTION)
<PHP>global lval.Token(l.createToken(l.Token())); return int(T_GLOBAL)
<PHP>goto lval.Token(l.createToken(l.Token())); return int(T_GOTO)
<PHP>if lval.Token(l.createToken(l.Token())); return int(T_IF)
<PHP>isset lval.Token(l.createToken(l.Token())); return int(T_ISSET)
<PHP>implements lval.Token(l.createToken(l.Token())); return int(T_IMPLEMENTS)
<PHP>instanceof lval.Token(l.createToken(l.Token())); return int(T_INSTANCEOF)
<PHP>insteadof lval.Token(l.createToken(l.Token())); return int(T_INSTEADOF)
<PHP>interface lval.Token(l.createToken(l.Token())); return int(T_INTERFACE)
<PHP>list lval.Token(l.createToken(l.Token())); return int(T_LIST)
<PHP>namespace lval.Token(l.createToken(l.Token())); return int(T_NAMESPACE)
<PHP>private lval.Token(l.createToken(l.Token())); return int(T_PRIVATE)
<PHP>public lval.Token(l.createToken(l.Token())); return int(T_PUBLIC)
<PHP>print lval.Token(l.createToken(l.Token())); return int(T_PRINT)
<PHP>protected lval.Token(l.createToken(l.Token())); return int(T_PROTECTED)
<PHP>return lval.Token(l.createToken(l.Token())); return int(T_RETURN)
<PHP>static lval.Token(l.createToken(l.Token())); return int(T_STATIC)
<PHP>switch lval.Token(l.createToken(l.Token())); return int(T_SWITCH)
<PHP>throw lval.Token(l.createToken(l.Token())); return int(T_THROW)
<PHP>trait lval.Token(l.createToken(l.Token())); return int(T_TRAIT)
<PHP>try lval.Token(l.createToken(l.Token())); return int(T_TRY)
<PHP>unset lval.Token(l.createToken(l.Token())); return int(T_UNSET)
<PHP>use lval.Token(l.createToken(l.Token())); return int(T_USE)
<PHP>var lval.Token(l.createToken(l.Token())); return int(T_VAR)
<PHP>while lval.Token(l.createToken(l.Token())); return int(T_WHILE)
<PHP>yield[ \t\n\r]+from lval.Token(l.createToken(l.Token())); return int(T_YIELD_FROM)
<PHP>yield lval.Token(l.createToken(l.Token())); return int(T_YIELD)
<PHP>include lval.Token(l.createToken(l.Token())); return int(T_INCLUDE)
<PHP>include_once lval.Token(l.createToken(l.Token())); return int(T_INCLUDE_ONCE)
<PHP>require lval.Token(l.createToken(l.Token())); return int(T_REQUIRE)
<PHP>require_once lval.Token(l.createToken(l.Token())); return int(T_REQUIRE_ONCE)
<PHP>__CLASS__ lval.Token(l.createToken(l.Token())); return int(T_CLASS_C)
<PHP>__DIR__ lval.Token(l.createToken(l.Token())); return int(T_DIR)
<PHP>__FILE__ lval.Token(l.createToken(l.Token())); return int(T_FILE)
<PHP>__FUNCTION__ lval.Token(l.createToken(l.Token())); return int(T_FUNC_C)
<PHP>__LINE__ lval.Token(l.createToken(l.Token())); return int(T_LINE)
<PHP>__NAMESPACE__ lval.Token(l.createToken(l.Token())); return int(T_NS_C)
<PHP>__METHOD__ lval.Token(l.createToken(l.Token())); return int(T_METHOD_C)
<PHP>__TRAIT__ lval.Token(l.createToken(l.Token())); return int(T_TRAIT_C)
<PHP>__halt_compiler lval.Token(l.createToken(l.Token())); return int(T_HALT_COMPILER)
<PHP>\([ \t]*array[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_ARRAY_CAST)
<PHP>\([ \t]*(bool|boolean)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_BOOL_CAST)
<PHP>\([ \t]*(real|double|float)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_DOUBLE_CAST)
<PHP>\([ \t]*(int|integer)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_INT_CAST)
<PHP>\([ \t]*object[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_OBJECT_CAST)
<PHP>\([ \t]*(string|binary)[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_STRING_CAST)
<PHP>\([ \t]*unset[ \t]*\) lval.Token(l.createToken(l.Token())); return int(T_UNSET_CAST)
<PHP>new lval.Token(l.createToken(l.Token())); return int(T_NEW)
<PHP>and lval.Token(l.createToken(l.Token())); return int(T_LOGICAL_AND)
<PHP>or lval.Token(l.createToken(l.Token())); return int(T_LOGICAL_OR)
<PHP>xor lval.Token(l.createToken(l.Token())); return int(T_LOGICAL_XOR)
<PHP>\\ lval.Token(l.createToken(l.Token())); return int(T_NS_SEPARATOR)
<PHP>\.\.\. lval.Token(l.createToken(l.Token())); return int(T_ELLIPSIS)
<PHP>:: lval.Token(l.createToken(l.Token())); return int(T_PAAMAYIM_NEKUDOTAYIM) // T_DOUBLE_COLON
<PHP>&& lval.Token(l.createToken(l.Token())); return int(T_BOOLEAN_AND)
<PHP>\|\| lval.Token(l.createToken(l.Token())); return int(T_BOOLEAN_OR)
<PHP>&= lval.Token(l.createToken(l.Token())); return int(T_AND_EQUAL)
<PHP>\|= lval.Token(l.createToken(l.Token())); return int(T_OR_EQUAL)
<PHP>\.= lval.Token(l.createToken(l.Token())); return int(T_CONCAT_EQUAL)
<PHP>\*= lval.Token(l.createToken(l.Token())); return int(T_MUL_EQUAL)
<PHP>\*\*= lval.Token(l.createToken(l.Token())); return int(T_POW_EQUAL)
<PHP>[/]= lval.Token(l.createToken(l.Token())); return int(T_DIV_EQUAL)
<PHP>\+= lval.Token(l.createToken(l.Token())); return int(T_PLUS_EQUAL)
<PHP>-= lval.Token(l.createToken(l.Token())); return int(T_MINUS_EQUAL)
<PHP>\^= lval.Token(l.createToken(l.Token())); return int(T_XOR_EQUAL)
<PHP>%= lval.Token(l.createToken(l.Token())); return int(T_MOD_EQUAL)
<PHP>-- lval.Token(l.createToken(l.Token())); return int(T_DEC)
<PHP>\+\+ lval.Token(l.createToken(l.Token())); return int(T_INC)
<PHP>=> lval.Token(l.createToken(l.Token())); return int(T_DOUBLE_ARROW)
<PHP>\<=\> lval.Token(l.createToken(l.Token())); return int(T_SPACESHIP)
<PHP>\!=|\<\> lval.Token(l.createToken(l.Token())); return int(T_IS_NOT_EQUAL)
<PHP>\!== lval.Token(l.createToken(l.Token())); return int(T_IS_NOT_IDENTICAL)
<PHP>== lval.Token(l.createToken(l.Token())); return int(T_IS_EQUAL)
<PHP>=== lval.Token(l.createToken(l.Token())); return int(T_IS_IDENTICAL)
<PHP>\<\<= lval.Token(l.createToken(l.Token())); return int(T_SL_EQUAL)
<PHP>\>\>= lval.Token(l.createToken(l.Token())); return int(T_SR_EQUAL)
<PHP>\>= lval.Token(l.createToken(l.Token())); return int(T_IS_GREATER_OR_EQUAL)
<PHP>\<= lval.Token(l.createToken(l.Token())); return int(T_IS_SMALLER_OR_EQUAL)
<PHP>\*\* lval.Token(l.createToken(l.Token())); return int(T_POW)
<PHP>\<\< lval.Token(l.createToken(l.Token())); return int(T_SL)
<PHP>\>\> lval.Token(l.createToken(l.Token())); return int(T_SR)
<PHP>\?\? lval.Token(l.createToken(l.Token())); return int(T_COALESCE)
<PHP>(#|[/][/])
tb := l.Token()
@ -319,16 +319,16 @@ NEW_LINE (\r|\n|\r\n)
<PHP>\{ l.pushState(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<PHP>\} l.popState(); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0])); l.PhpDocComment = ""
<PHP>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return T_VARIABLE
<PHP>{VAR_NAME} lval.Token(l.createToken(l.Token())); return T_STRING
<PHP>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_VARIABLE)
<PHP>{VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_STRING)
<PHP>-> l.begin(PROPERTY);lval.Token(l.createToken(l.Token())); return T_OBJECT_OPERATOR;
<PHP>-> l.begin(PROPERTY);lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR);
<PROPERTY>[ \t\n\r]+ l.addWhiteSpace(l.Token())
<PROPERTY>-> lval.Token(l.createToken(l.Token())); return T_OBJECT_OPERATOR;
<PROPERTY>{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return T_STRING;
<PROPERTY>-> lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR);
<PROPERTY>{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_STRING);
<PROPERTY>. l.ungetChars(1);l.begin(PHP)
<PHP>[\']([^\\\']*([\\].)*)*[\'] lval.Token(l.createToken(l.Token())); return T_CONSTANT_ENCAPSED_STRING;
<PHP>[\']([^\\\']*([\\].)*)*[\'] lval.Token(l.createToken(l.Token())); return int(T_CONSTANT_ENCAPSED_STRING);
<PHP>` l.begin(BACKQUOTE); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<BACKQUOTE>` l.begin(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
@ -394,7 +394,7 @@ NEW_LINE (\r|\n|\r\n)
l.ungetChars(ungetCnt)
lval.Token(l.createToken(heredocToken));
return T_START_HEREDOC
return int(T_START_HEREDOC)
<NOWDOC>.|[ \t\n\r]
searchLabel := []byte{}
@ -409,12 +409,14 @@ NEW_LINE (\r|\n|\r\n)
if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+1)
tb = tb[:len(tb)-1]
break;
}
if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel))
tb = tb[:len(tb)-1]
break;
}
@ -427,10 +429,10 @@ NEW_LINE (\r|\n|\r\n)
}
lval.Token(l.createToken(tb) )
return T_ENCAPSED_AND_WHITESPACE
return int(T_ENCAPSED_AND_WHITESPACE)
<HEREDOC_END>{VAR_NAME}\; l.begin(PHP);lval.Token(l.createToken(l.ungetChars(1))); return T_END_HEREDOC
<HEREDOC_END>{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return T_END_HEREDOC
<HEREDOC_END>{VAR_NAME}\; l.begin(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_END_HEREDOC)
<HEREDOC_END>{VAR_NAME} l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_END_HEREDOC)
<PHP>[b]?[\"]
binPrefix := l.Token()[0].Rune == 'b'
@ -453,7 +455,7 @@ NEW_LINE (\r|\n|\r\n)
switch c {
case '"' :
c = l.Next();
lval.Token(l.createToken(l.Token())); return T_CONSTANT_ENCAPSED_STRING
lval.Token(l.createToken(l.Token())); return int(T_CONSTANT_ENCAPSED_STRING)
break F;
case '$':
@ -480,8 +482,8 @@ NEW_LINE (\r|\n|\r\n)
}
<STRING>\" l.popState(); lval.Token(l.createToken(l.Token())); return Rune2Class(l.Token()[0].Rune)
<STRING,HEREDOC,BACKQUOTE>\{\$ lval.Token(l.createToken(l.ungetChars(1))); l.pushState(PHP); return T_CURLY_OPEN
<STRING,HEREDOC,BACKQUOTE>\$\{ l.pushState(STRING_VAR_NAME); lval.Token(l.createToken(l.Token())); return T_DOLLAR_OPEN_CURLY_BRACES
<STRING,HEREDOC,BACKQUOTE>\{\$ lval.Token(l.createToken(l.ungetChars(1))); l.pushState(PHP); return int(T_CURLY_OPEN)
<STRING,HEREDOC,BACKQUOTE>\$\{ l.pushState(STRING_VAR_NAME); lval.Token(l.createToken(l.Token())); return int(T_DOLLAR_OPEN_CURLY_BRACES)
<STRING,HEREDOC,BACKQUOTE>\${VAR_NAME} l.ungetChars(len(l.Token()));l.pushState(STRING_VAR)
<STRING>.|[ \t\n\r]
currentChar := l.Prev
@ -492,14 +494,14 @@ NEW_LINE (\r|\n|\r\n)
if c == '{' || isValidFirstVarNameRune(rune(c)) {
l.ungetChars(1)
lval.Token(l.createToken(tb[:len(tb)-1]));
return T_ENCAPSED_AND_WHITESPACE
return int(T_ENCAPSED_AND_WHITESPACE)
}
case '{':
if rune(c) == '$' {
l.ungetChars(1)
lval.Token(l.createToken(tb[:len(tb)-1]));
return T_ENCAPSED_AND_WHITESPACE
return int(T_ENCAPSED_AND_WHITESPACE)
}
case '\\':
@ -510,7 +512,7 @@ NEW_LINE (\r|\n|\r\n)
if rune(c) == '"' {
lval.Token(l.createToken(l.Token()));
return T_ENCAPSED_AND_WHITESPACE
return int(T_ENCAPSED_AND_WHITESPACE)
}
currentChar = l.Last
@ -532,14 +534,14 @@ NEW_LINE (\r|\n|\r\n)
if c == '{' || isValidFirstVarNameRune(rune(c)) {
l.ungetChars(1)
lval.Token(l.createToken(tb[:len(tb)-1]));
return T_ENCAPSED_AND_WHITESPACE
return int(T_ENCAPSED_AND_WHITESPACE)
}
case '{':
if rune(c) == '$' {
l.ungetChars(1)
lval.Token(l.createToken(tb[:len(tb)-1]));
return T_ENCAPSED_AND_WHITESPACE
return int(T_ENCAPSED_AND_WHITESPACE)
}
case '\\':
@ -550,7 +552,7 @@ NEW_LINE (\r|\n|\r\n)
if rune(c) == '`' {
lval.Token(l.createToken(l.Token()));
return T_ENCAPSED_AND_WHITESPACE
return int(T_ENCAPSED_AND_WHITESPACE)
}
currentChar = l.Last
@ -564,23 +566,20 @@ NEW_LINE (\r|\n|\r\n)
<HEREDOC>.|[ \t\n\r]
searchLabel := []byte{}
tb := []lex.Char{}
for {
if c == -1 {
break;
}
currentChar := l.Prev
tb := []lex.Char{currentChar}
HEREDOC_FOR:for {
nls := 0
switch c {
switch currentChar.Rune {
case '\r':
nls = 1
c := l.Next()
if c != '\n' {
nls = 0
l.ungetChars(0)
if c == '\n' {
nls = 1
currentChar := l.Last
tb = append(tb, currentChar)
c = l.Next();
}
fallthrough
@ -588,66 +587,84 @@ NEW_LINE (\r|\n|\r\n)
case '\n':
if l.heredocLabel + ";" == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+1+nls)
l.ungetChars(len(l.heredocLabel)+1+nls)
i := len(tb) - len(l.heredocLabel) - 3 - nls
if i < 1 {
break HEREDOC_FOR;
}
tb = tb[:i]
lval.Token(l.createToken(tb));
return T_ENCAPSED_AND_WHITESPACE
return int(T_ENCAPSED_AND_WHITESPACE)
}
if l.heredocLabel == string(searchLabel) {
l.begin(HEREDOC_END)
tb = l.ungetChars(len(l.heredocLabel)+nls)
l.ungetChars(len(l.heredocLabel)+nls)
i := len(tb) - len(l.heredocLabel) - 2 - nls
if i < 1 {
break HEREDOC_FOR;
}
tb = tb[:i]
lval.Token(l.createToken(tb));
return T_ENCAPSED_AND_WHITESPACE
return int(T_ENCAPSED_AND_WHITESPACE)
}
searchLabel = []byte{}
case '$':
c = l.Next();
if rune(c) == '{' || isValidFirstVarNameRune(rune(c)) {
tb = l.ungetChars(1)
lval.Token(l.createToken(tb));
return T_ENCAPSED_AND_WHITESPACE
if c == '{' || isValidFirstVarNameRune(rune(c)) {
l.ungetChars(1)
lval.Token(l.createToken(tb[:len(tb)-1]));
return int(T_ENCAPSED_AND_WHITESPACE)
}
l.ungetChars(0)
case '{':
c = l.Next();
if rune(c) == '$' {
tb = l.ungetChars(1)
lval.Token(l.createToken(tb));
return T_ENCAPSED_AND_WHITESPACE
l.ungetChars(1)
lval.Token(l.createToken(tb[:len(tb)-1]));
return int(T_ENCAPSED_AND_WHITESPACE)
}
l.ungetChars(0)
case '\\':
c = l.Next();
if c == '\n' || c == '\r' {
l.ungetChars(0)
if c != '\n' && c != '\r' {
currentChar := l.Last
tb = append(tb, currentChar)
c = l.Next();
}
default:
searchLabel = append(searchLabel, byte(rune(c)))
searchLabel = append(searchLabel, byte(rune(currentChar.Rune)))
}
if c == -1 {
break;
}
currentChar = l.Last
tb = append(tb, currentChar)
c = l.Next()
}
<STRING_VAR>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return T_VARIABLE
<STRING_VAR>->{VAR_NAME} lval.Token(l.createToken(l.ungetChars(len(l.Token())-2))); return T_OBJECT_OPERATOR
<STRING_VAR>{VAR_NAME} l.popState();lval.Token(l.createToken(l.Token())); return T_STRING
<STRING_VAR>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_VARIABLE)
<STRING_VAR>->{VAR_NAME} lval.Token(l.createToken(l.ungetChars(len(l.Token())-2))); return int(T_OBJECT_OPERATOR)
<STRING_VAR>{VAR_NAME} l.popState();lval.Token(l.createToken(l.Token())); return int(T_STRING)
<STRING_VAR>\[ l.pushState(STRING_VAR_INDEX);lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR>.|[ \t\n\r] l.ungetChars(1);l.popState()
<STRING_VAR_INDEX>{LNUM}|{HNUM}|{BNUM} lval.Token(l.createToken(l.Token())); return T_NUM_STRING
<STRING_VAR_INDEX>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return T_VARIABLE
<STRING_VAR_INDEX>{VAR_NAME} lval.Token(l.createToken(l.Token())); return T_STRING
<STRING_VAR_INDEX>{LNUM}|{HNUM}|{BNUM} lval.Token(l.createToken(l.Token())); return int(T_NUM_STRING)
<STRING_VAR_INDEX>\${VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_VARIABLE)
<STRING_VAR_INDEX>{VAR_NAME} lval.Token(l.createToken(l.Token())); return int(T_STRING)
<STRING_VAR_INDEX>\] l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_INDEX>[ \n\r\t\\'#] l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return T_ENCAPSED_AND_WHITESPACE
<STRING_VAR_INDEX>[ \n\r\t\\'#] l.popState(); l.popState();lval.Token(l.createToken(l.Token())); return int(T_ENCAPSED_AND_WHITESPACE)
<STRING_VAR_INDEX>{OPERATORS} lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_INDEX>. lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_NAME>{VAR_NAME}[\[\}] l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return T_STRING_VARNAME
<STRING_VAR_NAME>{VAR_NAME}[\[\}] l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_STRING_VARNAME)
<STRING_VAR_NAME>. l.ungetChars(1);l.popState();l.pushState(PHP)
%%

780
scanner/scanner_test.go
View File

@ -230,194 +230,194 @@ func TestTokens(t *testing.T) {
`
expected := []int{
scanner.T_INLINE_HTML,
scanner.Rune2Class(';'),
scanner.T_ECHO,
scanner.Rune2Class(';'),
expected := []string{
scanner.T_INLINE_HTML.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_ECHO.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_DNUMBER,
scanner.T_DNUMBER,
scanner.T_DNUMBER,
scanner.T_DNUMBER,
scanner.T_DNUMBER.String(),
scanner.T_DNUMBER.String(),
scanner.T_DNUMBER.String(),
scanner.T_DNUMBER.String(),
scanner.T_LNUMBER,
scanner.T_DNUMBER,
scanner.T_LNUMBER.String(),
scanner.T_DNUMBER.String(),
scanner.T_LNUMBER,
scanner.T_DNUMBER,
scanner.T_LNUMBER.String(),
scanner.T_DNUMBER.String(),
scanner.T_LNUMBER,
scanner.T_DNUMBER,
scanner.T_LNUMBER.String(),
scanner.T_DNUMBER.String(),
scanner.T_ABSTRACT,
scanner.T_ARRAY,
scanner.T_AS,
scanner.T_BREAK,
scanner.T_CALLABLE,
scanner.T_CASE,
scanner.T_CATCH,
scanner.T_CLASS,
scanner.T_CLONE,
scanner.T_CONST,
scanner.T_CONTINUE,
scanner.T_DECLARE,
scanner.T_DEFAULT,
scanner.T_DO,
scanner.T_ECHO,
scanner.T_ELSE,
scanner.T_ELSEIF,
scanner.T_EMPTY,
scanner.T_ENDDECLARE,
scanner.T_ENDFOR,
scanner.T_ENDFOREACH,
scanner.T_ENDIF,
scanner.T_ENDSWITCH,
scanner.T_ENDWHILE,
scanner.T_EVAL,
scanner.T_EXIT,
scanner.T_EXTENDS,
scanner.T_FINAL,
scanner.T_FINALLY,
scanner.T_FOR,
scanner.T_FOREACH,
scanner.T_FUNCTION,
scanner.T_FUNCTION,
scanner.T_GLOBAL,
scanner.T_GOTO,
scanner.T_IF,
scanner.T_ISSET,
scanner.T_IMPLEMENTS,
scanner.T_INSTANCEOF,
scanner.T_INSTEADOF,
scanner.T_INTERFACE,
scanner.T_LIST,
scanner.T_NAMESPACE,
scanner.T_PRIVATE,
scanner.T_PUBLIC,
scanner.T_PRINT,
scanner.T_PROTECTED,
scanner.T_RETURN,
scanner.T_STATIC,
scanner.T_SWITCH,
scanner.T_THROW,
scanner.T_TRAIT,
scanner.T_TRY,
scanner.T_UNSET,
scanner.T_USE,
scanner.T_VAR,
scanner.T_WHILE,
scanner.T_YIELD_FROM,
scanner.T_YIELD,
scanner.T_INCLUDE,
scanner.T_INCLUDE_ONCE,
scanner.T_REQUIRE,
scanner.T_REQUIRE_ONCE,
scanner.T_ABSTRACT.String(),
scanner.T_ARRAY.String(),
scanner.T_AS.String(),
scanner.T_BREAK.String(),
scanner.T_CALLABLE.String(),
scanner.T_CASE.String(),
scanner.T_CATCH.String(),
scanner.T_CLASS.String(),
scanner.T_CLONE.String(),
scanner.T_CONST.String(),
scanner.T_CONTINUE.String(),
scanner.T_DECLARE.String(),
scanner.T_DEFAULT.String(),
scanner.T_DO.String(),
scanner.T_ECHO.String(),
scanner.T_ELSE.String(),
scanner.T_ELSEIF.String(),
scanner.T_EMPTY.String(),
scanner.T_ENDDECLARE.String(),
scanner.T_ENDFOR.String(),
scanner.T_ENDFOREACH.String(),
scanner.T_ENDIF.String(),
scanner.T_ENDSWITCH.String(),
scanner.T_ENDWHILE.String(),
scanner.T_EVAL.String(),
scanner.T_EXIT.String(),
scanner.T_EXTENDS.String(),
scanner.T_FINAL.String(),
scanner.T_FINALLY.String(),
scanner.T_FOR.String(),
scanner.T_FOREACH.String(),
scanner.T_FUNCTION.String(),
scanner.T_FUNCTION.String(),
scanner.T_GLOBAL.String(),
scanner.T_GOTO.String(),
scanner.T_IF.String(),
scanner.T_ISSET.String(),
scanner.T_IMPLEMENTS.String(),
scanner.T_INSTANCEOF.String(),
scanner.T_INSTEADOF.String(),
scanner.T_INTERFACE.String(),
scanner.T_LIST.String(),
scanner.T_NAMESPACE.String(),
scanner.T_PRIVATE.String(),
scanner.T_PUBLIC.String(),
scanner.T_PRINT.String(),
scanner.T_PROTECTED.String(),
scanner.T_RETURN.String(),
scanner.T_STATIC.String(),
scanner.T_SWITCH.String(),
scanner.T_THROW.String(),
scanner.T_TRAIT.String(),
scanner.T_TRY.String(),
scanner.T_UNSET.String(),
scanner.T_USE.String(),
scanner.T_VAR.String(),
scanner.T_WHILE.String(),
scanner.T_YIELD_FROM.String(),
scanner.T_YIELD.String(),
scanner.T_INCLUDE.String(),
scanner.T_INCLUDE_ONCE.String(),
scanner.T_REQUIRE.String(),
scanner.T_REQUIRE_ONCE.String(),
scanner.T_CLASS_C,
scanner.T_DIR,
scanner.T_FILE,
scanner.T_FUNC_C,
scanner.T_LINE,
scanner.T_NS_C,
scanner.T_METHOD_C,
scanner.T_TRAIT_C,
scanner.T_HALT_COMPILER,
scanner.T_CLASS_C.String(),
scanner.T_DIR.String(),
scanner.T_FILE.String(),
scanner.T_FUNC_C.String(),
scanner.T_LINE.String(),
scanner.T_NS_C.String(),
scanner.T_METHOD_C.String(),
scanner.T_TRAIT_C.String(),
scanner.T_HALT_COMPILER.String(),
scanner.T_NEW,
scanner.T_LOGICAL_AND,
scanner.T_LOGICAL_OR,
scanner.T_LOGICAL_XOR,
scanner.T_NEW.String(),
scanner.T_LOGICAL_AND.String(),
scanner.T_LOGICAL_OR.String(),
scanner.T_LOGICAL_XOR.String(),
scanner.T_NS_SEPARATOR,
scanner.T_ELLIPSIS,
scanner.T_PAAMAYIM_NEKUDOTAYIM,
scanner.T_BOOLEAN_AND,
scanner.T_BOOLEAN_OR,
scanner.T_AND_EQUAL,
scanner.T_OR_EQUAL,
scanner.T_CONCAT_EQUAL,
scanner.T_MUL_EQUAL,
scanner.T_POW_EQUAL,
scanner.T_DIV_EQUAL,
scanner.T_PLUS_EQUAL,
scanner.T_MINUS_EQUAL,
scanner.T_XOR_EQUAL,
scanner.T_MOD_EQUAL,
scanner.T_DEC,
scanner.T_INC,
scanner.T_DOUBLE_ARROW,
scanner.T_SPACESHIP,
scanner.T_IS_NOT_EQUAL,
scanner.T_IS_NOT_EQUAL,
scanner.T_IS_NOT_IDENTICAL,
scanner.T_IS_EQUAL,
scanner.T_IS_IDENTICAL,
scanner.T_SL_EQUAL,
scanner.T_SR_EQUAL,
scanner.T_IS_GREATER_OR_EQUAL,
scanner.T_IS_SMALLER_OR_EQUAL,
scanner.T_POW,
scanner.T_SL,
scanner.T_SR,
scanner.T_COALESCE,
scanner.T_NS_SEPARATOR.String(),
scanner.T_ELLIPSIS.String(),
scanner.T_PAAMAYIM_NEKUDOTAYIM.String(),
scanner.T_BOOLEAN_AND.String(),
scanner.T_BOOLEAN_OR.String(),
scanner.T_AND_EQUAL.String(),
scanner.T_OR_EQUAL.String(),
scanner.T_CONCAT_EQUAL.String(),
scanner.T_MUL_EQUAL.String(),
scanner.T_POW_EQUAL.String(),
scanner.T_DIV_EQUAL.String(),
scanner.T_PLUS_EQUAL.String(),
scanner.T_MINUS_EQUAL.String(),
scanner.T_XOR_EQUAL.String(),
scanner.T_MOD_EQUAL.String(),
scanner.T_DEC.String(),
scanner.T_INC.String(),
scanner.T_DOUBLE_ARROW.String(),
scanner.T_SPACESHIP.String(),
scanner.T_IS_NOT_EQUAL.String(),
scanner.T_IS_NOT_EQUAL.String(),
scanner.T_IS_NOT_IDENTICAL.String(),
scanner.T_IS_EQUAL.String(),
scanner.T_IS_IDENTICAL.String(),
scanner.T_SL_EQUAL.String(),
scanner.T_SR_EQUAL.String(),
scanner.T_IS_GREATER_OR_EQUAL.String(),
scanner.T_IS_SMALLER_OR_EQUAL.String(),
scanner.T_POW.String(),
scanner.T_SL.String(),
scanner.T_SR.String(),
scanner.T_COALESCE.String(),
scanner.Rune2Class(';'),
scanner.Rune2Class(':'),
scanner.Rune2Class(','),
scanner.Rune2Class('.'),
scanner.Rune2Class('['),
scanner.Rune2Class(']'),
scanner.Rune2Class('('),
scanner.Rune2Class(')'),
scanner.Rune2Class('|'),
scanner.Rune2Class('/'),
scanner.Rune2Class('^'),
scanner.Rune2Class('&'),
scanner.Rune2Class('+'),
scanner.Rune2Class('-'),
scanner.Rune2Class('*'),
scanner.Rune2Class('='),
scanner.Rune2Class('%'),
scanner.Rune2Class('!'),
scanner.Rune2Class('~'),
scanner.Rune2Class('$'),
scanner.Rune2Class('<'),
scanner.Rune2Class('>'),
scanner.Rune2Class('?'),
scanner.Rune2Class('@'),
scanner.Rune2Class('{'),
scanner.Rune2Class('}'),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.LexerToken(scanner.Rune2Class(':')).String(),
scanner.LexerToken(scanner.Rune2Class(',')).String(),
scanner.LexerToken(scanner.Rune2Class('.')).String(),
scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.LexerToken(scanner.Rune2Class('(')).String(),
scanner.LexerToken(scanner.Rune2Class(')')).String(),
scanner.LexerToken(scanner.Rune2Class('|')).String(),
scanner.LexerToken(scanner.Rune2Class('/')).String(),
scanner.LexerToken(scanner.Rune2Class('^')).String(),
scanner.LexerToken(scanner.Rune2Class('&')).String(),
scanner.LexerToken(scanner.Rune2Class('+')).String(),
scanner.LexerToken(scanner.Rune2Class('-')).String(),
scanner.LexerToken(scanner.Rune2Class('*')).String(),
scanner.LexerToken(scanner.Rune2Class('=')).String(),
scanner.LexerToken(scanner.Rune2Class('%')).String(),
scanner.LexerToken(scanner.Rune2Class('!')).String(),
scanner.LexerToken(scanner.Rune2Class('~')).String(),
scanner.LexerToken(scanner.Rune2Class('$')).String(),
scanner.LexerToken(scanner.Rune2Class('<')).String(),
scanner.LexerToken(scanner.Rune2Class('>')).String(),
scanner.LexerToken(scanner.Rune2Class('?')).String(),
scanner.LexerToken(scanner.Rune2Class('@')).String(),
scanner.LexerToken(scanner.Rune2Class('{')).String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_VARIABLE,
scanner.T_STRING,
scanner.T_VARIABLE.String(),
scanner.T_STRING.String(),
scanner.T_OBJECT_OPERATOR,
scanner.T_OBJECT_OPERATOR,
scanner.T_STRING,
scanner.T_OBJECT_OPERATOR.String(),
scanner.T_OBJECT_OPERATOR.String(),
scanner.T_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING,
scanner.T_CONSTANT_ENCAPSED_STRING,
scanner.T_CONSTANT_ENCAPSED_STRING,
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.T_ARRAY_CAST,
scanner.T_BOOL_CAST,
scanner.T_BOOL_CAST,
scanner.T_DOUBLE_CAST,
scanner.T_DOUBLE_CAST,
scanner.T_DOUBLE_CAST,
scanner.T_INT_CAST,
scanner.T_INT_CAST,
scanner.T_OBJECT_CAST,
scanner.T_STRING_CAST,
scanner.T_STRING_CAST,
scanner.T_UNSET_CAST,
scanner.T_ARRAY_CAST.String(),
scanner.T_BOOL_CAST.String(),
scanner.T_BOOL_CAST.String(),
scanner.T_DOUBLE_CAST.String(),
scanner.T_DOUBLE_CAST.String(),
scanner.T_DOUBLE_CAST.String(),
scanner.T_INT_CAST.String(),
scanner.T_INT_CAST.String(),
scanner.T_OBJECT_CAST.String(),
scanner.T_STRING_CAST.String(),
scanner.T_STRING_CAST.String(),
scanner.T_UNSET_CAST.String(),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
actual := []string{}
for {
token := lexer.Lex(lv)
@ -425,7 +425,7 @@ func TestTokens(t *testing.T) {
break
}
actual = append(actual, token)
actual = append(actual, scanner.LexerToken(token).String())
}
assertEqual(t, expected, actual)
@ -448,65 +448,65 @@ func TestTeplateStringTokens(t *testing.T) {
"$0$foo"
`
expected := []int{
scanner.Rune2Class('"'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('"'),
expected := []string{
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_CURLY_OPEN,
scanner.T_VARIABLE,
scanner.Rune2Class('}'),
scanner.Rune2Class('"'),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_CURLY_OPEN,
scanner.T_VARIABLE,
scanner.Rune2Class('}'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_DOLLAR_OPEN_CURLY_BRACES,
scanner.T_STRING_VARNAME,
scanner.Rune2Class('}'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.Rune2Class('"'),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_DOLLAR_OPEN_CURLY_BRACES.String(),
scanner.T_STRING_VARNAME.String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'),
scanner.T_CURLY_OPEN,
scanner.T_VARIABLE,
scanner.Rune2Class('}'),
scanner.Rune2Class('"'),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'),
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.Rune2Class('"'),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'),
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.Rune2Class('"'),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('"'),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.Rune2Class('"'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('"'),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
actual := []string{}
for {
token := lexer.Lex(lv)
@ -514,7 +514,7 @@ func TestTeplateStringTokens(t *testing.T) {
break
}
actual = append(actual, token)
actual = append(actual, scanner.LexerToken(token).String())
}
assertEqual(t, expected, actual)
@ -534,65 +534,65 @@ func TestBackquoteStringTokens(t *testing.T) {
` + "`$0$foo`" + `
`
expected := []int{
scanner.Rune2Class('`'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('`'),
expected := []string{
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_CURLY_OPEN,
scanner.T_VARIABLE,
scanner.Rune2Class('}'),
scanner.Rune2Class('`'),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_CURLY_OPEN,
scanner.T_VARIABLE,
scanner.Rune2Class('}'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_DOLLAR_OPEN_CURLY_BRACES,
scanner.T_STRING_VARNAME,
scanner.Rune2Class('}'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.Rune2Class('`'),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_DOLLAR_OPEN_CURLY_BRACES.String(),
scanner.T_STRING_VARNAME.String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'),
scanner.T_CURLY_OPEN,
scanner.T_VARIABLE,
scanner.Rune2Class('}'),
scanner.Rune2Class('`'),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'),
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.Rune2Class('`'),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'),
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.Rune2Class('`'),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('`'),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.Rune2Class('`'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('`'),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('`')).String(),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
actual := []string{}
for {
token := lexer.Lex(lv)
@ -600,7 +600,7 @@ func TestBackquoteStringTokens(t *testing.T) {
break
}
actual = append(actual, token)
actual = append(actual, scanner.LexerToken(token).String())
}
assertEqual(t, expected, actual)
@ -630,63 +630,64 @@ CAT;
CAT;
`
expected := []int{
scanner.T_START_HEREDOC,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
expected := []string{
scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_END_HEREDOC.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_START_HEREDOC,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_END_HEREDOC.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_START_HEREDOC,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_OBJECT_OPERATOR,
scanner.T_STRING,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('['),
scanner.T_NUM_STRING,
scanner.Rune2Class(']'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('['),
scanner.T_NUM_STRING,
scanner.Rune2Class(']'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('['),
scanner.T_NUM_STRING,
scanner.Rune2Class(']'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('['),
scanner.T_STRING,
scanner.Rune2Class(']'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('['),
scanner.T_VARIABLE,
scanner.Rune2Class(']'),
scanner.T_ENCAPSED_AND_WHITESPACE, scanner.T_CURLY_OPEN,
scanner.T_VARIABLE,
scanner.Rune2Class('}'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_DOLLAR_OPEN_CURLY_BRACES,
scanner.T_STRING_VARNAME,
scanner.Rune2Class('}'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.T_OBJECT_OPERATOR.String(),
scanner.T_STRING.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_NUM_STRING.String(),
scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_NUM_STRING.String(),
scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_NUM_STRING.String(),
scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_STRING.String(),
scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_DOLLAR_OPEN_CURLY_BRACES.String(),
scanner.T_STRING_VARNAME.String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_END_HEREDOC.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
actual := []string{}
for {
token := lexer.Lex(lv)
@ -694,7 +695,7 @@ CAT;
break
}
actual = append(actual, token)
actual = append(actual, scanner.LexerToken(token).String())
}
assertEqual(t, expected, actual)
@ -723,44 +724,42 @@ $foo$bar\
CAT
`
expected := []int{
scanner.T_START_HEREDOC,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
expected := []string{
scanner.T_START_HEREDOC.String(),
scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_END_HEREDOC.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_START_HEREDOC,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC.String(),
scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_END_HEREDOC.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_START_HEREDOC,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.T_END_HEREDOC.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_START_HEREDOC,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.T_END_HEREDOC.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_START_HEREDOC,
scanner.T_VARIABLE,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.T_START_HEREDOC.String(),
scanner.T_VARIABLE.String(),
scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_END_HEREDOC.String(),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
actual := []string{}
for {
token := lexer.Lex(lv)
@ -768,7 +767,46 @@ CAT
break
}
actual = append(actual, token)
actual = append(actual, scanner.LexerToken(token).String())
}
assertEqual(t, expected, actual)
}
func TestHereDocTokens3(t *testing.T) {
src := `<?php
<<<"CAT"
\\{$a['b']}
CAT;
`
expected := []string{
scanner.T_START_HEREDOC.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_CURLY_OPEN.String(),
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('[')).String(),
scanner.T_CONSTANT_ENCAPSED_STRING.String(),
scanner.LexerToken(scanner.Rune2Class(']')).String(),
scanner.LexerToken(scanner.Rune2Class('}')).String(),
scanner.T_END_HEREDOC.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []string{}
for {
token := lexer.Lex(lv)
if token < 0 {
break
}
actual = append(actual, scanner.LexerToken(token).String())
}
assertEqual(t, expected, actual)
@ -780,20 +818,20 @@ func TestInlineHtmlNopTokens(t *testing.T) {
$a ?> test
`
expected := []int{
scanner.T_VARIABLE,
scanner.Rune2Class(';'),
scanner.T_INLINE_HTML,
expected := []string{
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_INLINE_HTML.String(),
scanner.T_VARIABLE,
scanner.Rune2Class(';'),
scanner.T_INLINE_HTML,
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class(';')).String(),
scanner.T_INLINE_HTML.String(),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lexer.WithMeta = true
lv := &lval{}
actual := []int{}
actual := []string{}
for {
token := lexer.Lex(lv)
@ -801,7 +839,7 @@ func TestInlineHtmlNopTokens(t *testing.T) {
break
}
actual = append(actual, token)
actual = append(actual, scanner.LexerToken(token).String())
}
assertEqual(t, expected, actual)
@ -810,12 +848,12 @@ func TestInlineHtmlNopTokens(t *testing.T) {
func TestStringTokensAfterVariable(t *testing.T) {
src := `<?php "test \"$var\""`
expected := []int{
scanner.Rune2Class('"'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.Rune2Class('"'),
expected := []string{
scanner.LexerToken(scanner.Rune2Class('"')).String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.T_VARIABLE.String(),
scanner.T_ENCAPSED_AND_WHITESPACE.String(),
scanner.LexerToken(scanner.Rune2Class('"')).String(),
}
expectedTokens := []string{
@ -828,7 +866,7 @@ func TestStringTokensAfterVariable(t *testing.T) {
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lv := &lval{}
actual := []int{}
actual := []string{}
actualTokens := []string{}
for {
@ -838,7 +876,7 @@ func TestStringTokensAfterVariable(t *testing.T) {
}
actualTokens = append(actualTokens, lv.Tkn.Value)
actual = append(actual, token)
actual = append(actual, scanner.LexerToken(token).String())
}
assertEqual(t, expected, actual)
@ -848,10 +886,10 @@ func TestStringTokensAfterVariable(t *testing.T) {
func TestSlashAfterVariable(t *testing.T) {
src := `<?php $foo/3`
expected := []int{
scanner.T_VARIABLE,
scanner.Rune2Class('/'),
scanner.T_LNUMBER,
expected := []string{
scanner.T_VARIABLE.String(),
scanner.LexerToken(scanner.Rune2Class('/')).String(),
scanner.T_LNUMBER.String(),
}
expectedTokens := []string{
@ -862,7 +900,7 @@ func TestSlashAfterVariable(t *testing.T) {
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lv := &lval{}
actual := []int{}
actual := []string{}
actualTokens := []string{}
for {
@ -872,7 +910,7 @@ func TestSlashAfterVariable(t *testing.T) {
}
actualTokens = append(actualTokens, lv.Tkn.Value)
actual = append(actual, token)
actual = append(actual, scanner.LexerToken(token).String())
}
assertEqual(t, expected, actual)