maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

scanner refactoring

Browse Source
This commit is contained in:
z7zmey 2019-12-21 14:44:25 +02:00
parent 498e93d178
commit d2c76460ab
3 changed files with 780 additions and 787 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scanner/lexer.go
View File

@ -75,18 +75,11 @@ func (l *Lexer) SetErrors(e []*errors.Error) {
l.Errors = e l.Errors = e
} }
func (lex *Lexer) createToken(lval Lval) *Token { func (lex *Lexer) setTokenPosition(token *Token) {
token := lex.TokenPool.Get()
token.FreeFloating = lex.FreeFloating
token.Value = string(lex.data[lex.ts:lex.te])
token.StartLine = lex.NewLines.GetLine(lex.ts) token.StartLine = lex.NewLines.GetLine(lex.ts)
token.EndLine = lex.NewLines.GetLine(lex.te - 1) token.EndLine = lex.NewLines.GetLine(lex.te - 1)
token.StartPos = lex.ts token.StartPos = lex.ts
token.EndPos = lex.te token.EndPos = lex.te
lval.Token(token)
return token
} }
func (lex *Lexer) addFreeFloating(t freefloating.StringType, ps, pe int) { func (lex *Lexer) addFreeFloating(t freefloating.StringType, ps, pe int) {

1214
scanner/scanner.go
View File

File diff suppressed because it is too large Load Diff

344
scanner/scanner.rl
View File

@ -32,6 +32,10 @@ func (lex *Lexer) Lex(lval Lval) int {
eof := lex.pe eof := lex.pe
var tok TokenID var tok TokenID
token := lex.TokenPool.Get()
token.FreeFloating = lex.FreeFloating
token.Value = string(lex.data[0:0])
lblStart := 0 lblStart := 0
lblEnd := 0 lblEnd := 0
@ -130,7 +134,7 @@ func (lex *Lexer) Lex(lval Lval) int {
main := |* main := |*
any_line+ -- '<?' => { any_line+ -- '<?' => {
lex.ungetStr("<") lex.ungetStr("<")
lex.createToken(lval) lex.setTokenPosition(token)
tok = T_INLINE_HTML; tok = T_INLINE_HTML;
fbreak; fbreak;
}; };
@ -144,7 +148,7 @@ func (lex *Lexer) Lex(lval Lval) int {
fnext php; fnext php;
}; };
'<?='i => { '<?='i => {
lex.createToken(lval); lex.setTokenPosition(token);
tok = T_ECHO; tok = T_ECHO;
fnext php; fnext php;
fbreak; fbreak;
@ -153,10 +157,10 @@ func (lex *Lexer) Lex(lval Lval) int {
php := |* php := |*
whitespace_line* => {lex.addFreeFloating(freefloating.WhiteSpaceType, lex.ts, lex.te)}; whitespace_line* => {lex.addFreeFloating(freefloating.WhiteSpaceType, lex.ts, lex.te)};
'?>' newline? => {lex.createToken(lval); tok = TokenID(int(';')); fnext main; fbreak;}; '?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext main; fbreak;};
';' whitespace_line* '?>' newline? => {lex.createToken(lval); tok = TokenID(int(';')); fnext main; fbreak;}; ';' whitespace_line* '?>' newline? => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext main; fbreak;};
(dnum | exponent_dnum) => {lex.createToken(lval); tok = T_DNUMBER; fbreak;}; (dnum | exponent_dnum) => {lex.setTokenPosition(token); tok = T_DNUMBER; fbreak;};
bnum => { bnum => {
firstNum := 2 firstNum := 2
for i := lex.ts + 2; i < lex.te; i++ { for i := lex.ts + 2; i < lex.te; i++ {
@ -166,15 +170,15 @@ func (lex *Lexer) Lex(lval Lval) int {
} }
if lex.te - lex.ts - firstNum < 64 { if lex.te - lex.ts - firstNum < 64 {
lex.createToken(lval); tok = T_LNUMBER; fbreak; lex.setTokenPosition(token); tok = T_LNUMBER; fbreak;
} }
lex.createToken(lval); tok = T_DNUMBER; fbreak; lex.setTokenPosition(token); tok = T_DNUMBER; fbreak;
}; };
lnum => { lnum => {
if lex.te - lex.ts < 20 { if lex.te - lex.ts < 20 {
lex.createToken(lval); tok = T_LNUMBER; fbreak; lex.setTokenPosition(token); tok = T_LNUMBER; fbreak;
} }
lex.createToken(lval); tok = T_DNUMBER; fbreak; lex.setTokenPosition(token); tok = T_DNUMBER; fbreak;
}; };
hnum => { hnum => {
firstNum := lex.ts + 2 firstNum := lex.ts + 2
@ -186,125 +190,125 @@ func (lex *Lexer) Lex(lval Lval) int {
length := lex.te - firstNum length := lex.te - firstNum
if length < 16 || (length == 16 && lex.data[firstNum] <= '7') { if length < 16 || (length == 16 && lex.data[firstNum] <= '7') {
lex.createToken(lval); tok = T_LNUMBER; fbreak; lex.setTokenPosition(token); tok = T_LNUMBER; fbreak;
} }
lex.createToken(lval); tok = T_DNUMBER; fbreak; lex.setTokenPosition(token); tok = T_DNUMBER; fbreak;
}; };
'abstract'i => {lex.createToken(lval); tok = T_ABSTRACT; fbreak;}; 'abstract'i => {lex.setTokenPosition(token); tok = T_ABSTRACT; fbreak;};
'array'i => {lex.createToken(lval); tok = T_ARRAY; fbreak;}; 'array'i => {lex.setTokenPosition(token); tok = T_ARRAY; fbreak;};
'as'i => {lex.createToken(lval); tok = T_AS; fbreak;}; 'as'i => {lex.setTokenPosition(token); tok = T_AS; fbreak;};
'break'i => {lex.createToken(lval); tok = T_BREAK; fbreak;}; 'break'i => {lex.setTokenPosition(token); tok = T_BREAK; fbreak;};
'callable'i => {lex.createToken(lval); tok = T_CALLABLE; fbreak;}; 'callable'i => {lex.setTokenPosition(token); tok = T_CALLABLE; fbreak;};
'case'i => {lex.createToken(lval); tok = T_CASE; fbreak;}; 'case'i => {lex.setTokenPosition(token); tok = T_CASE; fbreak;};
'catch'i => {lex.createToken(lval); tok = T_CATCH; fbreak;}; 'catch'i => {lex.setTokenPosition(token); tok = T_CATCH; fbreak;};
'class'i => {lex.createToken(lval); tok = T_CLASS; fbreak;}; 'class'i => {lex.setTokenPosition(token); tok = T_CLASS; fbreak;};
'clone'i => {lex.createToken(lval); tok = T_CLONE; fbreak;}; 'clone'i => {lex.setTokenPosition(token); tok = T_CLONE; fbreak;};
'const'i => {lex.createToken(lval); tok = T_CONST; fbreak;}; 'const'i => {lex.setTokenPosition(token); tok = T_CONST; fbreak;};
'continue'i => {lex.createToken(lval); tok = T_CONTINUE; fbreak;}; 'continue'i => {lex.setTokenPosition(token); tok = T_CONTINUE; fbreak;};
'declare'i => {lex.createToken(lval); tok = T_DECLARE; fbreak;}; 'declare'i => {lex.setTokenPosition(token); tok = T_DECLARE; fbreak;};
'default'i => {lex.createToken(lval); tok = T_DEFAULT; fbreak;}; 'default'i => {lex.setTokenPosition(token); tok = T_DEFAULT; fbreak;};
'do'i => {lex.createToken(lval); tok = T_DO; fbreak;}; 'do'i => {lex.setTokenPosition(token); tok = T_DO; fbreak;};
'echo'i => {lex.createToken(lval); tok = T_ECHO; fbreak;}; 'echo'i => {lex.setTokenPosition(token); tok = T_ECHO; fbreak;};
'else'i => {lex.createToken(lval); tok = T_ELSE; fbreak;}; 'else'i => {lex.setTokenPosition(token); tok = T_ELSE; fbreak;};
'elseif'i => {lex.createToken(lval); tok = T_ELSEIF; fbreak;}; 'elseif'i => {lex.setTokenPosition(token); tok = T_ELSEIF; fbreak;};
'empty'i => {lex.createToken(lval); tok = T_EMPTY; fbreak;}; 'empty'i => {lex.setTokenPosition(token); tok = T_EMPTY; fbreak;};
'enddeclare'i => {lex.createToken(lval); tok = T_ENDDECLARE; fbreak;}; 'enddeclare'i => {lex.setTokenPosition(token); tok = T_ENDDECLARE; fbreak;};
'endfor'i => {lex.createToken(lval); tok = T_ENDFOR; fbreak;}; 'endfor'i => {lex.setTokenPosition(token); tok = T_ENDFOR; fbreak;};
'endforeach'i => {lex.createToken(lval); tok = T_ENDFOREACH; fbreak;}; 'endforeach'i => {lex.setTokenPosition(token); tok = T_ENDFOREACH; fbreak;};
'endif'i => {lex.createToken(lval); tok = T_ENDIF; fbreak;}; 'endif'i => {lex.setTokenPosition(token); tok = T_ENDIF; fbreak;};
'endswitch'i => {lex.createToken(lval); tok = T_ENDSWITCH; fbreak;}; 'endswitch'i => {lex.setTokenPosition(token); tok = T_ENDSWITCH; fbreak;};
'endwhile'i => {lex.createToken(lval); tok = T_ENDWHILE; fbreak;}; 'endwhile'i => {lex.setTokenPosition(token); tok = T_ENDWHILE; fbreak;};
'eval'i => {lex.createToken(lval); tok = T_EVAL; fbreak;}; 'eval'i => {lex.setTokenPosition(token); tok = T_EVAL; fbreak;};
'exit'i | 'die'i => {lex.createToken(lval); tok = T_EXIT; fbreak;}; 'exit'i | 'die'i => {lex.setTokenPosition(token); tok = T_EXIT; fbreak;};
'extends'i => {lex.createToken(lval); tok = T_EXTENDS; fbreak;}; 'extends'i => {lex.setTokenPosition(token); tok = T_EXTENDS; fbreak;};
'final'i => {lex.createToken(lval); tok = T_FINAL; fbreak;}; 'final'i => {lex.setTokenPosition(token); tok = T_FINAL; fbreak;};
'finally'i => {lex.createToken(lval); tok = T_FINALLY; fbreak;}; 'finally'i => {lex.setTokenPosition(token); tok = T_FINALLY; fbreak;};
'for'i => {lex.createToken(lval); tok = T_FOR; fbreak;}; 'for'i => {lex.setTokenPosition(token); tok = T_FOR; fbreak;};
'foreach'i => {lex.createToken(lval); tok = T_FOREACH; fbreak;}; 'foreach'i => {lex.setTokenPosition(token); tok = T_FOREACH; fbreak;};
'function'i | 'cfunction'i => {lex.createToken(lval); tok = T_FUNCTION; fbreak;}; 'function'i | 'cfunction'i => {lex.setTokenPosition(token); tok = T_FUNCTION; fbreak;};
'global'i => {lex.createToken(lval); tok = T_GLOBAL; fbreak;}; 'global'i => {lex.setTokenPosition(token); tok = T_GLOBAL; fbreak;};
'goto'i => {lex.createToken(lval); tok = T_GOTO; fbreak;}; 'goto'i => {lex.setTokenPosition(token); tok = T_GOTO; fbreak;};
'if'i => {lex.createToken(lval); tok = T_IF; fbreak;}; 'if'i => {lex.setTokenPosition(token); tok = T_IF; fbreak;};
'isset'i => {lex.createToken(lval); tok = T_ISSET; fbreak;}; 'isset'i => {lex.setTokenPosition(token); tok = T_ISSET; fbreak;};
'implements'i => {lex.createToken(lval); tok = T_IMPLEMENTS; fbreak;}; 'implements'i => {lex.setTokenPosition(token); tok = T_IMPLEMENTS; fbreak;};
'instanceof'i => {lex.createToken(lval); tok = T_INSTANCEOF; fbreak;}; 'instanceof'i => {lex.setTokenPosition(token); tok = T_INSTANCEOF; fbreak;};
'insteadof'i => {lex.createToken(lval); tok = T_INSTEADOF; fbreak;}; 'insteadof'i => {lex.setTokenPosition(token); tok = T_INSTEADOF; fbreak;};
'interface'i => {lex.createToken(lval); tok = T_INTERFACE; fbreak;}; 'interface'i => {lex.setTokenPosition(token); tok = T_INTERFACE; fbreak;};
'list'i => {lex.createToken(lval); tok = T_LIST; fbreak;}; 'list'i => {lex.setTokenPosition(token); tok = T_LIST; fbreak;};
'namespace'i => {lex.createToken(lval); tok = T_NAMESPACE; fbreak;}; 'namespace'i => {lex.setTokenPosition(token); tok = T_NAMESPACE; fbreak;};
'private'i => {lex.createToken(lval); tok = T_PRIVATE; fbreak;}; 'private'i => {lex.setTokenPosition(token); tok = T_PRIVATE; fbreak;};
'public'i => {lex.createToken(lval); tok = T_PUBLIC; fbreak;}; 'public'i => {lex.setTokenPosition(token); tok = T_PUBLIC; fbreak;};
'print'i => {lex.createToken(lval); tok = T_PRINT; fbreak;}; 'print'i => {lex.setTokenPosition(token); tok = T_PRINT; fbreak;};
'protected'i => {lex.createToken(lval); tok = T_PROTECTED; fbreak;}; 'protected'i => {lex.setTokenPosition(token); tok = T_PROTECTED; fbreak;};
'return'i => {lex.createToken(lval); tok = T_RETURN; fbreak;}; 'return'i => {lex.setTokenPosition(token); tok = T_RETURN; fbreak;};
'static'i => {lex.createToken(lval); tok = T_STATIC; fbreak;}; 'static'i => {lex.setTokenPosition(token); tok = T_STATIC; fbreak;};
'switch'i => {lex.createToken(lval); tok = T_SWITCH; fbreak;}; 'switch'i => {lex.setTokenPosition(token); tok = T_SWITCH; fbreak;};
'throw'i => {lex.createToken(lval); tok = T_THROW; fbreak;}; 'throw'i => {lex.setTokenPosition(token); tok = T_THROW; fbreak;};
'trait'i => {lex.createToken(lval); tok = T_TRAIT; fbreak;}; 'trait'i => {lex.setTokenPosition(token); tok = T_TRAIT; fbreak;};
'try'i => {lex.createToken(lval); tok = T_TRY; fbreak;}; 'try'i => {lex.setTokenPosition(token); tok = T_TRY; fbreak;};
'unset'i => {lex.createToken(lval); tok = T_UNSET; fbreak;}; 'unset'i => {lex.setTokenPosition(token); tok = T_UNSET; fbreak;};
'use'i => {lex.createToken(lval); tok = T_USE; fbreak;}; 'use'i => {lex.setTokenPosition(token); tok = T_USE; fbreak;};
'var'i => {lex.createToken(lval); tok = T_VAR; fbreak;}; 'var'i => {lex.setTokenPosition(token); tok = T_VAR; fbreak;};
'while'i => {lex.createToken(lval); tok = T_WHILE; fbreak;}; 'while'i => {lex.setTokenPosition(token); tok = T_WHILE; fbreak;};
'yield'i whitespace_line* 'from'i => {lex.createToken(lval); tok = T_YIELD_FROM; fbreak;}; 'yield'i whitespace_line* 'from'i => {lex.setTokenPosition(token); tok = T_YIELD_FROM; fbreak;};
'yield'i => {lex.createToken(lval); tok = T_YIELD; fbreak;}; 'yield'i => {lex.setTokenPosition(token); tok = T_YIELD; fbreak;};
'include'i => {lex.createToken(lval); tok = T_INCLUDE; fbreak;}; 'include'i => {lex.setTokenPosition(token); tok = T_INCLUDE; fbreak;};
'include_once'i => {lex.createToken(lval); tok = T_INCLUDE_ONCE; fbreak;}; 'include_once'i => {lex.setTokenPosition(token); tok = T_INCLUDE_ONCE; fbreak;};
'require'i => {lex.createToken(lval); tok = T_REQUIRE; fbreak;}; 'require'i => {lex.setTokenPosition(token); tok = T_REQUIRE; fbreak;};
'require_once'i => {lex.createToken(lval); tok = T_REQUIRE_ONCE; fbreak;}; 'require_once'i => {lex.setTokenPosition(token); tok = T_REQUIRE_ONCE; fbreak;};
'__CLASS__'i => {lex.createToken(lval); tok = T_CLASS_C; fbreak;}; '__CLASS__'i => {lex.setTokenPosition(token); tok = T_CLASS_C; fbreak;};
'__DIR__'i => {lex.createToken(lval); tok = T_DIR; fbreak;}; '__DIR__'i => {lex.setTokenPosition(token); tok = T_DIR; fbreak;};
'__FILE__'i => {lex.createToken(lval); tok = T_FILE; fbreak;}; '__FILE__'i => {lex.setTokenPosition(token); tok = T_FILE; fbreak;};
'__FUNCTION__'i => {lex.createToken(lval); tok = T_FUNC_C; fbreak;}; '__FUNCTION__'i => {lex.setTokenPosition(token); tok = T_FUNC_C; fbreak;};
'__LINE__'i => {lex.createToken(lval); tok = T_LINE; fbreak;}; '__LINE__'i => {lex.setTokenPosition(token); tok = T_LINE; fbreak;};
'__NAMESPACE__'i => {lex.createToken(lval); tok = T_NS_C; fbreak;}; '__NAMESPACE__'i => {lex.setTokenPosition(token); tok = T_NS_C; fbreak;};
'__METHOD__'i => {lex.createToken(lval); tok = T_METHOD_C; fbreak;}; '__METHOD__'i => {lex.setTokenPosition(token); tok = T_METHOD_C; fbreak;};
'__TRAIT__'i => {lex.createToken(lval); tok = T_TRAIT_C; fbreak;}; '__TRAIT__'i => {lex.setTokenPosition(token); tok = T_TRAIT_C; fbreak;};
'__halt_compiler'i => {lex.createToken(lval); tok = T_HALT_COMPILER; fnext halt_compiller_open_parenthesis; fbreak;}; '__halt_compiler'i => {lex.setTokenPosition(token); tok = T_HALT_COMPILER; fnext halt_compiller_open_parenthesis; fbreak;};
'new'i => {lex.createToken(lval); tok = T_NEW; fbreak;}; 'new'i => {lex.setTokenPosition(token); tok = T_NEW; fbreak;};
'and'i => {lex.createToken(lval); tok = T_LOGICAL_AND; fbreak;}; 'and'i => {lex.setTokenPosition(token); tok = T_LOGICAL_AND; fbreak;};
'or'i => {lex.createToken(lval); tok = T_LOGICAL_OR; fbreak;}; 'or'i => {lex.setTokenPosition(token); tok = T_LOGICAL_OR; fbreak;};
'xor'i => {lex.createToken(lval); tok = T_LOGICAL_XOR; fbreak;}; 'xor'i => {lex.setTokenPosition(token); tok = T_LOGICAL_XOR; fbreak;};
'\\' => {lex.createToken(lval); tok = T_NS_SEPARATOR; fbreak;}; '\\' => {lex.setTokenPosition(token); tok = T_NS_SEPARATOR; fbreak;};
'...' => {lex.createToken(lval); tok = T_ELLIPSIS; fbreak;}; '...' => {lex.setTokenPosition(token); tok = T_ELLIPSIS; fbreak;};
'::' => {lex.createToken(lval); tok = T_PAAMAYIM_NEKUDOTAYIM; fbreak;}; '::' => {lex.setTokenPosition(token); tok = T_PAAMAYIM_NEKUDOTAYIM; fbreak;};
'&&' => {lex.createToken(lval); tok = T_BOOLEAN_AND; fbreak;}; '&&' => {lex.setTokenPosition(token); tok = T_BOOLEAN_AND; fbreak;};
'||' => {lex.createToken(lval); tok = T_BOOLEAN_OR; fbreak;}; '||' => {lex.setTokenPosition(token); tok = T_BOOLEAN_OR; fbreak;};
'&=' => {lex.createToken(lval); tok = T_AND_EQUAL; fbreak;}; '&=' => {lex.setTokenPosition(token); tok = T_AND_EQUAL; fbreak;};
'|=' => {lex.createToken(lval); tok = T_OR_EQUAL; fbreak;}; '|=' => {lex.setTokenPosition(token); tok = T_OR_EQUAL; fbreak;};
'.=' => {lex.createToken(lval); tok = T_CONCAT_EQUAL; fbreak;}; '.=' => {lex.setTokenPosition(token); tok = T_CONCAT_EQUAL; fbreak;};
'*=' => {lex.createToken(lval); tok = T_MUL_EQUAL; fbreak;}; '*=' => {lex.setTokenPosition(token); tok = T_MUL_EQUAL; fbreak;};
'**=' => {lex.createToken(lval); tok = T_POW_EQUAL; fbreak;}; '**=' => {lex.setTokenPosition(token); tok = T_POW_EQUAL; fbreak;};
'/=' => {lex.createToken(lval); tok = T_DIV_EQUAL; fbreak;}; '/=' => {lex.setTokenPosition(token); tok = T_DIV_EQUAL; fbreak;};
'+=' => {lex.createToken(lval); tok = T_PLUS_EQUAL; fbreak;}; '+=' => {lex.setTokenPosition(token); tok = T_PLUS_EQUAL; fbreak;};
'-=' => {lex.createToken(lval); tok = T_MINUS_EQUAL; fbreak;}; '-=' => {lex.setTokenPosition(token); tok = T_MINUS_EQUAL; fbreak;};
'^=' => {lex.createToken(lval); tok = T_XOR_EQUAL; fbreak;}; '^=' => {lex.setTokenPosition(token); tok = T_XOR_EQUAL; fbreak;};
'%=' => {lex.createToken(lval); tok = T_MOD_EQUAL; fbreak;}; '%=' => {lex.setTokenPosition(token); tok = T_MOD_EQUAL; fbreak;};
'--' => {lex.createToken(lval); tok = T_DEC; fbreak;}; '--' => {lex.setTokenPosition(token); tok = T_DEC; fbreak;};
'++' => {lex.createToken(lval); tok = T_INC; fbreak;}; '++' => {lex.setTokenPosition(token); tok = T_INC; fbreak;};
'=>' => {lex.createToken(lval); tok = T_DOUBLE_ARROW; fbreak;}; '=>' => {lex.setTokenPosition(token); tok = T_DOUBLE_ARROW; fbreak;};
'<=>' => {lex.createToken(lval); tok = T_SPACESHIP; fbreak;}; '<=>' => {lex.setTokenPosition(token); tok = T_SPACESHIP; fbreak;};
'!=' | '<>' => {lex.createToken(lval); tok = T_IS_NOT_EQUAL; fbreak;}; '!=' | '<>' => {lex.setTokenPosition(token); tok = T_IS_NOT_EQUAL; fbreak;};
'!==' => {lex.createToken(lval); tok = T_IS_NOT_IDENTICAL; fbreak;}; '!==' => {lex.setTokenPosition(token); tok = T_IS_NOT_IDENTICAL; fbreak;};
'==' => {lex.createToken(lval); tok = T_IS_EQUAL; fbreak;}; '==' => {lex.setTokenPosition(token); tok = T_IS_EQUAL; fbreak;};
'===' => {lex.createToken(lval); tok = T_IS_IDENTICAL; fbreak;}; '===' => {lex.setTokenPosition(token); tok = T_IS_IDENTICAL; fbreak;};
'<<=' => {lex.createToken(lval); tok = T_SL_EQUAL; fbreak;}; '<<=' => {lex.setTokenPosition(token); tok = T_SL_EQUAL; fbreak;};
'>>=' => {lex.createToken(lval); tok = T_SR_EQUAL; fbreak;}; '>>=' => {lex.setTokenPosition(token); tok = T_SR_EQUAL; fbreak;};
'>=' => {lex.createToken(lval); tok = T_IS_GREATER_OR_EQUAL; fbreak;}; '>=' => {lex.setTokenPosition(token); tok = T_IS_GREATER_OR_EQUAL; fbreak;};
'<=' => {lex.createToken(lval); tok = T_IS_SMALLER_OR_EQUAL; fbreak;}; '<=' => {lex.setTokenPosition(token); tok = T_IS_SMALLER_OR_EQUAL; fbreak;};
'**' => {lex.createToken(lval); tok = T_POW; fbreak;}; '**' => {lex.setTokenPosition(token); tok = T_POW; fbreak;};
'<<' => {lex.createToken(lval); tok = T_SL; fbreak;}; '<<' => {lex.setTokenPosition(token); tok = T_SL; fbreak;};
'>>' => {lex.createToken(lval); tok = T_SR; fbreak;}; '>>' => {lex.setTokenPosition(token); tok = T_SR; fbreak;};
'??' => {lex.createToken(lval); tok = T_COALESCE; fbreak;}; '??' => {lex.setTokenPosition(token); tok = T_COALESCE; fbreak;};
'(' whitespace* 'array'i whitespace* ')' => {lex.createToken(lval); tok = T_ARRAY_CAST; fbreak;}; '(' whitespace* 'array'i whitespace* ')' => {lex.setTokenPosition(token); tok = T_ARRAY_CAST; fbreak;};
'(' whitespace* ('bool'i|'boolean'i) whitespace* ')' => {lex.createToken(lval); tok = T_BOOL_CAST; fbreak;}; '(' whitespace* ('bool'i|'boolean'i) whitespace* ')' => {lex.setTokenPosition(token); tok = T_BOOL_CAST; fbreak;};
'(' whitespace* ('real'i|'double'i|'float'i) whitespace* ')' => {lex.createToken(lval); tok = T_DOUBLE_CAST; fbreak;}; '(' whitespace* ('real'i|'double'i|'float'i) whitespace* ')' => {lex.setTokenPosition(token); tok = T_DOUBLE_CAST; fbreak;};
'(' whitespace* ('int'i|'integer'i) whitespace* ')' => {lex.createToken(lval); tok = T_INT_CAST; fbreak;}; '(' whitespace* ('int'i|'integer'i) whitespace* ')' => {lex.setTokenPosition(token); tok = T_INT_CAST; fbreak;};
'(' whitespace* 'object'i whitespace* ')' => {lex.createToken(lval); tok = T_OBJECT_CAST; fbreak;}; '(' whitespace* 'object'i whitespace* ')' => {lex.setTokenPosition(token); tok = T_OBJECT_CAST; fbreak;};
'(' whitespace* ('string'i|'binary'i) whitespace* ')' => {lex.createToken(lval); tok = T_STRING_CAST; fbreak;}; '(' whitespace* ('string'i|'binary'i) whitespace* ')' => {lex.setTokenPosition(token); tok = T_STRING_CAST; fbreak;};
'(' whitespace* 'unset'i whitespace* ')' => {lex.createToken(lval); tok = T_UNSET_CAST; fbreak;}; '(' whitespace* 'unset'i whitespace* ')' => {lex.setTokenPosition(token); tok = T_UNSET_CAST; fbreak;};
('#' | '//') any_line* when is_not_comment_end => { ('#' | '//') any_line* when is_not_comment_end => {
lex.ungetStr("?>") lex.ungetStr("?>")
@ -325,27 +329,27 @@ func (lex *Lexer) Lex(lval Lval) int {
operators => { operators => {
// rune, _ := utf8.DecodeRune(lex.data[lex.ts:lex.te]); // rune, _ := utf8.DecodeRune(lex.data[lex.ts:lex.te]);
// tok = TokenID(Rune2Class(rune)); // tok = TokenID(Rune2Class(rune));
lex.createToken(lval); lex.setTokenPosition(token);
tok = TokenID(int(lex.data[lex.ts])); tok = TokenID(int(lex.data[lex.ts]));
fbreak; fbreak;
}; };
"{" => { lex.createToken(lval); tok = TokenID(int('{')); lex.call(ftargs, fentry(php)); goto _out; }; "{" => { lex.setTokenPosition(token); tok = TokenID(int('{')); lex.call(ftargs, fentry(php)); goto _out; };
"}" => { lex.createToken(lval); tok = TokenID(int('}')); lex.ret(1); lex.PhpDocComment = ""; goto _out;}; "}" => { lex.setTokenPosition(token); tok = TokenID(int('}')); lex.ret(1); lex.PhpDocComment = ""; goto _out;};
"$" varname => { lex.createToken(lval); tok = T_VARIABLE; fbreak; }; "$" varname => { lex.setTokenPosition(token); tok = T_VARIABLE; fbreak; };
varname => { lex.createToken(lval); tok = T_STRING; fbreak; }; varname => { lex.setTokenPosition(token); tok = T_STRING; fbreak; };
"->" => { lex.createToken(lval); tok = T_OBJECT_OPERATOR; fnext property; fbreak; }; "->" => { lex.setTokenPosition(token); tok = T_OBJECT_OPERATOR; fnext property; fbreak; };
constant_string => { constant_string => {
lex.createToken(lval); lex.setTokenPosition(token);
tok = T_CONSTANT_ENCAPSED_STRING; tok = T_CONSTANT_ENCAPSED_STRING;
fbreak; fbreak;
}; };
"b"i? "<<<" [ \t]* ( heredoc_label | ("'" heredoc_label "'") | ('"' heredoc_label '"') ) newline => { "b"i? "<<<" [ \t]* ( heredoc_label | ("'" heredoc_label "'") | ('"' heredoc_label '"') ) newline => {
lex.heredocLabel = lex.data[lblStart:lblEnd] lex.heredocLabel = lex.data[lblStart:lblEnd]
lex.createToken(lval); lex.setTokenPosition(token);
tok = T_START_HEREDOC; tok = T_START_HEREDOC;
if lex.isHeredocEnd(lex.p+1) { if lex.isHeredocEnd(lex.p+1) {
@ -357,8 +361,8 @@ func (lex *Lexer) Lex(lval Lval) int {
} }
fbreak; fbreak;
}; };
"`" => {lex.createToken(lval); tok = TokenID(int('`')); fnext backqote; fbreak;}; "`" => {lex.setTokenPosition(token); tok = TokenID(int('`')); fnext backqote; fbreak;};
'"' => {lex.createToken(lval); tok = TokenID(int('"')); fnext template_string; fbreak;}; '"' => {lex.setTokenPosition(token); tok = TokenID(int('"')); fnext template_string; fbreak;};
any_line => { any_line => {
c := lex.data[lex.p] c := lex.data[lex.p]
@ -368,14 +372,14 @@ func (lex *Lexer) Lex(lval Lval) int {
property := |* property := |*
whitespace_line* => {lex.addFreeFloating(freefloating.WhiteSpaceType, lex.ts, lex.te)}; whitespace_line* => {lex.addFreeFloating(freefloating.WhiteSpaceType, lex.ts, lex.te)};
"->" => {lex.createToken(lval); tok = T_OBJECT_OPERATOR; fbreak;}; "->" => {lex.setTokenPosition(token); tok = T_OBJECT_OPERATOR; fbreak;};
varname => {lex.createToken(lval); tok = T_STRING; fnext php; fbreak;}; varname => {lex.setTokenPosition(token); tok = T_STRING; fnext php; fbreak;};
any => {lex.ungetCnt(1); fgoto php;}; any => {lex.ungetCnt(1); fgoto php;};
*|; *|;
nowdoc := |* nowdoc := |*
any_line* when is_not_heredoc_end => { any_line* when is_not_heredoc_end => {
lex.createToken(lval); lex.setTokenPosition(token);
tok = T_ENCAPSED_AND_WHITESPACE; tok = T_ENCAPSED_AND_WHITESPACE;
fnext heredoc_end; fnext heredoc_end;
fbreak; fbreak;
@ -383,11 +387,11 @@ func (lex *Lexer) Lex(lval Lval) int {
*|; *|;
heredoc := |* heredoc := |*
"{$" => {lex.ungetCnt(1); lex.createToken(lval); tok = T_CURLY_OPEN; lex.call(ftargs, fentry(php)); goto _out;}; "{$" => {lex.ungetCnt(1); lex.setTokenPosition(token); tok = T_CURLY_OPEN; lex.call(ftargs, fentry(php)); goto _out;};
"${" => {lex.createToken(lval); tok = T_DOLLAR_OPEN_CURLY_BRACES; lex.call(ftargs, fentry(string_var_name)); goto _out;}; "${" => {lex.setTokenPosition(token); tok = T_DOLLAR_OPEN_CURLY_BRACES; lex.call(ftargs, fentry(string_var_name)); goto _out;};
"$" => {lex.ungetCnt(1); fcall string_var;}; "$" => {lex.ungetCnt(1); fcall string_var;};
any_line* when is_not_heredoc_end_or_var => { any_line* when is_not_heredoc_end_or_var => {
lex.createToken(lval); lex.setTokenPosition(token);
tok = T_ENCAPSED_AND_WHITESPACE; tok = T_ENCAPSED_AND_WHITESPACE;
if lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' { if lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
@ -398,24 +402,24 @@ func (lex *Lexer) Lex(lval Lval) int {
*|; *|;
backqote := |* backqote := |*
"{$" => {lex.ungetCnt(1); lex.createToken(lval); tok = T_CURLY_OPEN; lex.call(ftargs, fentry(php)); goto _out;}; "{$" => {lex.ungetCnt(1); lex.setTokenPosition(token); tok = T_CURLY_OPEN; lex.call(ftargs, fentry(php)); goto _out;};
"${" => {lex.createToken(lval); tok = T_DOLLAR_OPEN_CURLY_BRACES; lex.call(ftargs, fentry(string_var_name)); goto _out;}; "${" => {lex.setTokenPosition(token); tok = T_DOLLAR_OPEN_CURLY_BRACES; lex.call(ftargs, fentry(string_var_name)); goto _out;};
"$" => {lex.ungetCnt(1); fcall string_var;}; "$" => {lex.ungetCnt(1); fcall string_var;};
'`' => {lex.createToken(lval); tok = TokenID(int('`')); fnext php; fbreak;}; '`' => {lex.setTokenPosition(token); tok = TokenID(int('`')); fnext php; fbreak;};
any_line* when is_not_backqoute_end_or_var => { any_line* when is_not_backqoute_end_or_var => {
lex.createToken(lval); lex.setTokenPosition(token);
tok = T_ENCAPSED_AND_WHITESPACE; tok = T_ENCAPSED_AND_WHITESPACE;
fbreak; fbreak;
}; };
*|; *|;
template_string := |* template_string := |*
"{$" => {lex.ungetCnt(1); lex.createToken(lval); tok = T_CURLY_OPEN; lex.call(ftargs, fentry(php)); goto _out;}; "{$" => {lex.ungetCnt(1); lex.setTokenPosition(token); tok = T_CURLY_OPEN; lex.call(ftargs, fentry(php)); goto _out;};
"${" => {lex.createToken(lval); tok = T_DOLLAR_OPEN_CURLY_BRACES; lex.call(ftargs, fentry(string_var_name)); goto _out;}; "${" => {lex.setTokenPosition(token); tok = T_DOLLAR_OPEN_CURLY_BRACES; lex.call(ftargs, fentry(string_var_name)); goto _out;};
"$" => {lex.ungetCnt(1); fcall string_var;}; "$" => {lex.ungetCnt(1); fcall string_var;};
'"' => {lex.createToken(lval); tok = TokenID(int('"')); fnext php; fbreak;}; '"' => {lex.setTokenPosition(token); tok = TokenID(int('"')); fnext php; fbreak;};
any_line* when is_not_string_end_or_var => { any_line* when is_not_string_end_or_var => {
lex.createToken(lval); lex.setTokenPosition(token);
tok = T_ENCAPSED_AND_WHITESPACE; tok = T_ENCAPSED_AND_WHITESPACE;
fbreak; fbreak;
}; };
@ -423,13 +427,13 @@ func (lex *Lexer) Lex(lval Lval) int {
heredoc_end := |* heredoc_end := |*
varname -- ";" => { varname -- ";" => {
lex.createToken(lval); lex.setTokenPosition(token);
tok = T_END_HEREDOC; tok = T_END_HEREDOC;
fnext php; fnext php;
fbreak; fbreak;
}; };
varname => { varname => {
lex.createToken(lval); lex.setTokenPosition(token);
tok = T_END_HEREDOC; tok = T_END_HEREDOC;
fnext php; fnext php;
fbreak; fbreak;
@ -437,20 +441,20 @@ func (lex *Lexer) Lex(lval Lval) int {
*|; *|;
string_var := |* string_var := |*
'$' varname => {lex.createToken(lval); tok = T_VARIABLE; fbreak;}; '$' varname => {lex.setTokenPosition(token); tok = T_VARIABLE; fbreak;};
'->' varname_first => {lex.ungetCnt(1); lex.createToken(lval); tok = T_OBJECT_OPERATOR; fbreak;}; '->' varname_first => {lex.ungetCnt(1); lex.setTokenPosition(token); tok = T_OBJECT_OPERATOR; fbreak;};
varname => {lex.createToken(lval); tok = T_STRING; fbreak;}; varname => {lex.setTokenPosition(token); tok = T_STRING; fbreak;};
'[' => {lex.createToken(lval); tok = TokenID(int('[')); lex.call(ftargs, fentry(string_var_index)); goto _out;}; '[' => {lex.setTokenPosition(token); tok = TokenID(int('[')); lex.call(ftargs, fentry(string_var_index)); goto _out;};
any => {lex.ungetCnt(1); fret;}; any => {lex.ungetCnt(1); fret;};
*|; *|;
string_var_index := |* string_var_index := |*
lnum | hnum | bnum => {lex.createToken(lval); tok = T_NUM_STRING; fbreak;}; lnum | hnum | bnum => {lex.setTokenPosition(token); tok = T_NUM_STRING; fbreak;};
'$' varname => {lex.createToken(lval); tok = T_VARIABLE; fbreak;}; '$' varname => {lex.setTokenPosition(token); tok = T_VARIABLE; fbreak;};
varname => {lex.createToken(lval); tok = T_STRING; fbreak;}; varname => {lex.setTokenPosition(token); tok = T_STRING; fbreak;};
whitespace_line | [\\'#] => {lex.createToken(lval); tok = T_ENCAPSED_AND_WHITESPACE; lex.ret(2); goto _out;}; whitespace_line | [\\'#] => {lex.setTokenPosition(token); tok = T_ENCAPSED_AND_WHITESPACE; lex.ret(2); goto _out;};
operators > (svi, 1) => {lex.createToken(lval); tok = TokenID(int(lex.data[lex.ts])); fbreak;}; operators > (svi, 1) => {lex.setTokenPosition(token); tok = TokenID(int(lex.data[lex.ts])); fbreak;};
']' > (svi, 2) => {lex.createToken(lval); tok = TokenID(int(']')); lex.ret(2); goto _out;}; ']' > (svi, 2) => {lex.setTokenPosition(token); tok = TokenID(int(']')); lex.ret(2); goto _out;};
any_line => { any_line => {
c := lex.data[lex.p] c := lex.data[lex.p]
lex.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c)); lex.Error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c));
@ -458,25 +462,25 @@ func (lex *Lexer) Lex(lval Lval) int {
*|; *|;
string_var_name := |* string_var_name := |*
varname ("[" | "}") => {lex.ungetCnt(1); lex.createToken(lval); tok = T_STRING_VARNAME; fnext php; fbreak;}; varname ("[" | "}") => {lex.ungetCnt(1); lex.setTokenPosition(token); tok = T_STRING_VARNAME; fnext php; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_open_parenthesis := |* halt_compiller_open_parenthesis := |*
whitespace_line* => {lex.addFreeFloating(freefloating.WhiteSpaceType, lex.ts, lex.te)}; whitespace_line* => {lex.addFreeFloating(freefloating.WhiteSpaceType, lex.ts, lex.te)};
"(" => {lex.createToken(lval); tok = TokenID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;}; "(" => {lex.setTokenPosition(token); tok = TokenID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_close_parenthesis := |* halt_compiller_close_parenthesis := |*
whitespace_line* => {lex.addFreeFloating(freefloating.WhiteSpaceType, lex.ts, lex.te)}; whitespace_line* => {lex.addFreeFloating(freefloating.WhiteSpaceType, lex.ts, lex.te)};
")" => {lex.createToken(lval); tok = TokenID(int(')')); fnext halt_compiller_close_semicolon; fbreak;}; ")" => {lex.setTokenPosition(token); tok = TokenID(int(')')); fnext halt_compiller_close_semicolon; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
halt_compiller_close_semicolon := |* halt_compiller_close_semicolon := |*
whitespace_line* => {lex.addFreeFloating(freefloating.WhiteSpaceType, lex.ts, lex.te)}; whitespace_line* => {lex.addFreeFloating(freefloating.WhiteSpaceType, lex.ts, lex.te)};
";" => {lex.createToken(lval); tok = TokenID(int(';')); fnext halt_compiller_end; fbreak;}; ";" => {lex.setTokenPosition(token); tok = TokenID(int(';')); fnext halt_compiller_end; fbreak;};
any => {lex.ungetCnt(1); fnext php;}; any => {lex.ungetCnt(1); fnext php;};
*|; *|;
@ -487,14 +491,10 @@ func (lex *Lexer) Lex(lval Lval) int {
write exec; write exec;
}%% }%%
// always return same $end token token.FreeFloating = lex.FreeFloating
if tok == 0 { token.Value = string(lex.data[lex.ts:lex.te])
if lex.lastToken == nil {
lex.ts, lex.te = 0, 0 lval.Token(token)
lex.lastToken = lex.createToken(lval)
}
lval.Token(lex.lastToken);
}
return int(tok); return int(tok);
} }