use Token.File to handle new lines

This commit is contained in:
z7zmey 2018-01-08 17:02:56 +02:00
parent 45c085a3f4
commit f2cf2061ee
3 changed files with 345 additions and 362 deletions

View File

@ -21,7 +21,6 @@ const (
type lexer struct {
*lex.Lexer
stateStack []int
lineNumber int
phpDocComment string
comments *[]comment.Comment
}
@ -46,7 +45,7 @@ func newLexer(src io.Reader, fName string) *lexer {
if err != nil {
panic(err)
}
return &lexer{lx, []int{0}, 1, "", nil}
return &lexer{lx, []int{0}, "", nil}
}
func (l *lexer) ungetN(n int) []byte {
@ -88,31 +87,17 @@ func (l *lexer) getCurrentState() int {
return l.stateStack[len(l.stateStack)-1]
}
func (l *lexer) handleNewLine(tokenBytes []byte) ([]byte, int, int, int, int) {
startln := l.lineNumber
func (l *lexer) newToken(tokenBytes []byte) t.Token {
tokenBytesEnd := len(tokenBytes) - 1
var prev byte
for _, b := range tokenBytes {
if b == '\n' || prev == '\r' {
l.lineNumber++
}
prev = b
}
// handle last \r
if prev == '\r' {
l.lineNumber++
}
startLine := l.File.Line(l.First.Pos())
lastChar := l.Token()[tokenBytesEnd]
endLine := l.File.Line(lastChar.Pos())
startPos := int(l.First.Pos())
endPos := startPos + len(tokenBytes) - 1
return tokenBytes, startln, l.lineNumber, startPos, endPos
}
endPos := startPos + tokenBytesEnd
func (l *lexer) newToken() t.Token {
return t.NewToken(l.handleNewLine(l.TokenBytes(nil))).SetComments(l.comments)
return t.NewToken(tokenBytes, startLine, endLine, startPos, endPos).SetComments(l.comments)
}
func (l *lexer) addComment(c comment.Comment) {

File diff suppressed because it is too large Load Diff

View File

@ -10,8 +10,7 @@ package parser
import (
"fmt"
"bytes"
"github.com/z7zmey/php-parser/token"
"github.com/z7zmey/php-parser/comment"
"github.com/z7zmey/php-parser/comment"
)
const (
@ -57,7 +56,7 @@ NEW_LINE (\r|\n|\r\n)
%%
c = l.Rule0()
<INITIAL>[ \t\n\r]+ lval.token = l.newToken();
<INITIAL>[ \t\n\r]+ lval.token = l.newToken(l.TokenBytes(nil));
<INITIAL>.
tb := []byte{}
@ -81,17 +80,17 @@ NEW_LINE (\r|\n|\r\n)
c = l.Next()
}
lval.token = token.NewToken(l.handleNewLine(tb));
lval.token = l.newToken(tb)
return T_INLINE_HTML
<INITIAL>\<\?php([ \t]|{NEW_LINE}) l.begin(PHP);lval.token = l.newToken();// return T_OPEN_TAG;
<INITIAL>\<\? l.begin(PHP);lval.token = l.newToken();// return T_OPEN_TAG;
<INITIAL>\<\?= l.begin(PHP);lval.token = l.newToken(); return T_ECHO;
<INITIAL>\<\?php([ \t]|{NEW_LINE}) l.begin(PHP);lval.token = l.newToken(l.TokenBytes(nil));// return T_OPEN_TAG;
<INITIAL>\<\? l.begin(PHP);lval.token = l.newToken(l.TokenBytes(nil));// return T_OPEN_TAG;
<INITIAL>\<\?= l.begin(PHP);lval.token = l.newToken(l.TokenBytes(nil)); return T_ECHO;
<PHP>[ \t\n\r]+ lval.token = l.newToken();// return T_WHITESPACE
<PHP>\?\>{NEW_LINE}? l.begin(INITIAL);lval.token = l.newToken(); return rune2Class(';');
<PHP>[ \t\n\r]+ lval.token = l.newToken(l.TokenBytes(nil));// return T_WHITESPACE
<PHP>\?\>{NEW_LINE}? l.begin(INITIAL);lval.token = l.newToken(l.TokenBytes(nil)); return rune2Class(';');
<PHP>{DNUM}|{EXPONENT_DNUM} lval.token = l.newToken(); return T_DNUMBER
<PHP>{DNUM}|{EXPONENT_DNUM} lval.token = l.newToken(l.TokenBytes(nil)); return T_DNUMBER
<PHP>{BNUM}
tb := l.TokenBytes(nil)
i:=2
@ -105,15 +104,15 @@ NEW_LINE (\r|\n|\r\n)
}
}
if len(tb) - i < 64 {
lval.token = l.newToken(); return T_LNUMBER
lval.token = l.newToken(l.TokenBytes(nil)); return T_LNUMBER
} else {
lval.token = l.newToken(); return T_DNUMBER
lval.token = l.newToken(l.TokenBytes(nil)); return T_DNUMBER
}
<PHP>{LNUM}
if len(l.TokenBytes(nil)) < 20 {
lval.token = l.newToken(); return T_LNUMBER
lval.token = l.newToken(l.TokenBytes(nil)); return T_LNUMBER
} else {
lval.token = l.newToken(); return T_DNUMBER
lval.token = l.newToken(l.TokenBytes(nil)); return T_DNUMBER
}
<PHP>{HNUM}
tb := l.TokenBytes(nil)
@ -129,125 +128,125 @@ NEW_LINE (\r|\n|\r\n)
}
length := len(tb) - i
if length < 16 || (length == 16 && tb[i] <= '7') {
lval.token = l.newToken(); return T_LNUMBER
lval.token = l.newToken(l.TokenBytes(nil)); return T_LNUMBER
} else {
lval.token = l.newToken(); return T_DNUMBER
lval.token = l.newToken(l.TokenBytes(nil)); return T_DNUMBER
}
<PHP>abstract lval.token = l.newToken(); return T_ABSTRACT
<PHP>array lval.token = l.newToken(); return T_ARRAY
<PHP>as lval.token = l.newToken(); return T_AS
<PHP>break lval.token = l.newToken(); return T_BREAK
<PHP>callable lval.token = l.newToken(); return T_CALLABLE
<PHP>case lval.token = l.newToken(); return T_CASE
<PHP>catch lval.token = l.newToken(); return T_CATCH
<PHP>class lval.token = l.newToken(); return T_CLASS
<PHP>clone lval.token = l.newToken(); return T_CLONE
<PHP>const lval.token = l.newToken(); return T_CONST;
<PHP>continue lval.token = l.newToken(); return T_CONTINUE;
<PHP>declare lval.token = l.newToken(); return T_DECLARE;
<PHP>default lval.token = l.newToken(); return T_DEFAULT;
<PHP>do lval.token = l.newToken(); return T_DO;
<PHP>echo lval.token = l.newToken(); return T_ECHO;
<PHP>else lval.token = l.newToken(); return T_ELSE;
<PHP>elseif lval.token = l.newToken(); return T_ELSEIF;
<PHP>empty lval.token = l.newToken(); return T_EMPTY;
<PHP>enddeclare lval.token = l.newToken(); return T_ENDDECLARE
<PHP>endfor lval.token = l.newToken(); return T_ENDFOR
<PHP>endforeach lval.token = l.newToken(); return T_ENDFOREACH
<PHP>endif lval.token = l.newToken(); return T_ENDIF
<PHP>endswitch lval.token = l.newToken(); return T_ENDSWITCH
<PHP>endwhile lval.token = l.newToken(); return T_ENDWHILE
<PHP>eval lval.token = l.newToken(); return T_EVAL
<PHP>exit|die lval.token = l.newToken(); return T_EXIT
<PHP>extends lval.token = l.newToken(); return T_EXTENDS
<PHP>final lval.token = l.newToken(); return T_FINAL
<PHP>finally lval.token = l.newToken(); return T_FINALLY
<PHP>for lval.token = l.newToken(); return T_FOR
<PHP>foreach lval.token = l.newToken(); return T_FOREACH
<PHP>function|cfunction lval.token = l.newToken(); return T_FUNCTION
<PHP>global lval.token = l.newToken(); return T_GLOBAL
<PHP>goto lval.token = l.newToken(); return T_GOTO
<PHP>if lval.token = l.newToken(); return T_IF
<PHP>isset lval.token = l.newToken(); return T_ISSET
<PHP>implements lval.token = l.newToken(); return T_IMPLEMENTS
<PHP>instanceof lval.token = l.newToken(); return T_INSTANCEOF
<PHP>insteadof lval.token = l.newToken(); return T_INSTEADOF
<PHP>interface lval.token = l.newToken(); return T_INTERFACE
<PHP>list lval.token = l.newToken(); return T_LIST
<PHP>namespace lval.token = l.newToken(); return T_NAMESPACE
<PHP>private lval.token = l.newToken(); return T_PRIVATE
<PHP>public lval.token = l.newToken(); return T_PUBLIC
<PHP>print lval.token = l.newToken(); return T_PRINT
<PHP>protected lval.token = l.newToken(); return T_PROTECTED
<PHP>return lval.token = l.newToken(); return T_RETURN
<PHP>static lval.token = l.newToken(); return T_STATIC
<PHP>switch lval.token = l.newToken(); return T_SWITCH
<PHP>throw lval.token = l.newToken(); return T_THROW
<PHP>trait lval.token = l.newToken(); return T_TRAIT
<PHP>try lval.token = l.newToken(); return T_TRY
<PHP>unset lval.token = l.newToken(); return T_UNSET
<PHP>use lval.token = l.newToken(); return T_USE
<PHP>var lval.token = l.newToken(); return T_VAR
<PHP>while lval.token = l.newToken(); return T_WHILE
<PHP>yield[ \t\n\r]+from[^a-zA-Z0-9_\x80-\xff] lval.token = l.newToken(); return T_YIELD_FROM
<PHP>yield lval.token = l.newToken(); return T_YIELD
<PHP>include lval.token = l.newToken(); return T_INCLUDE
<PHP>include_once lval.token = l.newToken(); return T_INCLUDE_ONCE
<PHP>require lval.token = l.newToken(); return T_REQUIRE
<PHP>require_once lval.token = l.newToken(); return T_REQUIRE_ONCE
<PHP>__CLASS__ lval.token = l.newToken(); return T_CLASS_C
<PHP>__DIR__ lval.token = l.newToken(); return T_DIR
<PHP>__FILE__ lval.token = l.newToken(); return T_FILE
<PHP>__FUNCTION__ lval.token = l.newToken(); return T_FUNC_C
<PHP>__LINE__ lval.token = l.newToken(); return T_LINE
<PHP>__NAMESPACE__ lval.token = l.newToken(); return T_NS_C
<PHP>__METHOD__ lval.token = l.newToken(); return T_METHOD_C
<PHP>__TRAIT__ lval.token = l.newToken(); return T_TRAIT_C
<PHP>__halt_compiler lval.token = l.newToken(); return T_HALT_COMPILER
<PHP>\([ \t]*array[ \t]*\) lval.token = l.newToken(); return T_ARRAY_CAST
<PHP>\([ \t]*(bool|boolean)[ \t]*\) lval.token = l.newToken(); return T_BOOL_CAST
<PHP>\([ \t]*(real|double|float)[ \t]*\) lval.token = l.newToken(); return T_DOUBLE_CAST
<PHP>\([ \t]*(int|integer)[ \t]*\) lval.token = l.newToken(); return T_INT_CAST
<PHP>\([ \t]*object[ \t]*\) lval.token = l.newToken(); return T_OBJECT_CAST
<PHP>\([ \t]*string[ \t]*\) lval.token = l.newToken(); return T_STRING_CAST
<PHP>\([ \t]*unset[ \t]*\) lval.token = l.newToken(); return T_UNSET_CAST
<PHP>new lval.token = l.newToken(); return T_NEW
<PHP>and lval.token = l.newToken(); return T_LOGICAL_AND
<PHP>or lval.token = l.newToken(); return T_LOGICAL_OR
<PHP>xor lval.token = l.newToken(); return T_LOGICAL_XOR
<PHP>\\ lval.token = l.newToken(); return T_NS_SEPARATOR
<PHP>\.\.\. lval.token = l.newToken(); return T_ELLIPSIS;
<PHP>:: lval.token = l.newToken(); return T_PAAMAYIM_NEKUDOTAYIM; // T_DOUBLE_COLON
<PHP>&& lval.token = l.newToken(); return T_BOOLEAN_AND
<PHP>\|\| lval.token = l.newToken(); return T_BOOLEAN_OR
<PHP>&= lval.token = l.newToken(); return T_AND_EQUAL
<PHP>\|= lval.token = l.newToken(); return T_OR_EQUAL
<PHP>\.= lval.token = l.newToken(); return T_CONCAT_EQUAL;
<PHP>\*= lval.token = l.newToken(); return T_MUL_EQUAL
<PHP>\*\*= lval.token = l.newToken(); return T_POW_EQUAL
<PHP>[/]= lval.token = l.newToken(); return T_DIV_EQUAL;
<PHP>\+= lval.token = l.newToken(); return T_PLUS_EQUAL
<PHP>-= lval.token = l.newToken(); return T_MINUS_EQUAL
<PHP>\^= lval.token = l.newToken(); return T_XOR_EQUAL
<PHP>%= lval.token = l.newToken(); return T_MOD_EQUAL
<PHP>-- lval.token = l.newToken(); return T_DEC;
<PHP>\+\+ lval.token = l.newToken(); return T_INC
<PHP>=> lval.token = l.newToken(); return T_DOUBLE_ARROW;
<PHP>\<=\> lval.token = l.newToken(); return T_SPACESHIP
<PHP>\!=|\<\> lval.token = l.newToken(); return T_IS_NOT_EQUAL
<PHP>\!== lval.token = l.newToken(); return T_IS_NOT_IDENTICAL
<PHP>== lval.token = l.newToken(); return T_IS_EQUAL
<PHP>=== lval.token = l.newToken(); return T_IS_IDENTICAL
<PHP>\<\<= lval.token = l.newToken(); return T_SL_EQUAL
<PHP>\>\>= lval.token = l.newToken(); return T_SR_EQUAL
<PHP>\>= lval.token = l.newToken(); return T_IS_GREATER_OR_EQUAL
<PHP>\<= lval.token = l.newToken(); return T_IS_SMALLER_OR_EQUAL
<PHP>\*\* lval.token = l.newToken(); return T_POW
<PHP>\<\< lval.token = l.newToken(); return T_SL
<PHP>\>\> lval.token = l.newToken(); return T_SR
<PHP>\?\? lval.token = l.newToken(); return T_COALESCE
<PHP>(#|[/][/]).*{NEW_LINE} lval.token = l.newToken();// return T_COMMENT; // TODO: handle ?>
<PHP>abstract lval.token = l.newToken(l.TokenBytes(nil)); return T_ABSTRACT
<PHP>array lval.token = l.newToken(l.TokenBytes(nil)); return T_ARRAY
<PHP>as lval.token = l.newToken(l.TokenBytes(nil)); return T_AS
<PHP>break lval.token = l.newToken(l.TokenBytes(nil)); return T_BREAK
<PHP>callable lval.token = l.newToken(l.TokenBytes(nil)); return T_CALLABLE
<PHP>case lval.token = l.newToken(l.TokenBytes(nil)); return T_CASE
<PHP>catch lval.token = l.newToken(l.TokenBytes(nil)); return T_CATCH
<PHP>class lval.token = l.newToken(l.TokenBytes(nil)); return T_CLASS
<PHP>clone lval.token = l.newToken(l.TokenBytes(nil)); return T_CLONE
<PHP>const lval.token = l.newToken(l.TokenBytes(nil)); return T_CONST;
<PHP>continue lval.token = l.newToken(l.TokenBytes(nil)); return T_CONTINUE;
<PHP>declare lval.token = l.newToken(l.TokenBytes(nil)); return T_DECLARE;
<PHP>default lval.token = l.newToken(l.TokenBytes(nil)); return T_DEFAULT;
<PHP>do lval.token = l.newToken(l.TokenBytes(nil)); return T_DO;
<PHP>echo lval.token = l.newToken(l.TokenBytes(nil)); return T_ECHO;
<PHP>else lval.token = l.newToken(l.TokenBytes(nil)); return T_ELSE;
<PHP>elseif lval.token = l.newToken(l.TokenBytes(nil)); return T_ELSEIF;
<PHP>empty lval.token = l.newToken(l.TokenBytes(nil)); return T_EMPTY;
<PHP>enddeclare lval.token = l.newToken(l.TokenBytes(nil)); return T_ENDDECLARE
<PHP>endfor lval.token = l.newToken(l.TokenBytes(nil)); return T_ENDFOR
<PHP>endforeach lval.token = l.newToken(l.TokenBytes(nil)); return T_ENDFOREACH
<PHP>endif lval.token = l.newToken(l.TokenBytes(nil)); return T_ENDIF
<PHP>endswitch lval.token = l.newToken(l.TokenBytes(nil)); return T_ENDSWITCH
<PHP>endwhile lval.token = l.newToken(l.TokenBytes(nil)); return T_ENDWHILE
<PHP>eval lval.token = l.newToken(l.TokenBytes(nil)); return T_EVAL
<PHP>exit|die lval.token = l.newToken(l.TokenBytes(nil)); return T_EXIT
<PHP>extends lval.token = l.newToken(l.TokenBytes(nil)); return T_EXTENDS
<PHP>final lval.token = l.newToken(l.TokenBytes(nil)); return T_FINAL
<PHP>finally lval.token = l.newToken(l.TokenBytes(nil)); return T_FINALLY
<PHP>for lval.token = l.newToken(l.TokenBytes(nil)); return T_FOR
<PHP>foreach lval.token = l.newToken(l.TokenBytes(nil)); return T_FOREACH
<PHP>function|cfunction lval.token = l.newToken(l.TokenBytes(nil)); return T_FUNCTION
<PHP>global lval.token = l.newToken(l.TokenBytes(nil)); return T_GLOBAL
<PHP>goto lval.token = l.newToken(l.TokenBytes(nil)); return T_GOTO
<PHP>if lval.token = l.newToken(l.TokenBytes(nil)); return T_IF
<PHP>isset lval.token = l.newToken(l.TokenBytes(nil)); return T_ISSET
<PHP>implements lval.token = l.newToken(l.TokenBytes(nil)); return T_IMPLEMENTS
<PHP>instanceof lval.token = l.newToken(l.TokenBytes(nil)); return T_INSTANCEOF
<PHP>insteadof lval.token = l.newToken(l.TokenBytes(nil)); return T_INSTEADOF
<PHP>interface lval.token = l.newToken(l.TokenBytes(nil)); return T_INTERFACE
<PHP>list lval.token = l.newToken(l.TokenBytes(nil)); return T_LIST
<PHP>namespace lval.token = l.newToken(l.TokenBytes(nil)); return T_NAMESPACE
<PHP>private lval.token = l.newToken(l.TokenBytes(nil)); return T_PRIVATE
<PHP>public lval.token = l.newToken(l.TokenBytes(nil)); return T_PUBLIC
<PHP>print lval.token = l.newToken(l.TokenBytes(nil)); return T_PRINT
<PHP>protected lval.token = l.newToken(l.TokenBytes(nil)); return T_PROTECTED
<PHP>return lval.token = l.newToken(l.TokenBytes(nil)); return T_RETURN
<PHP>static lval.token = l.newToken(l.TokenBytes(nil)); return T_STATIC
<PHP>switch lval.token = l.newToken(l.TokenBytes(nil)); return T_SWITCH
<PHP>throw lval.token = l.newToken(l.TokenBytes(nil)); return T_THROW
<PHP>trait lval.token = l.newToken(l.TokenBytes(nil)); return T_TRAIT
<PHP>try lval.token = l.newToken(l.TokenBytes(nil)); return T_TRY
<PHP>unset lval.token = l.newToken(l.TokenBytes(nil)); return T_UNSET
<PHP>use lval.token = l.newToken(l.TokenBytes(nil)); return T_USE
<PHP>var lval.token = l.newToken(l.TokenBytes(nil)); return T_VAR
<PHP>while lval.token = l.newToken(l.TokenBytes(nil)); return T_WHILE
<PHP>yield[ \t\n\r]+from[^a-zA-Z0-9_\x80-\xff] lval.token = l.newToken(l.TokenBytes(nil)); return T_YIELD_FROM
<PHP>yield lval.token = l.newToken(l.TokenBytes(nil)); return T_YIELD
<PHP>include lval.token = l.newToken(l.TokenBytes(nil)); return T_INCLUDE
<PHP>include_once lval.token = l.newToken(l.TokenBytes(nil)); return T_INCLUDE_ONCE
<PHP>require lval.token = l.newToken(l.TokenBytes(nil)); return T_REQUIRE
<PHP>require_once lval.token = l.newToken(l.TokenBytes(nil)); return T_REQUIRE_ONCE
<PHP>__CLASS__ lval.token = l.newToken(l.TokenBytes(nil)); return T_CLASS_C
<PHP>__DIR__ lval.token = l.newToken(l.TokenBytes(nil)); return T_DIR
<PHP>__FILE__ lval.token = l.newToken(l.TokenBytes(nil)); return T_FILE
<PHP>__FUNCTION__ lval.token = l.newToken(l.TokenBytes(nil)); return T_FUNC_C
<PHP>__LINE__ lval.token = l.newToken(l.TokenBytes(nil)); return T_LINE
<PHP>__NAMESPACE__ lval.token = l.newToken(l.TokenBytes(nil)); return T_NS_C
<PHP>__METHOD__ lval.token = l.newToken(l.TokenBytes(nil)); return T_METHOD_C
<PHP>__TRAIT__ lval.token = l.newToken(l.TokenBytes(nil)); return T_TRAIT_C
<PHP>__halt_compiler lval.token = l.newToken(l.TokenBytes(nil)); return T_HALT_COMPILER
<PHP>\([ \t]*array[ \t]*\) lval.token = l.newToken(l.TokenBytes(nil)); return T_ARRAY_CAST
<PHP>\([ \t]*(bool|boolean)[ \t]*\) lval.token = l.newToken(l.TokenBytes(nil)); return T_BOOL_CAST
<PHP>\([ \t]*(real|double|float)[ \t]*\) lval.token = l.newToken(l.TokenBytes(nil)); return T_DOUBLE_CAST
<PHP>\([ \t]*(int|integer)[ \t]*\) lval.token = l.newToken(l.TokenBytes(nil)); return T_INT_CAST
<PHP>\([ \t]*object[ \t]*\) lval.token = l.newToken(l.TokenBytes(nil)); return T_OBJECT_CAST
<PHP>\([ \t]*string[ \t]*\) lval.token = l.newToken(l.TokenBytes(nil)); return T_STRING_CAST
<PHP>\([ \t]*unset[ \t]*\) lval.token = l.newToken(l.TokenBytes(nil)); return T_UNSET_CAST
<PHP>new lval.token = l.newToken(l.TokenBytes(nil)); return T_NEW
<PHP>and lval.token = l.newToken(l.TokenBytes(nil)); return T_LOGICAL_AND
<PHP>or lval.token = l.newToken(l.TokenBytes(nil)); return T_LOGICAL_OR
<PHP>xor lval.token = l.newToken(l.TokenBytes(nil)); return T_LOGICAL_XOR
<PHP>\\ lval.token = l.newToken(l.TokenBytes(nil)); return T_NS_SEPARATOR
<PHP>\.\.\. lval.token = l.newToken(l.TokenBytes(nil)); return T_ELLIPSIS;
<PHP>:: lval.token = l.newToken(l.TokenBytes(nil)); return T_PAAMAYIM_NEKUDOTAYIM; // T_DOUBLE_COLON
<PHP>&& lval.token = l.newToken(l.TokenBytes(nil)); return T_BOOLEAN_AND
<PHP>\|\| lval.token = l.newToken(l.TokenBytes(nil)); return T_BOOLEAN_OR
<PHP>&= lval.token = l.newToken(l.TokenBytes(nil)); return T_AND_EQUAL
<PHP>\|= lval.token = l.newToken(l.TokenBytes(nil)); return T_OR_EQUAL
<PHP>\.= lval.token = l.newToken(l.TokenBytes(nil)); return T_CONCAT_EQUAL;
<PHP>\*= lval.token = l.newToken(l.TokenBytes(nil)); return T_MUL_EQUAL
<PHP>\*\*= lval.token = l.newToken(l.TokenBytes(nil)); return T_POW_EQUAL
<PHP>[/]= lval.token = l.newToken(l.TokenBytes(nil)); return T_DIV_EQUAL;
<PHP>\+= lval.token = l.newToken(l.TokenBytes(nil)); return T_PLUS_EQUAL
<PHP>-= lval.token = l.newToken(l.TokenBytes(nil)); return T_MINUS_EQUAL
<PHP>\^= lval.token = l.newToken(l.TokenBytes(nil)); return T_XOR_EQUAL
<PHP>%= lval.token = l.newToken(l.TokenBytes(nil)); return T_MOD_EQUAL
<PHP>-- lval.token = l.newToken(l.TokenBytes(nil)); return T_DEC;
<PHP>\+\+ lval.token = l.newToken(l.TokenBytes(nil)); return T_INC
<PHP>=> lval.token = l.newToken(l.TokenBytes(nil)); return T_DOUBLE_ARROW;
<PHP>\<=\> lval.token = l.newToken(l.TokenBytes(nil)); return T_SPACESHIP
<PHP>\!=|\<\> lval.token = l.newToken(l.TokenBytes(nil)); return T_IS_NOT_EQUAL
<PHP>\!== lval.token = l.newToken(l.TokenBytes(nil)); return T_IS_NOT_IDENTICAL
<PHP>== lval.token = l.newToken(l.TokenBytes(nil)); return T_IS_EQUAL
<PHP>=== lval.token = l.newToken(l.TokenBytes(nil)); return T_IS_IDENTICAL
<PHP>\<\<= lval.token = l.newToken(l.TokenBytes(nil)); return T_SL_EQUAL
<PHP>\>\>= lval.token = l.newToken(l.TokenBytes(nil)); return T_SR_EQUAL
<PHP>\>= lval.token = l.newToken(l.TokenBytes(nil)); return T_IS_GREATER_OR_EQUAL
<PHP>\<= lval.token = l.newToken(l.TokenBytes(nil)); return T_IS_SMALLER_OR_EQUAL
<PHP>\*\* lval.token = l.newToken(l.TokenBytes(nil)); return T_POW
<PHP>\<\< lval.token = l.newToken(l.TokenBytes(nil)); return T_SL
<PHP>\>\> lval.token = l.newToken(l.TokenBytes(nil)); return T_SR
<PHP>\?\? lval.token = l.newToken(l.TokenBytes(nil)); return T_COALESCE
<PHP>(#|[/][/]).*{NEW_LINE} lval.token = l.newToken(l.TokenBytes(nil));// return T_COMMENT; // TODO: handle ?>
<PHP>([/][*])|([/][*][*])
tb := l.TokenBytes(nil)
is_doc_comment := false
@ -270,7 +269,7 @@ NEW_LINE (\r|\n|\r\n)
}
}
lval.token = token.NewToken(l.handleNewLine(l.TokenBytes(nil)))
lval.token = l.newToken(l.TokenBytes(nil))
if is_doc_comment {
l.phpDocComment = string(l.TokenBytes(nil))
l.addComment(comment.NewDocComment(string(l.TokenBytes(nil))))
@ -280,23 +279,23 @@ NEW_LINE (\r|\n|\r\n)
// return T_COMMENT
}
<PHP>{OPERATORS} lval.token = l.newToken(); return rune2Class(rune(l.TokenBytes(nil)[0]))
<PHP>{OPERATORS} lval.token = l.newToken(l.TokenBytes(nil)); return rune2Class(rune(l.TokenBytes(nil)[0]))
<PHP>\{ l.pushState(PHP); lval.token = l.newToken(); return rune2Class(rune(l.TokenBytes(nil)[0]))
<PHP>\} l.popState(); lval.token = l.newToken(); return rune2Class(rune(l.TokenBytes(nil)[0])); l.phpDocComment = ""
<PHP>\${VAR_NAME} lval.token = l.newToken(); return T_VARIABLE
<PHP>{VAR_NAME} lval.token = l.newToken(); return T_STRING
<PHP>\{ l.pushState(PHP); lval.token = l.newToken(l.TokenBytes(nil)); return rune2Class(rune(l.TokenBytes(nil)[0]))
<PHP>\} l.popState(); lval.token = l.newToken(l.TokenBytes(nil)); return rune2Class(rune(l.TokenBytes(nil)[0])); l.phpDocComment = ""
<PHP>\${VAR_NAME} lval.token = l.newToken(l.TokenBytes(nil)); return T_VARIABLE
<PHP>{VAR_NAME} lval.token = l.newToken(l.TokenBytes(nil)); return T_STRING
<PHP>-> l.begin(PROPERTY);lval.token = l.newToken(); return T_OBJECT_OPERATOR;
<PROPERTY>[ \t\n\r]+ lval.token = l.newToken(); return T_WHITESPACE;
<PROPERTY>-> lval.token = l.newToken(); return T_OBJECT_OPERATOR;
<PROPERTY>{VAR_NAME} l.begin(PHP);lval.token = l.newToken(); return T_STRING;
<PHP>-> l.begin(PROPERTY);lval.token = l.newToken(l.TokenBytes(nil)); return T_OBJECT_OPERATOR;
<PROPERTY>[ \t\n\r]+ lval.token = l.newToken(l.TokenBytes(nil)); return T_WHITESPACE;
<PROPERTY>-> lval.token = l.newToken(l.TokenBytes(nil)); return T_OBJECT_OPERATOR;
<PROPERTY>{VAR_NAME} l.begin(PHP);lval.token = l.newToken(l.TokenBytes(nil)); return T_STRING;
<PROPERTY>. l.ungetN(1);l.begin(PHP)
<PHP>[\']([^\\\']*([\\].)*)*[\'] lval.token = l.newToken(); return T_CONSTANT_ENCAPSED_STRING;
<PHP>[\']([^\\\']*([\\].)*)*[\'] lval.token = l.newToken(l.TokenBytes(nil)); return T_CONSTANT_ENCAPSED_STRING;
<PHP>` l.begin(BACKQUOTE); lval.token = l.newToken(); return rune2Class(rune(l.TokenBytes(nil)[0]))
<BACKQUOTE>` l.begin(PHP); lval.token = l.newToken(); return rune2Class(rune(l.TokenBytes(nil)[0]))
<PHP>` l.begin(BACKQUOTE); lval.token = l.newToken(l.TokenBytes(nil)); return rune2Class(rune(l.TokenBytes(nil)[0]))
<BACKQUOTE>` l.begin(PHP); lval.token = l.newToken(l.TokenBytes(nil)); return rune2Class(rune(l.TokenBytes(nil)[0]))
<PHP>[b]?\<\<\<[ \t]*({VAR_NAME}|([']{VAR_NAME}['])|(["]{VAR_NAME}["])){NEW_LINE}
tb := l.TokenBytes(nil)
@ -356,7 +355,7 @@ NEW_LINE (\r|\n|\r\n)
l.ungetN(ungetCnt)
lval.token = token.NewToken(l.handleNewLine(tb));
lval.token = l.newToken(tb);
return T_START_HEREDOC
<NOWDOC>.
@ -383,10 +382,10 @@ NEW_LINE (\r|\n|\r\n)
c = l.Next()
}
lval.token = token.NewToken(l.handleNewLine(tb));
lval.token = l.newToken(tb)
return T_ENCAPSED_AND_WHITESPACE
<HEREDOC_END>{VAR_NAME}\; l.begin(PHP);lval.token = token.NewToken(l.handleNewLine(l.ungetN(1))); return T_END_HEREDOC
<HEREDOC_END>{VAR_NAME}\; l.begin(PHP);lval.token = l.newToken(l.ungetN(1)); return T_END_HEREDOC
<PHP>[b]?[\"]
binPrefix := l.TokenBytes(nil)[0] == 'b'
@ -398,7 +397,7 @@ NEW_LINE (\r|\n|\r\n)
tokenBytes := l.TokenBytes(nil)[:cnt]
l.pushState(STRING)
lval.token = token.NewToken(l.handleNewLine(tokenBytes)); return rune2Class('"')
lval.token = l.newToken(tokenBytes); return rune2Class('"')
}
F:for {
@ -409,7 +408,7 @@ NEW_LINE (\r|\n|\r\n)
switch c {
case '"' :
c = l.Next();
lval.token = l.newToken(); return T_CONSTANT_ENCAPSED_STRING
lval.token = l.newToken(l.TokenBytes(nil)); return T_CONSTANT_ENCAPSED_STRING
break F;
case '$':
@ -435,9 +434,9 @@ NEW_LINE (\r|\n|\r\n)
c = l.Next()
}
<STRING>\" l.popState(); lval.token = l.newToken(); return rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING,HEREDOC,BACKQUOTE>\{\$ lval.token = token.NewToken(l.handleNewLine(l.ungetN(1))); l.pushState(PHP); return T_CURLY_OPEN
<STRING,HEREDOC,BACKQUOTE>\$\{ l.pushState(STRING_VAR_NAME);lval.token = l.newToken(); return T_DOLLAR_OPEN_CURLY_BRACES
<STRING>\" l.popState(); lval.token = l.newToken(l.TokenBytes(nil)); return rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING,HEREDOC,BACKQUOTE>\{\$ lval.token = l.newToken(l.ungetN(1)); l.pushState(PHP); return T_CURLY_OPEN
<STRING,HEREDOC,BACKQUOTE>\$\{ l.pushState(STRING_VAR_NAME);lval.token = l.newToken(l.TokenBytes(nil)); return T_DOLLAR_OPEN_CURLY_BRACES
<STRING,HEREDOC,BACKQUOTE>\$ l.ungetN(1);l.pushState(STRING_VAR)
<STRING>.|[ \t\n\r]
F1:for {
@ -447,7 +446,7 @@ NEW_LINE (\r|\n|\r\n)
switch c {
case '"' :
lval.token = l.newToken();
lval.token = l.newToken(l.TokenBytes(nil));
return T_ENCAPSED_AND_WHITESPACE
break F1;
@ -456,7 +455,7 @@ NEW_LINE (\r|\n|\r\n)
if rune(c) == '{' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' || c >= '\u007f' && c <= 'ÿ' {
l.ungetN(1)
tb := l.TokenBytes(nil)
lval.token = token.NewToken(l.handleNewLine(tb[:len(tb)-1]));
lval.token = l.newToken(tb[:len(tb)-1]);
return T_ENCAPSED_AND_WHITESPACE
break F1;
}
@ -467,7 +466,7 @@ NEW_LINE (\r|\n|\r\n)
if rune(c) == '$' {
l.ungetN(1)
tb := l.TokenBytes(nil)
lval.token = token.NewToken(l.handleNewLine(tb[:len(tb)-1]));
lval.token = l.newToken(tb[:len(tb)-1]);
return T_ENCAPSED_AND_WHITESPACE
break F1;
}
@ -488,7 +487,7 @@ NEW_LINE (\r|\n|\r\n)
switch c {
case '`' :
lval.token = l.newToken();
lval.token = l.newToken(l.TokenBytes(nil));
return T_ENCAPSED_AND_WHITESPACE
break F2;
@ -497,7 +496,7 @@ NEW_LINE (\r|\n|\r\n)
if rune(c) == '{' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' || c >= '\u007f' && c <= 'ÿ' {
l.ungetN(1)
tb := l.TokenBytes(nil)
lval.token = token.NewToken(l.handleNewLine(tb[:len(tb)-1]));
lval.token = l.newToken(tb[:len(tb)-1]);
return T_ENCAPSED_AND_WHITESPACE
break F2;
}
@ -508,7 +507,7 @@ NEW_LINE (\r|\n|\r\n)
if rune(c) == '$' {
l.ungetN(1)
tb := l.TokenBytes(nil)
lval.token = token.NewToken(l.handleNewLine(tb[:len(tb)-1]));
lval.token = l.newToken(tb[:len(tb)-1]);
return T_ENCAPSED_AND_WHITESPACE
break F2;
}
@ -570,24 +569,24 @@ NEW_LINE (\r|\n|\r\n)
c = l.Next()
}
lval.token = token.NewToken(l.handleNewLine(tb));
lval.token = l.newToken(tb);
return T_ENCAPSED_AND_WHITESPACE
<STRING_VAR>\${VAR_NAME} lval.token = l.newToken(); return T_VARIABLE
<STRING_VAR>->{VAR_NAME} lval.token = token.NewToken(l.handleNewLine(l.ungetN(len(l.TokenBytes(nil))-2))); return T_OBJECT_OPERATOR
<STRING_VAR>{VAR_NAME} l.popState();lval.token = l.newToken(); return T_STRING
<STRING_VAR>\[ l.pushState(STRING_VAR_INDEX);lval.token = l.newToken(); return rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR>\${VAR_NAME} lval.token = l.newToken(l.TokenBytes(nil)); return T_VARIABLE
<STRING_VAR>->{VAR_NAME} lval.token = l.newToken(l.ungetN(len(l.TokenBytes(nil))-2)); return T_OBJECT_OPERATOR
<STRING_VAR>{VAR_NAME} l.popState();lval.token = l.newToken(l.TokenBytes(nil)); return T_STRING
<STRING_VAR>\[ l.pushState(STRING_VAR_INDEX);lval.token = l.newToken(l.TokenBytes(nil)); return rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR>.|[ \t\n\r] l.ungetN(1);l.popState()
<STRING_VAR_INDEX>{LNUM}|{HNUM}|{BNUM} lval.token = l.newToken(); return T_NUM_STRING
<STRING_VAR_INDEX>\${VAR_NAME} lval.token = l.newToken(); return T_VARIABLE
<STRING_VAR_INDEX>{VAR_NAME} lval.token = l.newToken(); return T_STRING
<STRING_VAR_INDEX>\] l.popState(); l.popState();lval.token = l.newToken(); return rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_INDEX>[ \n\r\t\\'#] l.popState(); l.popState();lval.token = l.newToken(); return T_ENCAPSED_AND_WHITESPACE
<STRING_VAR_INDEX>{OPERATORS} lval.token = l.newToken(); return rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_INDEX>. lval.token = l.newToken(); return rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_INDEX>{LNUM}|{HNUM}|{BNUM} lval.token = l.newToken(l.TokenBytes(nil)); return T_NUM_STRING
<STRING_VAR_INDEX>\${VAR_NAME} lval.token = l.newToken(l.TokenBytes(nil)); return T_VARIABLE
<STRING_VAR_INDEX>{VAR_NAME} lval.token = l.newToken(l.TokenBytes(nil)); return T_STRING
<STRING_VAR_INDEX>\] l.popState(); l.popState();lval.token = l.newToken(l.TokenBytes(nil)); return rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_INDEX>[ \n\r\t\\'#] l.popState(); l.popState();lval.token = l.newToken(l.TokenBytes(nil)); return T_ENCAPSED_AND_WHITESPACE
<STRING_VAR_INDEX>{OPERATORS} lval.token = l.newToken(l.TokenBytes(nil)); return rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_INDEX>. lval.token = l.newToken(l.TokenBytes(nil)); return rune2Class(rune(l.TokenBytes(nil)[0]))
<STRING_VAR_NAME>{VAR_NAME}[\[\}] l.popState();l.pushState(PHP);lval.token = token.NewToken(l.handleNewLine(l.ungetN(1))); return T_STRING_VARNAME
<STRING_VAR_NAME>{VAR_NAME}[\[\}] l.popState();l.pushState(PHP);lval.token = l.newToken(l.ungetN(1)); return T_STRING_VARNAME
<STRING_VAR_NAME>. l.ungetN(1);l.popState();l.pushState(PHP)
%%