php-parser/internal/scanner/lexer.go
2020-05-18 20:07:17 +03:00

258 lines
4.9 KiB
Go

package scanner
import (
"bytes"
"strings"
"github.com/z7zmey/php-parser/internal/version"
"github.com/z7zmey/php-parser/pkg/errors"
"github.com/z7zmey/php-parser/pkg/position"
"github.com/z7zmey/php-parser/pkg/token"
)
type Scanner interface {
Lex() *Token
ReturnTokenToPool(t *Token)
GetWithHiddenTokens() bool
SetWithHiddenTokens(bool)
}
type Lexer struct {
data []byte
errHandlerFunc func(*errors.Error)
p, pe, cs int
ts, te, act int
stack []int
top int
heredocLabel []byte
TokenPool *TokenPool
HiddenTokens []token.Token
WithHiddenTokens bool
NewLines NewLines
PHPVersion string
}
func NewLexer(data []byte, errHandlerFunc func(*errors.Error)) *Lexer {
lex := &Lexer{
data: data,
errHandlerFunc: errHandlerFunc,
pe: len(data),
stack: make([]int, 0),
TokenPool: &TokenPool{},
NewLines: NewLines{make([]int, 0, 128)},
}
initLexer(lex)
return lex
}
func (l *Lexer) ReturnTokenToPool(t *Token) {
l.TokenPool.Put(t)
}
func (l *Lexer) GetWithHiddenTokens() bool {
return l.WithHiddenTokens
}
func (l *Lexer) SetWithHiddenTokens(b bool) {
l.WithHiddenTokens = b
}
func (lex *Lexer) setTokenPosition(token *Token) {
token.Position.StartLine = lex.NewLines.GetLine(lex.ts)
token.Position.EndLine = lex.NewLines.GetLine(lex.te - 1)
token.Position.StartPos = lex.ts
token.Position.EndPos = lex.te
}
func (lex *Lexer) addToken(id TokenID, ps, pe int) {
if !lex.WithHiddenTokens {
return
}
lex.HiddenTokens = append(lex.HiddenTokens, token.Token{
ID: token.ID(id),
Value: lex.data[ps:pe],
})
}
func (lex *Lexer) isNotStringVar() bool {
p := lex.p
if lex.data[p-1] == '\\' && lex.data[p-2] != '\\' {
return true
}
if len(lex.data) < p+1 {
return true
}
if lex.data[p] == '$' && (lex.data[p+1] == '{' || isValidVarNameStart(lex.data[p+1])) {
return false
}
if lex.data[p] == '{' && lex.data[p+1] == '$' {
return false
}
return true
}
func (lex *Lexer) isNotStringEnd(s byte) bool {
p := lex.p
if lex.data[p-1] == '\\' && lex.data[p-2] != '\\' {
return true
}
return !(lex.data[p] == s)
}
func (lex *Lexer) isHeredocEnd(p int) bool {
r, err := version.Compare(lex.PHPVersion, "7.3")
if err != nil {
return lex.isHeredocEndSince73(p)
}
if r == -1 {
return lex.isHeredocEndBefore73(p)
}
return lex.isHeredocEndSince73(p)
}
func (lex *Lexer) isHeredocEndBefore73(p int) bool {
if lex.data[p-1] != '\r' && lex.data[p-1] != '\n' {
return false
}
l := len(lex.heredocLabel)
if len(lex.data) < p+l {
return false
}
if len(lex.data) > p+l && lex.data[p+l] != ';' && lex.data[p+l] != '\r' && lex.data[p+l] != '\n' {
return false
}
if len(lex.data) > p+l+1 && lex.data[p+l] == ';' && lex.data[p+l+1] != '\r' && lex.data[p+l+1] != '\n' {
return false
}
return bytes.Equal(lex.heredocLabel, lex.data[p:p+l])
}
func (lex *Lexer) isHeredocEndSince73(p int) bool {
if lex.data[p-1] != '\r' && lex.data[p-1] != '\n' {
return false
}
for lex.data[p] == ' ' || lex.data[p] == '\t' {
p++
}
l := len(lex.heredocLabel)
if len(lex.data) < p+l {
return false
}
if len(lex.data) > p+l && isValidVarName(lex.data[p+l]) {
return false
}
a := string(lex.heredocLabel)
b := string(lex.data[p : p+l])
_, _ = a, b
if bytes.Equal(lex.heredocLabel, lex.data[p:p+l]) {
lex.p = p
return true
}
return false
}
func (lex *Lexer) isNotHeredocEnd(p int) bool {
return !lex.isHeredocEnd(p)
}
func (lex *Lexer) growCallStack() {
if lex.top == len(lex.stack) {
lex.stack = append(lex.stack, 0)
}
}
func (lex *Lexer) isNotPhpCloseToken() bool {
if lex.p+1 == len(lex.data) {
return true
}
return lex.data[lex.p] != '?' || lex.data[lex.p+1] != '>'
}
func (lex *Lexer) isNotNewLine() bool {
if lex.data[lex.p] == '\n' && lex.data[lex.p-1] == '\r' {
return true
}
return lex.data[lex.p-1] != '\n' && lex.data[lex.p-1] != '\r'
}
func (lex *Lexer) call(state int, fnext int) {
lex.growCallStack()
lex.stack[lex.top] = state
lex.top++
lex.p++
lex.cs = fnext
}
func (lex *Lexer) ret(n int) {
lex.top = lex.top - n
if lex.top < 0 {
lex.top = 0
}
lex.cs = lex.stack[lex.top]
lex.p++
}
func (lex *Lexer) ungetStr(s string) {
tokenStr := string(lex.data[lex.ts:lex.te])
if strings.HasSuffix(tokenStr, s) {
lex.ungetCnt(len(s))
}
}
func (lex *Lexer) ungetCnt(n int) {
lex.p = lex.p - n
lex.te = lex.te - n
}
func (lex *Lexer) error(msg string) {
if lex.errHandlerFunc == nil {
return
}
pos := position.NewPosition(
lex.NewLines.GetLine(lex.ts),
lex.NewLines.GetLine(lex.te-1),
lex.ts,
lex.te,
)
lex.errHandlerFunc(errors.NewError(msg, pos))
}
func isValidVarNameStart(r byte) bool {
return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || r == '_' || (r >= 0x80 && r <= 0xff)
}
func isValidVarName(r byte) bool {
return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || (r >= 0x80 && r <= 0xff)
}