php-parser/internal/scanner/lexer.go

247 lines
4.8 KiB
Go
Raw Normal View History

2018-01-24 16:42:23 +00:00
package scanner
import (
"bytes"
2019-03-10 21:37:01 +00:00
"strings"
2018-01-24 16:42:23 +00:00
2020-05-12 21:16:36 +00:00
"github.com/z7zmey/php-parser/internal/version"
"github.com/z7zmey/php-parser/pkg/errors"
"github.com/z7zmey/php-parser/pkg/position"
"github.com/z7zmey/php-parser/pkg/token"
2018-01-24 16:42:23 +00:00
)
type Lexer struct {
data []byte
phpVersion string
errHandlerFunc func(*errors.Error)
p, pe, cs int
ts, te, act int
stack []int
top int
2019-03-10 21:37:01 +00:00
heredocLabel []byte
2020-08-17 17:31:04 +00:00
tokenPool *token.Pool
positionPool *position.Pool
newLines NewLines
2018-01-24 16:42:23 +00:00
}
2020-08-17 17:31:04 +00:00
func NewLexer(data []byte, phpVersion string, errHandlerFunc func(*errors.Error)) *Lexer {
lex := &Lexer{
data: data,
phpVersion: phpVersion,
errHandlerFunc: errHandlerFunc,
pe: len(data),
stack: make([]int, 0),
2020-08-17 17:31:04 +00:00
tokenPool: token.NewPool(position.DefaultBlockSize),
2020-12-08 00:08:59 +00:00
positionPool: position.NewPool(token.DefaultBlockSize),
2020-08-17 17:31:04 +00:00
newLines: NewLines{make([]int, 0, 128)},
}
initLexer(lex)
return lex
2018-01-24 16:42:23 +00:00
}
2020-08-17 17:31:04 +00:00
func (lex *Lexer) setTokenPosition(token *token.Token) {
pos := lex.positionPool.Get()
pos.StartLine = lex.newLines.GetLine(lex.ts)
pos.EndLine = lex.newLines.GetLine(lex.te - 1)
pos.StartPos = lex.ts
pos.EndPos = lex.te
2019-03-10 21:37:01 +00:00
2020-08-17 17:31:04 +00:00
token.Position = pos
2019-03-10 21:37:01 +00:00
}
2020-12-07 23:23:48 +00:00
func (lex *Lexer) addFreeFloatingToken(t *token.Token, id token.ID, ps, pe int) {
2020-08-17 17:31:04 +00:00
skippedTkn := lex.tokenPool.Get()
skippedTkn.ID = id
skippedTkn.Value = lex.data[ps:pe]
lex.setTokenPosition(skippedTkn)
2020-12-07 23:23:48 +00:00
if t.FreeFloating == nil {
t.FreeFloating = make([]*token.Token, 0, 2)
2019-03-10 21:37:01 +00:00
}
2020-12-07 23:23:48 +00:00
t.FreeFloating = append(t.FreeFloating, skippedTkn)
}
2019-03-10 21:37:01 +00:00
func (lex *Lexer) isNotStringVar() bool {
p := lex.p
if lex.data[p-1] == '\\' && lex.data[p-2] != '\\' {
return true
}
2018-01-24 16:42:23 +00:00
2019-03-10 21:37:01 +00:00
if len(lex.data) < p+1 {
return true
}
2018-01-24 16:42:23 +00:00
2019-03-10 21:37:01 +00:00
if lex.data[p] == '$' && (lex.data[p+1] == '{' || isValidVarNameStart(lex.data[p+1])) {
return false
2018-01-24 16:42:23 +00:00
}
2019-03-10 21:37:01 +00:00
if lex.data[p] == '{' && lex.data[p+1] == '$' {
return false
}
2018-01-24 16:42:23 +00:00
2019-03-10 21:37:01 +00:00
return true
2018-01-24 16:42:23 +00:00
}
2019-03-10 21:37:01 +00:00
func (lex *Lexer) isNotStringEnd(s byte) bool {
p := lex.p
if lex.data[p-1] == '\\' && lex.data[p-2] != '\\' {
return true
}
return !(lex.data[p] == s)
2018-01-24 16:42:23 +00:00
}
2019-03-10 21:37:01 +00:00
func (lex *Lexer) isHeredocEnd(p int) bool {
r, err := version.Compare(lex.phpVersion, "7.3")
2019-12-26 15:57:56 +00:00
if err != nil {
2019-12-26 13:41:06 +00:00
return lex.isHeredocEndSince73(p)
}
2019-12-26 15:57:56 +00:00
if r == -1 {
2019-12-26 13:41:06 +00:00
return lex.isHeredocEndBefore73(p)
}
return lex.isHeredocEndSince73(p)
}
func (lex *Lexer) isHeredocEndBefore73(p int) bool {
2019-03-10 21:37:01 +00:00
if lex.data[p-1] != '\r' && lex.data[p-1] != '\n' {
return false
2018-01-24 16:42:23 +00:00
}
2019-03-10 21:37:01 +00:00
l := len(lex.heredocLabel)
if len(lex.data) < p+l {
return false
}
if len(lex.data) > p+l && lex.data[p+l] != ';' && lex.data[p+l] != '\r' && lex.data[p+l] != '\n' {
return false
}
if len(lex.data) > p+l+1 && lex.data[p+l] == ';' && lex.data[p+l+1] != '\r' && lex.data[p+l+1] != '\n' {
return false
}
2018-01-24 16:42:23 +00:00
2019-03-10 21:37:01 +00:00
return bytes.Equal(lex.heredocLabel, lex.data[p:p+l])
2018-01-24 16:42:23 +00:00
}
2019-12-26 13:41:06 +00:00
func (lex *Lexer) isHeredocEndSince73(p int) bool {
if lex.data[p-1] != '\r' && lex.data[p-1] != '\n' {
return false
}
for lex.data[p] == ' ' || lex.data[p] == '\t' {
p++
}
l := len(lex.heredocLabel)
if len(lex.data) < p+l {
return false
}
if len(lex.data) > p+l && isValidVarName(lex.data[p+l]) {
return false
}
a := string(lex.heredocLabel)
b := string(lex.data[p : p+l])
_, _ = a, b
if bytes.Equal(lex.heredocLabel, lex.data[p:p+l]) {
lex.p = p
return true
}
return false
}
2019-03-10 21:37:01 +00:00
func (lex *Lexer) isNotHeredocEnd(p int) bool {
return !lex.isHeredocEnd(p)
2018-01-24 16:42:23 +00:00
}
2019-03-10 21:37:01 +00:00
func (lex *Lexer) growCallStack() {
if lex.top == len(lex.stack) {
lex.stack = append(lex.stack, 0)
}
}
2018-01-24 16:42:23 +00:00
2019-03-10 21:37:01 +00:00
func (lex *Lexer) isNotPhpCloseToken() bool {
if lex.p+1 == len(lex.data) {
return true
}
2019-03-10 21:37:01 +00:00
return lex.data[lex.p] != '?' || lex.data[lex.p+1] != '>'
}
2019-03-10 21:37:01 +00:00
func (lex *Lexer) isNotNewLine() bool {
if lex.data[lex.p] == '\n' && lex.data[lex.p-1] == '\r' {
return true
}
2018-06-24 07:19:44 +00:00
2019-03-10 21:37:01 +00:00
return lex.data[lex.p-1] != '\n' && lex.data[lex.p-1] != '\r'
2018-01-24 16:42:23 +00:00
}
2019-03-10 21:37:01 +00:00
func (lex *Lexer) call(state int, fnext int) {
lex.growCallStack()
2019-03-10 21:37:01 +00:00
lex.stack[lex.top] = state
lex.top++
2019-03-10 21:37:01 +00:00
lex.p++
lex.cs = fnext
}
2019-03-10 21:37:01 +00:00
func (lex *Lexer) ret(n int) {
lex.top = lex.top - n
if lex.top < 0 {
lex.top = 0
}
lex.cs = lex.stack[lex.top]
lex.p++
}
2019-03-10 21:37:01 +00:00
func (lex *Lexer) ungetStr(s string) {
tokenStr := string(lex.data[lex.ts:lex.te])
if strings.HasSuffix(tokenStr, s) {
lex.ungetCnt(len(s))
}
2019-03-10 21:37:01 +00:00
}
2019-03-10 21:37:01 +00:00
func (lex *Lexer) ungetCnt(n int) {
lex.p = lex.p - n
lex.te = lex.te - n
}
2018-04-15 18:39:26 +00:00
func (lex *Lexer) error(msg string) {
if lex.errHandlerFunc == nil {
return
}
2018-04-15 18:39:26 +00:00
pos := position.NewPosition(
lex.newLines.GetLine(lex.ts),
lex.newLines.GetLine(lex.te-1),
2019-03-10 21:37:01 +00:00
lex.ts,
lex.te,
2018-04-15 18:39:26 +00:00
)
lex.errHandlerFunc(errors.NewError(msg, pos))
2019-03-10 21:37:01 +00:00
}
func isValidVarNameStart(r byte) bool {
return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || r == '_' || r >= 0x80
2019-12-26 13:41:06 +00:00
}
func isValidVarName(r byte) bool {
return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r >= 0x80
2019-12-26 13:41:06 +00:00
}