PHP 8 (#1)

PHP 8 Update - nullsafe operator (?->) - Remove (real) cast - Named arguments - Remove (unset) cast - Remove {} access - match expression - Union types in type hints and static typehint - Block catch without variable - Trailing comma in parameter lists - throw can be used as an expression - Concatenation precedence - Declaring properties in the constructor - Attributes - Names in the namespace are treated as a single token - Trailing comma in closure use list - Check that ::class on object works - Deferencable changes and arbitrary expressions in new/instanceof
2021-07-30 20:53:27 +03:00
parent 367eff9de6
commit 049ce7ddc6
40 changed files with 108754 additions and 264 deletions
--- a/internal/php5/php5.go
+++ b/internal/php5/php5.go
@@ -333,7 +333,6 @@ var yyToknames = [...]string{
 	"'>'",
 	"'.'",
 }
-
 var yyStatenames = [...]string{}

 const yyEofCode = 1
@@ -589,6 +588,7 @@ const yyPrivate = 57344
 const yyLast = 8973

 var yyAct = [...]int{
+
 	102, 571, 1007, 565, 420, 966, 844, 715, 808, 607,
 	123, 131, 919, 200, 827, 735, 567, 684, 452, 594,
 	461, 816, 474, 592, 61, 579, 421, 319, 390, 142,
@@ -1488,8 +1488,8 @@ var yyAct = [...]int{
 	0, 0, 0, 536, 0, 525, 0, 0, 0, 535,
 	534, 532, 533,
 }
-
 var yyPact = [...]int{
+
 	-1000, -1000, 2243, -1000, -1000, -1000, -1000, -1000, 321, 547,
 	755, 70, -1000, 286, -1000, -1000, 946, -1000, 242, 242,
 	5391, 319, 242, 6739, 6627, 6515, 383, 172, 795, 6851,
@@ -1594,8 +1594,8 @@ var yyPact = [...]int{
 	-1000, -1000, -1000, -1000, 718, 2559, 807, -1000, 36, 396,
 	-1000, 2401, -1000,
 }
-
 var yyPgo = [...]int{
+
 	0, 29, 1177, 22, 8, 13, 1175, 1167, 40, 33,
 	27, 940, 1165, 1164, 36, 217, 1179, 91, 1036, 72,
 	99, 59, 810, 200, 1163, 31, 1162, 1154, 1152, 42,
@@ -1613,8 +1613,8 @@ var yyPgo = [...]int{
 	23, 968, 19, 9, 965, 37, 964, 963, 962, 960,
 	182, 923,
 }
-
 var yyR1 = [...]int{
+
 	0, 151, 122, 122, 115, 115, 7, 7, 7, 7,
 	7, 7, 7, 7, 7, 7, 7, 7, 117, 117,
 	8, 8, 8, 8, 118, 118, 9, 9, 9, 9,
@@ -1669,8 +1669,8 @@ var yyR1 = [...]int{
 	41, 41, 41, 107, 107, 40, 40, 34, 34, 35,
 	36,
 }
-
 var yyR2 = [...]int{
+
 	0, 1, 2, 0, 1, 3, 1, 1, 1, 1,
 	4, 3, 5, 4, 3, 4, 4, 2, 3, 1,
 	1, 3, 2, 4, 3, 1, 1, 3, 2, 4,
@@ -1725,8 +1725,8 @@ var yyR2 = [...]int{
 	4, 2, 2, 1, 3, 1, 1, 3, 3, 3,
 	3,
 }
-
 var yyChk = [...]int{
+
 	-1000, -151, -122, -7, 2, -29, -55, -56, 52, 80,
 	45, -57, -30, 10, -66, -67, 39, 144, 7, 21,
 	20, 23, 30, 34, 35, 40, -54, 47, 99, 19,
@@ -1831,8 +1831,8 @@ var yyChk = [...]int{
 	146, 144, -49, -65, 148, -123, -49, 145, 12, 149,
 	144, -123, 145,
 }
-
 var yyDef = [...]int{
+
 	3, -2, -2, 2, 6, 7, 8, 9, 0, 0,
 	0, 0, 45, 4, 87, 88, 0, 39, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0, 331, 0,
@@ -1937,8 +1937,8 @@ var yyDef = [...]int{
 	196, 39, 188, 82, 0, 0, 0, 197, 0, 0,
 	39, 0, 83,
 }
-
 var yyTok1 = [...]int{
+
 	1, 3, 3, 3, 3, 3, 3, 3, 3, 3,
 	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
 	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
@@ -1953,8 +1953,8 @@ var yyTok1 = [...]int{
 	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
 	3, 3, 3, 144, 161, 145, 157,
 }
-
 var yyTok2 = [...]int{
+
 	2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
 	12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
 	22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
@@ -1970,7 +1970,6 @@ var yyTok2 = [...]int{
 	122, 123, 124, 125, 126, 127, 128, 129, 130, 131,
 	132, 133, 134, 135, 136, 137, 138, 139, 140, 141,
 }
-
 var yyTok3 = [...]int{
 	0,
 }
--- a/internal/php7/php7.go
+++ b/internal/php7/php7.go
--- a/internal/php8/builder.go
+++ b/internal/php8/builder.go
--- a/internal/php8/lexer.go
+++ b/internal/php8/lexer.go
@@ -0,0 +1,251 @@
+package php8
+
+import (
+	"bytes"
+	"strings"
+
+	"github.com/z7zmey/php-parser/pkg/conf"
+	"github.com/z7zmey/php-parser/pkg/errors"
+	"github.com/z7zmey/php-parser/pkg/position"
+	"github.com/z7zmey/php-parser/pkg/token"
+	"github.com/z7zmey/php-parser/pkg/version"
+)
+
+type Lexer struct {
+	data           []byte
+	phpVersion     *version.Version
+	errHandlerFunc func(*errors.Error)
+
+	p, pe, cs   int
+	ts, te, act int
+	stack       []int
+	top         int
+
+	heredocLabel []byte
+	tokenPool    *token.Pool
+	positionPool *position.Pool
+	newLines     NewLines
+}
+
+func NewLexer(data []byte, config conf.Config) *Lexer {
+	lex := &Lexer{
+		data:           data,
+		phpVersion:     config.Version,
+		errHandlerFunc: config.ErrorHandlerFunc,
+
+		pe:    len(data),
+		stack: make([]int, 0),
+
+		tokenPool:    token.NewPool(position.DefaultBlockSize),
+		positionPool: position.NewPool(token.DefaultBlockSize),
+		newLines:     NewLines{make([]int, 0, 128)},
+	}
+
+	initLexer(lex)
+
+	return lex
+}
+
+func (lex *Lexer) setTokenPosition(token *token.Token) {
+	pos := lex.positionPool.Get()
+
+	pos.StartLine = lex.newLines.GetLine(lex.ts)
+	pos.EndLine = lex.newLines.GetLine(lex.te - 1)
+	pos.StartPos = lex.ts
+	pos.EndPos = lex.te
+
+	token.Position = pos
+}
+
+func (lex *Lexer) addFreeFloatingToken(t *token.Token, id token.ID, ps, pe int) {
+	skippedTkn := lex.tokenPool.Get()
+	skippedTkn.ID = id
+	skippedTkn.Value = lex.data[ps:pe]
+
+	lex.setTokenPosition(skippedTkn)
+
+	if t.FreeFloating == nil {
+		t.FreeFloating = make([]*token.Token, 0, 2)
+	}
+
+	t.FreeFloating = append(t.FreeFloating, skippedTkn)
+}
+
+func (lex *Lexer) isNotStringVar() bool {
+	p := lex.p
+	if lex.data[p-1] == '\\' && lex.data[p-2] != '\\' {
+		return true
+	}
+
+	if len(lex.data) < p+1 {
+		return true
+	}
+
+	if lex.data[p] == '$' && (lex.data[p+1] == '{' || isValidVarNameStart(lex.data[p+1])) {
+		return false
+	}
+
+	if lex.data[p] == '{' && lex.data[p+1] == '$' {
+		return false
+	}
+
+	return true
+}
+
+func (lex *Lexer) isNotStringEnd(s byte) bool {
+	p := lex.p
+	if lex.data[p-1] == '\\' && lex.data[p-2] != '\\' {
+		return true
+	}
+
+	return !(lex.data[p] == s)
+}
+
+func (lex *Lexer) isHeredocEnd(p int) bool {
+	o, err := version.New("7.3")
+	if err != nil {
+		panic(err)
+	}
+
+	if lex.phpVersion.GreaterOrEqual(o) {
+		return lex.isHeredocEndSince73(p)
+	}
+
+	return lex.isHeredocEndBefore73(p)
+}
+
+func (lex *Lexer) isHeredocEndBefore73(p int) bool {
+	if lex.data[p-1] != '\r' && lex.data[p-1] != '\n' {
+		return false
+	}
+
+	l := len(lex.heredocLabel)
+	if len(lex.data) < p+l {
+		return false
+	}
+
+	if len(lex.data) > p+l && lex.data[p+l] != ';' && lex.data[p+l] != '\r' && lex.data[p+l] != '\n' {
+		return false
+	}
+
+	if len(lex.data) > p+l+1 && lex.data[p+l] == ';' && lex.data[p+l+1] != '\r' && lex.data[p+l+1] != '\n' {
+		return false
+	}
+
+	return bytes.Equal(lex.heredocLabel, lex.data[p:p+l])
+}
+
+func (lex *Lexer) isHeredocEndSince73(p int) bool {
+	if lex.data[p-1] != '\r' && lex.data[p-1] != '\n' {
+		return false
+	}
+
+	if p == len(lex.data) {
+		return false
+	}
+
+	for lex.data[p] == ' ' || lex.data[p] == '\t' {
+		p++
+	}
+
+	l := len(lex.heredocLabel)
+	if len(lex.data) < p+l {
+		return false
+	}
+
+	if len(lex.data) > p+l && isValidVarName(lex.data[p+l]) {
+		return false
+	}
+
+	a := string(lex.heredocLabel)
+	b := string(lex.data[p : p+l])
+
+	_, _ = a, b
+
+	if bytes.Equal(lex.heredocLabel, lex.data[p:p+l]) {
+		lex.p = p
+		return true
+	}
+
+	return false
+}
+
+func (lex *Lexer) isNotHeredocEnd(p int) bool {
+	return !lex.isHeredocEnd(p)
+}
+
+func (lex *Lexer) growCallStack() {
+	if lex.top == len(lex.stack) {
+		lex.stack = append(lex.stack, 0)
+	}
+}
+
+func (lex *Lexer) isNotPhpCloseToken() bool {
+	if lex.p+1 == len(lex.data) {
+		return true
+	}
+
+	return lex.data[lex.p] != '?' || lex.data[lex.p+1] != '>'
+}
+
+func (lex *Lexer) isNotNewLine() bool {
+	if lex.data[lex.p] == '\n' && lex.data[lex.p-1] == '\r' {
+		return true
+	}
+
+	return lex.data[lex.p-1] != '\n' && lex.data[lex.p-1] != '\r'
+}
+
+func (lex *Lexer) call(state int, fnext int) {
+	lex.growCallStack()
+
+	lex.stack[lex.top] = state
+	lex.top++
+
+	lex.p++
+	lex.cs = fnext
+}
+
+func (lex *Lexer) ret(n int) {
+	lex.top = lex.top - n
+	if lex.top < 0 {
+		lex.top = 0
+	}
+	lex.cs = lex.stack[lex.top]
+	lex.p++
+}
+
+func (lex *Lexer) ungetStr(s string) {
+	tokenStr := string(lex.data[lex.ts:lex.te])
+	if strings.HasSuffix(tokenStr, s) {
+		lex.ungetCnt(len(s))
+	}
+}
+
+func (lex *Lexer) ungetCnt(n int) {
+	lex.p = lex.p - n
+	lex.te = lex.te - n
+}
+
+func (lex *Lexer) error(msg string) {
+	if lex.errHandlerFunc == nil {
+		return
+	}
+
+	pos := position.NewPosition(
+		lex.newLines.GetLine(lex.ts),
+		lex.newLines.GetLine(lex.te-1),
+		lex.ts,
+		lex.te,
+	)
+
+	lex.errHandlerFunc(errors.NewError(msg, pos))
+}
+
+func isValidVarNameStart(r byte) bool {
+	return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || r == '_' || r >= 0x80
+}
+
+func isValidVarName(r byte) bool {
+	return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r >= 0x80
+}
--- a/internal/php8/newline.go
+++ b/internal/php8/newline.go
@@ -0,0 +1,25 @@
+package php8
+
+type NewLines struct {
+	data []int
+}
+
+func (nl *NewLines) Append(p int) {
+	if len(nl.data) == 0 || nl.data[len(nl.data)-1] < p {
+		nl.data = append(nl.data, p)
+	}
+}
+
+func (nl *NewLines) GetLine(p int) int {
+	line := len(nl.data) + 1
+
+	for i := len(nl.data) - 1; i >= 0; i-- {
+		if p < nl.data[i] {
+			line = i + 1
+		} else {
+			break
+		}
+	}
+
+	return line
+}
--- a/internal/php8/node.go
+++ b/internal/php8/node.go
@@ -0,0 +1,99 @@
+package php8
+
+import (
+	"github.com/z7zmey/php-parser/pkg/ast"
+	"github.com/z7zmey/php-parser/pkg/position"
+	"github.com/z7zmey/php-parser/pkg/token"
+)
+
+type ParserBrackets struct {
+	Position        *position.Position
+	OpenBracketTkn  *token.Token
+	Child           ast.Vertex
+	CloseBracketTkn *token.Token
+}
+
+func (n *ParserBrackets) Accept(v ast.Visitor) {
+	// do nothing
+}
+
+func (n *ParserBrackets) GetPosition() *position.Position {
+	return n.Position
+}
+
+type ParserSeparatedList struct {
+	Position      *position.Position
+	Items         []ast.Vertex
+	SeparatorTkns []*token.Token
+}
+
+func (n *ParserSeparatedList) Accept(v ast.Visitor) {
+	// do nothing
+}
+
+func (n *ParserSeparatedList) GetPosition() *position.Position {
+	return n.Position
+}
+
+// TraitAdaptationList node
+type TraitAdaptationList struct {
+	Position             *position.Position
+	OpenCurlyBracketTkn  *token.Token
+	Adaptations          []ast.Vertex
+	CloseCurlyBracketTkn *token.Token
+}
+
+func (n *TraitAdaptationList) Accept(v ast.Visitor) {
+	// do nothing
+}
+
+func (n *TraitAdaptationList) GetPosition() *position.Position {
+	return n.Position
+}
+
+// ArgumentList node
+type ArgumentList struct {
+	Position            *position.Position
+	OpenParenthesisTkn  *token.Token
+	Arguments           []ast.Vertex
+	SeparatorTkns       []*token.Token
+	CloseParenthesisTkn *token.Token
+}
+
+func (n *ArgumentList) Accept(v ast.Visitor) {
+	// do nothing
+}
+
+func (n *ArgumentList) GetPosition() *position.Position {
+	return n.Position
+}
+
+type ReturnType struct {
+	Position *position.Position
+	ColonTkn *token.Token
+	Type     ast.Vertex
+}
+
+func (n *ReturnType) Accept(v ast.Visitor) {
+	// do nothing
+}
+
+func (n *ReturnType) GetPosition() *position.Position {
+	return n.Position
+}
+
+// TraitMethodRef node
+type TraitMethodRef struct {
+	Position       *position.Position
+	Trait          ast.Vertex
+	DoubleColonTkn *token.Token
+	Method         ast.Vertex
+}
+
+func (n *TraitMethodRef) Accept(v ast.Visitor) {
+	// do nothing
+}
+
+func (n *TraitMethodRef) GetPosition() *position.Position {
+	return n.Position
+}
--- a/internal/php8/parser.go
+++ b/internal/php8/parser.go
@@ -0,0 +1,66 @@
+package php8
+
+import (
+	"github.com/z7zmey/php-parser/internal/position"
+	"github.com/z7zmey/php-parser/pkg/ast"
+	"github.com/z7zmey/php-parser/pkg/conf"
+	"github.com/z7zmey/php-parser/pkg/errors"
+	"github.com/z7zmey/php-parser/pkg/token"
+)
+
+// Parser structure
+type Parser struct {
+	Lexer          *Lexer
+	currentToken   *token.Token
+	rootNode       ast.Vertex
+	errHandlerFunc func(*errors.Error)
+	builder        *Builder
+}
+
+// NewParser creates and returns new Parser
+func NewParser(lexer *Lexer, config conf.Config) *Parser {
+	p := &Parser{
+		Lexer:          lexer,
+		errHandlerFunc: config.ErrorHandlerFunc,
+	}
+	p.builder = NewBuilder(position.NewBuilder(), p)
+	return p
+}
+
+func (p *Parser) Lex(lval *yySymType) int {
+	t := p.Lexer.Lex()
+
+	p.currentToken = t
+	lval.token = t
+
+	return int(t.ID)
+}
+
+func (p *Parser) Error(msg string) {
+	if p.errHandlerFunc == nil {
+		return
+	}
+
+	p.errHandlerFunc(errors.NewError(msg, p.currentToken.Position))
+}
+
+// Parse the php7 Parser entrypoint
+func (p *Parser) Parse() int {
+	p.rootNode = nil
+
+	return yyParse(p)
+}
+
+// GetRootNode returns root node
+func (p *Parser) GetRootNode() ast.Vertex {
+	return p.rootNode
+}
+
+// helpers
+
+func lastNode(nn []ast.Vertex) ast.Vertex {
+	if len(nn) == 0 {
+		return nil
+	}
+	return nn[len(nn)-1]
+}
--- a/internal/php8/parser_php8_test.go
+++ b/internal/php8/parser_php8_test.go
--- a/internal/php8/parser_test.go
+++ b/internal/php8/parser_test.go
--- a/internal/php8/php8.go
+++ b/internal/php8/php8.go
--- a/internal/php8/php8.y
+++ b/internal/php8/php8.y
--- a/internal/php8/php8_bench_test.go
+++ b/internal/php8/php8_bench_test.go
@@ -0,0 +1,30 @@
+package php8_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/z7zmey/php-parser/internal/php8"
+	"github.com/z7zmey/php-parser/pkg/conf"
+	"github.com/z7zmey/php-parser/pkg/version"
+)
+
+func BenchmarkPhp8(b *testing.B) {
+	src, err := ioutil.ReadFile("test.php")
+
+	if err != nil {
+		b.Fatal("can not read test.php: " + err.Error())
+	}
+
+	for n := 0; n < b.N; n++ {
+		config := conf.Config{
+			Version: &version.Version{
+				Major: 8,
+				Minor: 8,
+			},
+		}
+		lexer := php8.NewLexer(src, config)
+		php8parser := php8.NewParser(lexer, config)
+		php8parser.Parse()
+	}
+}
--- a/internal/php8/scanner.go
+++ b/internal/php8/scanner.go
--- a/internal/php8/scanner.rl
+++ b/internal/php8/scanner.rl
@@ -0,0 +1,516 @@
+package php8
+
+import (
+    "fmt"
+    "strconv"
+    "strings"
+
+    "github.com/z7zmey/php-parser/pkg/token"
+)
+
+%%{ 
+    machine lexer;
+    write data;
+    access lex.;
+    variable p lex.p;
+    variable pe lex.pe;
+}%%
+
+func initLexer(lex *Lexer)  {
+    %% write init;
+}
+
+func (lex *Lexer) Lex() *token.Token {
+    eof := lex.pe
+    var tok token.ID
+
+    tkn := lex.tokenPool.Get()
+
+    lblStart := 0
+    lblEnd   := 0
+
+    _, _ = lblStart, lblEnd
+
+    %%{ 
+        action heredoc_lbl_start {lblStart = lex.p}
+        action heredoc_lbl_end   {lblEnd = lex.p}
+
+        action new_line   {
+            if lex.data[lex.p] == '\n' {
+                lex.newLines.Append(lex.p+1)
+            }
+
+            if lex.data[lex.p] == '\r' && lex.data[lex.p+1] != '\n' {
+                lex.newLines.Append(lex.p+1)
+            }
+        }
+
+        action is_not_heredoc_end { lex.isNotHeredocEnd(lex.p) }
+        action is_not_comment_end { lex.isNotPhpCloseToken() && lex.isNotNewLine()  }
+        action is_not_heredoc_end_or_var { lex.isNotHeredocEnd(lex.p) && lex.isNotStringVar() }
+        action is_not_string_end_or_var { lex.isNotStringEnd('"') && lex.isNotStringVar() }
+        action is_not_backqoute_end_or_var { lex.isNotStringEnd('`') && lex.isNotStringVar() }
+
+        newline = ('\r\n' >(nl, 1) | '\r' >(nl, 0) | '\n' >(nl, 0)) $new_line %{};
+        any_line = any | newline;
+        whitespace = [\t\v\f ];
+        whitespace_line = [\t\v\f ] | newline;
+
+        lnum = [0-9]+('_'[0-9]+)*;
+        dnum = (lnum?"." lnum)|(lnum"."lnum?);
+        hnum = '0x'[0-9a-fA-F]+('_'[0-9a-fA-F]+)*;
+        bnum = '0b'[01]+('_'[01]+)*;
+
+        exponent_dnum = (lnum | dnum) ('e'|'E') ('+'|'-')? lnum;
+        varname_first = [a-zA-Z_] | (0x0080..0x00FF);
+        varname_second = varname_first | [0-9];
+        varname       = varname_first (varname_second)*;
+        heredoc_label = varname >heredoc_lbl_start %heredoc_lbl_end;
+        operators     = ';'|':'|','|'.'|'['|']'|'('|')'|'|'|'/'|'^'|'&'|'+'|'-'|'*'|'='|'%'|'!'|'~'|'$'|'<'|'>'|'?'|'@';
+
+        prepush { lex.growCallStack(); }
+
+        constant_string =
+            start: (
+                "'"         -> qoute
+                | "b"i? '"' -> double_qoute
+            ),
+
+            # single qoute string
+
+            qoute: (
+                (any - [\\'\r\n]) -> qoute
+                | "\r" @new_line  -> qoute
+                | "\n" @new_line  -> qoute
+                | "\\"            -> qoute_any
+                | "'"             -> final
+            ),
+            qoute_any: (
+                (any - [\r\n])   -> qoute
+                | "\r" @new_line -> qoute
+                | "\n" @new_line -> qoute
+            ),
+
+            # double qoute string
+
+            double_qoute: (
+                (any - [\\"${\r\n]) -> double_qoute
+                | "\r" @new_line    -> double_qoute
+                | "\n" @new_line    -> double_qoute
+                | "\\"              -> double_qoute_any
+                | '"'               -> final
+                | '$'               -> double_qoute_nonvarname
+                | '{'               -> double_qoute_nondollar
+            ),
+            double_qoute_any: (
+                (any - [\r\n])     -> double_qoute
+                | "\r" @new_line   -> double_qoute
+                | "\n" @new_line   -> double_qoute
+            ),
+            double_qoute_nondollar: (
+                (any - [\\$"\r\n]) -> double_qoute
+                | "\r" @new_line   -> double_qoute
+                | "\n" @new_line   -> double_qoute
+                | "\\"             -> double_qoute_any
+                | '"'              -> final
+            ),
+            double_qoute_nonvarname: (
+                (any - [\\${"\r\n] - varname_first) -> double_qoute
+                | "\r" @new_line                    -> double_qoute
+                | "\n" @new_line                    -> double_qoute
+                | "\\"                              -> double_qoute_any
+                | '$'                               -> double_qoute_nonvarname
+                | '"'                               -> final
+            );
+
+        main := |*
+            "#!" any* :>> newline => {
+                lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
+            };
+            any => {
+                fnext html;
+                lex.ungetCnt(1)
+            };
+        *|;
+
+        html := |*
+            any_line+ -- '<?' => {
+                lex.ungetStr("<")
+                lex.setTokenPosition(tkn)
+                tok = token.T_INLINE_HTML;
+                fbreak;
+            };
+            '<?' => {
+                lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.te)
+                fnext php;
+            };
+            '<?php'i ( [ \t] | newline ) => {
+                lex.ungetCnt(lex.te - lex.ts - 5)
+                lex.addFreeFloatingToken(tkn, token.T_OPEN_TAG, lex.ts, lex.ts+5)
+                fnext php;
+            };
+            '<?='i => {
+                lex.setTokenPosition(tkn);
+                tok = token.T_ECHO;
+                fnext php;
+                fbreak;
+            };
+        *|;
+
+        php := |*
+            whitespace_line*                   => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
+            '?>' newline?                      => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;};
+            ';' whitespace_line* '?>' newline? => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext html; fbreak;};
+
+            (dnum | exponent_dnum)          => {lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;};
+            bnum => {
+                s := strings.Replace(string(lex.data[lex.ts+2:lex.te]), "_", "", -1)
+                _, err := strconv.ParseInt(s, 2, 0)
+
+                if err == nil {
+                    lex.setTokenPosition(tkn); tok = token.T_LNUMBER; fbreak;
+                }
+
+                lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;
+            };
+            lnum => {
+                base := 10
+                if lex.data[lex.ts] == '0' {
+                    base = 8
+                }
+
+                s := strings.Replace(string(lex.data[lex.ts:lex.te]), "_", "", -1)
+                _, err := strconv.ParseInt(s, base, 0)
+
+                if err == nil {
+                    lex.setTokenPosition(tkn); tok = token.T_LNUMBER; fbreak;
+                }
+
+                lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;
+            };
+            hnum => {
+                s := strings.Replace(string(lex.data[lex.ts+2:lex.te]), "_", "", -1)
+                _, err := strconv.ParseInt(s, 16, 0)
+
+                if err == nil {
+                    lex.setTokenPosition(tkn); tok = token.T_LNUMBER; fbreak;
+                }
+
+                lex.setTokenPosition(tkn); tok = token.T_DNUMBER; fbreak;
+            };
+
+            'namespace'i ('\\' varname)+      => {lex.setTokenPosition(tkn); tok = token.T_NAME_RELATIVE; fbreak;};
+            varname ('\\' varname)+           => {lex.setTokenPosition(tkn); tok = token.T_NAME_QUALIFIED; fbreak;};
+            '\\' varname ('\\' varname)*      => {lex.setTokenPosition(tkn); tok = token.T_NAME_FULLY_QUALIFIED; fbreak;};
+            '\\'                              => {lex.setTokenPosition(tkn); tok = token.T_NS_SEPARATOR; fbreak;};
+
+            'abstract'i                       => {lex.setTokenPosition(tkn); tok = token.T_ABSTRACT; fbreak;};
+            'array'i                          => {lex.setTokenPosition(tkn); tok = token.T_ARRAY; fbreak;};
+            'as'i                             => {lex.setTokenPosition(tkn); tok = token.T_AS; fbreak;};
+            'break'i                          => {lex.setTokenPosition(tkn); tok = token.T_BREAK; fbreak;};
+            'callable'i                       => {lex.setTokenPosition(tkn); tok = token.T_CALLABLE; fbreak;};
+            'case'i                           => {lex.setTokenPosition(tkn); tok = token.T_CASE; fbreak;};
+            'catch'i                          => {lex.setTokenPosition(tkn); tok = token.T_CATCH; fbreak;};
+            'class'i                          => {lex.setTokenPosition(tkn); tok = token.T_CLASS; fbreak;};
+            'clone'i                          => {lex.setTokenPosition(tkn); tok = token.T_CLONE; fbreak;};
+            'const'i                          => {lex.setTokenPosition(tkn); tok = token.T_CONST; fbreak;};
+            'continue'i                       => {lex.setTokenPosition(tkn); tok = token.T_CONTINUE; fbreak;};
+            'declare'i                        => {lex.setTokenPosition(tkn); tok = token.T_DECLARE; fbreak;};
+            'default'i                        => {lex.setTokenPosition(tkn); tok = token.T_DEFAULT; fbreak;};
+            'do'i                             => {lex.setTokenPosition(tkn); tok = token.T_DO; fbreak;};
+            'echo'i                           => {lex.setTokenPosition(tkn); tok = token.T_ECHO; fbreak;};
+            'else'i                           => {lex.setTokenPosition(tkn); tok = token.T_ELSE; fbreak;};
+            'elseif'i                         => {lex.setTokenPosition(tkn); tok = token.T_ELSEIF; fbreak;};
+            'empty'i                          => {lex.setTokenPosition(tkn); tok = token.T_EMPTY; fbreak;};
+            'enddeclare'i                     => {lex.setTokenPosition(tkn); tok = token.T_ENDDECLARE; fbreak;};
+            'endfor'i                         => {lex.setTokenPosition(tkn); tok = token.T_ENDFOR; fbreak;};
+            'endforeach'i                     => {lex.setTokenPosition(tkn); tok = token.T_ENDFOREACH; fbreak;};
+            'endif'i                          => {lex.setTokenPosition(tkn); tok = token.T_ENDIF; fbreak;};
+            'endswitch'i                      => {lex.setTokenPosition(tkn); tok = token.T_ENDSWITCH; fbreak;};
+            'endwhile'i                       => {lex.setTokenPosition(tkn); tok = token.T_ENDWHILE; fbreak;};
+            'eval'i                           => {lex.setTokenPosition(tkn); tok = token.T_EVAL; fbreak;};
+            'exit'i | 'die'i                  => {lex.setTokenPosition(tkn); tok = token.T_EXIT; fbreak;};
+            'extends'i                        => {lex.setTokenPosition(tkn); tok = token.T_EXTENDS; fbreak;};
+            'final'i                          => {lex.setTokenPosition(tkn); tok = token.T_FINAL; fbreak;};
+            'finally'i                        => {lex.setTokenPosition(tkn); tok = token.T_FINALLY; fbreak;};
+            'for'i                            => {lex.setTokenPosition(tkn); tok = token.T_FOR; fbreak;};
+            'foreach'i                        => {lex.setTokenPosition(tkn); tok = token.T_FOREACH; fbreak;};
+            'function'i | 'cfunction'i        => {lex.setTokenPosition(tkn); tok = token.T_FUNCTION; fbreak;};
+            'fn'i                             => {lex.setTokenPosition(tkn); tok = token.T_FN; fbreak;};
+            'global'i                         => {lex.setTokenPosition(tkn); tok = token.T_GLOBAL; fbreak;};
+            'goto'i                           => {lex.setTokenPosition(tkn); tok = token.T_GOTO; fbreak;};
+            'if'i                             => {lex.setTokenPosition(tkn); tok = token.T_IF; fbreak;};
+            'isset'i                          => {lex.setTokenPosition(tkn); tok = token.T_ISSET; fbreak;};
+            'implements'i                     => {lex.setTokenPosition(tkn); tok = token.T_IMPLEMENTS; fbreak;};
+            'instanceof'i                     => {lex.setTokenPosition(tkn); tok = token.T_INSTANCEOF; fbreak;};
+            'insteadof'i                      => {lex.setTokenPosition(tkn); tok = token.T_INSTEADOF; fbreak;};
+            'interface'i                      => {lex.setTokenPosition(tkn); tok = token.T_INTERFACE; fbreak;};
+            'list'i                           => {lex.setTokenPosition(tkn); tok = token.T_LIST; fbreak;};
+            'namespace'i                      => {lex.setTokenPosition(tkn); tok = token.T_NAMESPACE; fbreak;};
+            'private'i                        => {lex.setTokenPosition(tkn); tok = token.T_PRIVATE; fbreak;};
+            'public'i                         => {lex.setTokenPosition(tkn); tok = token.T_PUBLIC; fbreak;};
+            'print'i                          => {lex.setTokenPosition(tkn); tok = token.T_PRINT; fbreak;};
+            'protected'i                      => {lex.setTokenPosition(tkn); tok = token.T_PROTECTED; fbreak;};
+            'return'i                         => {lex.setTokenPosition(tkn); tok = token.T_RETURN; fbreak;};
+            'static'i                         => {lex.setTokenPosition(tkn); tok = token.T_STATIC; fbreak;};
+            'switch'i                         => {lex.setTokenPosition(tkn); tok = token.T_SWITCH; fbreak;};
+            'match'i                          => {lex.setTokenPosition(tkn); tok = token.T_MATCH; fbreak;};
+            'throw'i                          => {lex.setTokenPosition(tkn); tok = token.T_THROW; fbreak;};
+            'trait'i                          => {lex.setTokenPosition(tkn); tok = token.T_TRAIT; fbreak;};
+            'try'i                            => {lex.setTokenPosition(tkn); tok = token.T_TRY; fbreak;};
+            'unset'i                          => {lex.setTokenPosition(tkn); tok = token.T_UNSET; fbreak;};
+            'use'i                            => {lex.setTokenPosition(tkn); tok = token.T_USE; fbreak;};
+            'var'i                            => {lex.setTokenPosition(tkn); tok = token.T_VAR; fbreak;};
+            'while'i                          => {lex.setTokenPosition(tkn); tok = token.T_WHILE; fbreak;};
+            'yield'i whitespace_line+ 'from'i => {lex.setTokenPosition(tkn); tok = token.T_YIELD_FROM; fbreak;};
+            'yield'i                          => {lex.setTokenPosition(tkn); tok = token.T_YIELD; fbreak;};
+            'include'i                        => {lex.setTokenPosition(tkn); tok = token.T_INCLUDE; fbreak;};
+            'include_once'i                   => {lex.setTokenPosition(tkn); tok = token.T_INCLUDE_ONCE; fbreak;};
+            'require'i                        => {lex.setTokenPosition(tkn); tok = token.T_REQUIRE; fbreak;};
+            'require_once'i                   => {lex.setTokenPosition(tkn); tok = token.T_REQUIRE_ONCE; fbreak;};
+            '__CLASS__'i                      => {lex.setTokenPosition(tkn); tok = token.T_CLASS_C; fbreak;};
+            '__DIR__'i                        => {lex.setTokenPosition(tkn); tok = token.T_DIR; fbreak;};
+            '__FILE__'i                       => {lex.setTokenPosition(tkn); tok = token.T_FILE; fbreak;};
+            '__FUNCTION__'i                   => {lex.setTokenPosition(tkn); tok = token.T_FUNC_C; fbreak;};
+            '__LINE__'i                       => {lex.setTokenPosition(tkn); tok = token.T_LINE; fbreak;};
+            '__NAMESPACE__'i                  => {lex.setTokenPosition(tkn); tok = token.T_NS_C; fbreak;};
+            '__METHOD__'i                     => {lex.setTokenPosition(tkn); tok = token.T_METHOD_C; fbreak;};
+            '__TRAIT__'i                      => {lex.setTokenPosition(tkn); tok = token.T_TRAIT_C; fbreak;};
+            '__halt_compiler'i                => {lex.setTokenPosition(tkn); tok = token.T_HALT_COMPILER; fnext halt_compiller_open_parenthesis; fbreak;};
+            'new'i                            => {lex.setTokenPosition(tkn); tok = token.T_NEW; fbreak;};
+            'and'i                            => {lex.setTokenPosition(tkn); tok = token.T_LOGICAL_AND; fbreak;};
+            'or'i                             => {lex.setTokenPosition(tkn); tok = token.T_LOGICAL_OR; fbreak;};
+            'xor'i                            => {lex.setTokenPosition(tkn); tok = token.T_LOGICAL_XOR; fbreak;};
+            '#['                              => {lex.setTokenPosition(tkn); tok = token.T_ATTRIBUTE; fbreak;};
+            '...'                             => {lex.setTokenPosition(tkn); tok = token.T_ELLIPSIS; fbreak;};
+            '::'                              => {lex.setTokenPosition(tkn); tok = token.T_PAAMAYIM_NEKUDOTAYIM; fbreak;};
+            '&&'                              => {lex.setTokenPosition(tkn); tok = token.T_BOOLEAN_AND; fbreak;};
+            '||'                              => {lex.setTokenPosition(tkn); tok = token.T_BOOLEAN_OR; fbreak;};
+            '&='                              => {lex.setTokenPosition(tkn); tok = token.T_AND_EQUAL; fbreak;};
+            '|='                              => {lex.setTokenPosition(tkn); tok = token.T_OR_EQUAL; fbreak;};
+            '.='                              => {lex.setTokenPosition(tkn); tok = token.T_CONCAT_EQUAL; fbreak;};
+            '*='                              => {lex.setTokenPosition(tkn); tok = token.T_MUL_EQUAL; fbreak;};
+            '**='                             => {lex.setTokenPosition(tkn); tok = token.T_POW_EQUAL; fbreak;};
+            '/='                              => {lex.setTokenPosition(tkn); tok = token.T_DIV_EQUAL; fbreak;};
+            '+='                              => {lex.setTokenPosition(tkn); tok = token.T_PLUS_EQUAL; fbreak;};
+            '-='                              => {lex.setTokenPosition(tkn); tok = token.T_MINUS_EQUAL; fbreak;};
+            '^='                              => {lex.setTokenPosition(tkn); tok = token.T_XOR_EQUAL; fbreak;};
+            '%='                              => {lex.setTokenPosition(tkn); tok = token.T_MOD_EQUAL; fbreak;};
+            '--'                              => {lex.setTokenPosition(tkn); tok = token.T_DEC; fbreak;};
+            '++'                              => {lex.setTokenPosition(tkn); tok = token.T_INC; fbreak;};
+            '=>'                              => {lex.setTokenPosition(tkn); tok = token.T_DOUBLE_ARROW; fbreak;};
+            '<=>'                             => {lex.setTokenPosition(tkn); tok = token.T_SPACESHIP; fbreak;};
+            '!=' | '<>'                       => {lex.setTokenPosition(tkn); tok = token.T_IS_NOT_EQUAL; fbreak;};
+            '!=='                             => {lex.setTokenPosition(tkn); tok = token.T_IS_NOT_IDENTICAL; fbreak;};
+            '=='                              => {lex.setTokenPosition(tkn); tok = token.T_IS_EQUAL; fbreak;};
+            '==='                             => {lex.setTokenPosition(tkn); tok = token.T_IS_IDENTICAL; fbreak;};
+            '<<='                             => {lex.setTokenPosition(tkn); tok = token.T_SL_EQUAL; fbreak;};
+            '>>='                             => {lex.setTokenPosition(tkn); tok = token.T_SR_EQUAL; fbreak;};
+            '>='                              => {lex.setTokenPosition(tkn); tok = token.T_IS_GREATER_OR_EQUAL; fbreak;};
+            '<='                              => {lex.setTokenPosition(tkn); tok = token.T_IS_SMALLER_OR_EQUAL; fbreak;};
+            '**'                              => {lex.setTokenPosition(tkn); tok = token.T_POW; fbreak;};
+            '<<'                              => {lex.setTokenPosition(tkn); tok = token.T_SL; fbreak;};
+            '>>'                              => {lex.setTokenPosition(tkn); tok = token.T_SR; fbreak;};
+            '??'                              => {lex.setTokenPosition(tkn); tok = token.T_COALESCE; fbreak;};
+            '??='                             => {lex.setTokenPosition(tkn); tok = token.T_COALESCE_EQUAL; fbreak;};
+
+            '(' whitespace* 'array'i whitespace* ')'                     => {lex.setTokenPosition(tkn); tok = token.T_ARRAY_CAST; fbreak;};
+            '(' whitespace* ('bool'i|'boolean'i) whitespace* ')'         => {lex.setTokenPosition(tkn); tok = token.T_BOOL_CAST; fbreak;};
+            '(' whitespace* ('real'i) whitespace* ')'                    => {lex.error(fmt.Sprintf("The (real) cast has been removed, use (float) instead")); fbreak;};
+            '(' whitespace* ('double'i|'float'i) whitespace* ')'         => {lex.setTokenPosition(tkn); tok = token.T_DOUBLE_CAST; fbreak;};
+            '(' whitespace* ('int'i|'integer'i) whitespace* ')'          => {lex.setTokenPosition(tkn); tok = token.T_INT_CAST; fbreak;};
+            '(' whitespace* 'object'i whitespace* ')'                    => {lex.setTokenPosition(tkn); tok = token.T_OBJECT_CAST; fbreak;};
+            '(' whitespace* ('string'i|'binary'i) whitespace* ')'        => {lex.setTokenPosition(tkn); tok = token.T_STRING_CAST; fbreak;};
+            '(' whitespace* 'unset'i whitespace* ')'                     => {lex.error(fmt.Sprintf("The (unset) cast is no longer supported")); fbreak;};
+
+            (('#' ^'[') | '//') any_line* when is_not_comment_end => {
+                lex.ungetStr("?>")
+                lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
+            };
+            '#' => {
+                lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
+            };
+            '/*' any_line* :>> '*/' {
+                isDocComment := false;
+                if lex.te - lex.ts > 4 && string(lex.data[lex.ts:lex.ts+3]) == "/**" {
+                    isDocComment = true;
+                }
+
+                if isDocComment {
+                    lex.addFreeFloatingToken(tkn, token.T_DOC_COMMENT, lex.ts, lex.te)
+                } else {
+                    lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
+                }
+            };
+
+            operators => {
+                lex.setTokenPosition(tkn);
+                tok = token.ID(int(lex.data[lex.ts]));
+                fbreak;
+            };
+
+            "{"          => { lex.setTokenPosition(tkn); tok = token.ID(int('{')); lex.call(ftargs, fentry(php)); goto _out; };
+            "}"          => { lex.setTokenPosition(tkn); tok = token.ID(int('}')); lex.ret(1); goto _out;};
+            "$" varname  => { lex.setTokenPosition(tkn); tok = token.T_VARIABLE; fbreak; };
+            varname      => { lex.setTokenPosition(tkn); tok = token.T_STRING;   fbreak; };
+
+            "->"         => { lex.setTokenPosition(tkn); tok = token.T_OBJECT_OPERATOR; fnext property; fbreak; };
+            "?->"        => { lex.setTokenPosition(tkn); tok = token.T_NULLSAFE_OBJECT_OPERATOR; fnext property; fbreak; };
+
+            constant_string => {
+                lex.setTokenPosition(tkn);
+                tok = token.T_CONSTANT_ENCAPSED_STRING;
+                fbreak;
+            };
+
+            "b"i? "<<<" [ \t]* ( heredoc_label | ("'" heredoc_label "'") | ('"' heredoc_label '"') ) newline  => {
+                lex.heredocLabel = lex.data[lblStart:lblEnd]
+                lex.setTokenPosition(tkn);
+                tok = token.T_START_HEREDOC;
+
+                if lex.isHeredocEnd(lex.p+1) {
+                    fnext heredoc_end;
+                } else if lex.data[lblStart-1] == '\'' {
+                    fnext nowdoc;
+                } else {
+                    fnext heredoc;
+                }
+                fbreak;
+            };
+            "`" => {lex.setTokenPosition(tkn); tok = token.ID(int('`')); fnext backqote; fbreak;};
+            '"' => {lex.setTokenPosition(tkn); tok = token.ID(int('"')); fnext template_string; fbreak;};
+
+            any_line => {
+                c := lex.data[lex.p]
+                lex.error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c));
+            };
+        *|;
+
+        property := |*
+            whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
+            "->"             => {lex.setTokenPosition(tkn); tok = token.T_OBJECT_OPERATOR; fbreak;};
+            "?->"            => {lex.setTokenPosition(tkn); tok = token.T_NULLSAFE_OBJECT_OPERATOR; fbreak;};
+            varname          => {lex.setTokenPosition(tkn); tok = token.T_STRING; fnext php; fbreak;};
+            any              => {lex.ungetCnt(1); fgoto php;};
+        *|;
+
+        nowdoc := |*
+            any_line* when is_not_heredoc_end => {
+                lex.setTokenPosition(tkn);
+                tok = token.T_ENCAPSED_AND_WHITESPACE;
+                fnext heredoc_end;
+                fbreak;
+            };
+        *|;
+
+        heredoc := |*
+            "{$" => {lex.ungetCnt(1); lex.setTokenPosition(tkn); tok = token.T_CURLY_OPEN; lex.call(ftargs, fentry(php)); goto _out;};
+            "${" => {lex.setTokenPosition(tkn); tok = token.T_DOLLAR_OPEN_CURLY_BRACES; lex.call(ftargs, fentry(string_var_name)); goto _out;};
+            "$"  => {lex.ungetCnt(1); fcall string_var;};
+            any_line* when is_not_heredoc_end_or_var => {
+                lex.setTokenPosition(tkn);
+                tok = token.T_ENCAPSED_AND_WHITESPACE;
+
+                if len(lex.data) > lex.p+1 && lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
+                    fnext heredoc_end;
+                }
+                fbreak;
+            };
+        *|;
+
+        backqote := |*
+            "{$"              => {lex.ungetCnt(1); lex.setTokenPosition(tkn); tok = token.T_CURLY_OPEN; lex.call(ftargs, fentry(php)); goto _out;};
+            "${"              => {lex.setTokenPosition(tkn); tok = token.T_DOLLAR_OPEN_CURLY_BRACES; lex.call(ftargs, fentry(string_var_name)); goto _out;};
+            "$" varname_first => {lex.ungetCnt(2); fcall string_var;};
+            '`'               => {lex.setTokenPosition(tkn); tok = token.ID(int('`')); fnext php; fbreak;};
+            any_line* when is_not_backqoute_end_or_var => {
+                lex.setTokenPosition(tkn);
+                tok = token.T_ENCAPSED_AND_WHITESPACE;
+                fbreak;
+            };
+        *|;
+
+        template_string := |*
+            "{$"               => {lex.ungetCnt(1); lex.setTokenPosition(tkn); tok = token.T_CURLY_OPEN; lex.call(ftargs, fentry(php)); goto _out;};
+            "${"               => {lex.setTokenPosition(tkn); tok = token.T_DOLLAR_OPEN_CURLY_BRACES; lex.call(ftargs, fentry(string_var_name)); goto _out;};
+            "$" varname_first  => {lex.ungetCnt(2); fcall string_var;};
+            '"'                => {lex.setTokenPosition(tkn); tok = token.ID(int('"')); fnext php; fbreak;};
+            any_line* when is_not_string_end_or_var => {
+                lex.setTokenPosition(tkn);
+                tok = token.T_ENCAPSED_AND_WHITESPACE;
+                fbreak;
+            };
+        *|;
+
+        heredoc_end := |*
+            varname -- ";" => {
+                lex.setTokenPosition(tkn);
+                tok = token.T_END_HEREDOC;
+                fnext php;
+                fbreak;
+            };
+            varname => {
+                lex.setTokenPosition(tkn);
+                tok = token.T_END_HEREDOC;
+                fnext php;
+                fbreak;
+            };
+        *|;
+
+        string_var := |*
+            '$' varname         => {lex.setTokenPosition(tkn); tok = token.T_VARIABLE; fbreak;};
+            '->' varname_first  => {lex.ungetCnt(1); lex.setTokenPosition(tkn); tok = token.T_OBJECT_OPERATOR; fbreak;};
+            '?->' varname_first => {lex.ungetCnt(1); lex.setTokenPosition(tkn); tok = token.T_NULLSAFE_OBJECT_OPERATOR; fbreak;};
+            varname             => {lex.setTokenPosition(tkn); tok = token.T_STRING; fbreak;};
+            '['                 => {lex.setTokenPosition(tkn); tok = token.ID(int('[')); lex.call(ftargs, fentry(string_var_index)); goto _out;};
+            any                 => {lex.ungetCnt(1); fret;};
+        *|;
+
+        string_var_index := |*
+            lnum | hnum | bnum       => {lex.setTokenPosition(tkn); tok = token.T_NUM_STRING; fbreak;};
+            '$' varname              => {lex.setTokenPosition(tkn); tok = token.T_VARIABLE; fbreak;};
+            varname                  => {lex.setTokenPosition(tkn); tok = token.T_STRING; fbreak;};
+            whitespace_line | [\\'#] => {lex.setTokenPosition(tkn); tok = token.T_ENCAPSED_AND_WHITESPACE; lex.ret(2); goto _out;};
+            operators > (svi, 1)     => {lex.setTokenPosition(tkn); tok = token.ID(int(lex.data[lex.ts])); fbreak;};
+            ']'       > (svi, 2)     => {lex.setTokenPosition(tkn); tok = token.ID(int(']')); lex.ret(2); goto _out;};
+            any_line => {
+                c := lex.data[lex.p]
+                lex.error(fmt.Sprintf("WARNING: Unexpected character in input: '%c' (ASCII=%d)", c, c));
+            };
+        *|;
+
+        string_var_name := |*
+            varname ("[" | "}") => {lex.ungetCnt(1); lex.setTokenPosition(tkn); tok = token.T_STRING_VARNAME; fnext php; fbreak;};
+            any                 => {lex.ungetCnt(1); fnext php;};
+        *|;
+
+        halt_compiller_open_parenthesis := |*
+            whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
+            "("              => {lex.setTokenPosition(tkn); tok = token.ID(int('(')); fnext halt_compiller_close_parenthesis; fbreak;};
+            any              => {lex.ungetCnt(1); fnext php;};
+        *|;
+
+        halt_compiller_close_parenthesis := |*
+            whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
+            ")"              => {lex.setTokenPosition(tkn); tok = token.ID(int(')')); fnext halt_compiller_close_semicolon; fbreak;};
+            any              => {lex.ungetCnt(1); fnext php;};
+        *|;
+
+        halt_compiller_close_semicolon := |*
+            whitespace_line* => {lex.addFreeFloatingToken(tkn, token.T_WHITESPACE, lex.ts, lex.te)};
+            ";"              => {lex.setTokenPosition(tkn); tok = token.ID(int(';')); fnext halt_compiller_end; fbreak;};
+            any              => {lex.ungetCnt(1); fnext php;};
+        *|;
+
+        halt_compiller_end := |*
+            any_line* => { lex.addFreeFloatingToken(tkn, token.T_HALT_COMPILER, lex.ts, lex.te); };
+        *|;
+
+        write exec;
+    }%%
+
+    tkn.Value = lex.data[lex.ts:lex.te]
+    tkn.ID = token.ID(tok)
+
+    return tkn
+}
--- a/internal/php8/scanner_php8_test.go
+++ b/internal/php8/scanner_php8_test.go
@@ -0,0 +1,346 @@
+package php8_test
+
+import (
+	"testing"
+
+	"github.com/z7zmey/php-parser/internal/php8"
+	"github.com/z7zmey/php-parser/internal/tester"
+	"github.com/z7zmey/php-parser/pkg/conf"
+	"github.com/z7zmey/php-parser/pkg/token"
+	"gotest.tools/assert"
+)
+
+func TestNullsafeMethodCallTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStringTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = "<?php $a?->foo();"
+	suite.Expected = []string{
+		"$a",
+		"?->",
+		"foo",
+		"(",
+		")",
+		";",
+	}
+	suite.Run()
+}
+
+func TestNullsafePropertyFetchTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStringTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = "<?php $a?->prop;"
+	suite.Expected = []string{
+		"$a",
+		"?->",
+		"prop",
+		";",
+	}
+	suite.Run()
+}
+
+func TestNullsafePropertyFetchInStringTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStringTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = "<?php \"$a?->prop\";"
+	suite.Expected = []string{
+		"\"",
+		"$a",
+		"?->",
+		"prop",
+		"\"",
+		";",
+	}
+	suite.Run()
+}
+
+func TestNullsafeMethodCallTokensFreeFloating(t *testing.T) {
+	suite := tester.NewLexerTokenFreeFloatingTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = `<?php
+	$a ?-> bar ( '' ) ;`
+
+	suite.Expected = [][]*token.Token{
+		{
+			{
+				ID:    token.T_OPEN_TAG,
+				Value: []byte("<?php"),
+			},
+			{
+				ID:    token.T_WHITESPACE,
+				Value: []byte("\n\t"),
+			},
+		},
+		{
+			{
+				ID:    token.T_WHITESPACE,
+				Value: []byte(" "),
+			},
+		},
+		{
+			{
+				ID:    token.T_WHITESPACE,
+				Value: []byte(" "),
+			},
+		},
+		{
+			{
+				ID:    token.T_WHITESPACE,
+				Value: []byte(" "),
+			},
+		},
+		{
+			{
+				ID:    token.T_WHITESPACE,
+				Value: []byte(" "),
+			},
+		},
+		{
+			{
+				ID:    token.T_WHITESPACE,
+				Value: []byte(" "),
+			},
+		},
+		{
+			{
+				ID:    token.T_WHITESPACE,
+				Value: []byte(" "),
+			},
+		},
+	}
+
+	suite.Run()
+}
+
+func TestMatchStringTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStringTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = "<?php match($a) {}"
+	suite.Expected = []string{
+		"match",
+		"(",
+		"$a",
+		")",
+		"{",
+		"}",
+	}
+	suite.Run()
+}
+
+func TestMatchWithConditionStringTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStringTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = "<?php match($a) { 10 => 100 }"
+	suite.Expected = []string{
+		"match",
+		"(",
+		"$a",
+		")",
+		"{",
+		"10",
+		"=>",
+		"100",
+		"}",
+	}
+	suite.Run()
+}
+
+func TestMatchWithDefaultStringTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStringTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = "<?php match($a) { default => 10 }"
+	suite.Expected = []string{
+		"match",
+		"(",
+		"$a",
+		")",
+		"{",
+		"default",
+		"=>",
+		"10",
+		"}",
+	}
+	suite.Run()
+}
+
+func TestAttributeTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStringTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = "<?php #[ FooAttribute]"
+	suite.Expected = []string{
+		"#[",
+		"FooAttribute",
+		"]",
+	}
+	suite.Run()
+}
+
+func TestCommentEnd2(t *testing.T) {
+	src := `<?php //`
+
+	expected := []*token.Token{
+		{
+			ID:    token.T_OPEN_TAG,
+			Value: []byte("<?php"),
+		},
+
+		{
+			ID:    token.T_WHITESPACE,
+			Value: []byte(" "),
+		},
+		{
+			ID:    token.T_COMMENT,
+			Value: []byte("//"),
+		},
+	}
+
+	lexer := php8.NewLexer([]byte(src), conf.Config{})
+
+	tkn := lexer.Lex()
+
+	actual := tkn.FreeFloating
+	for _, v := range actual {
+		v.Position = nil
+	}
+
+	assert.DeepEqual(t, expected, actual)
+}
+
+func TestCommentEnd3(t *testing.T) {
+	src := `<?php #`
+
+	expected := []*token.Token{
+		{
+			ID:    token.T_OPEN_TAG,
+			Value: []byte("<?php"),
+		},
+
+		{
+			ID:    token.T_WHITESPACE,
+			Value: []byte(" "),
+		},
+		{
+			ID:    token.T_COMMENT,
+			Value: []byte("#"),
+		},
+	}
+
+	lexer := php8.NewLexer([]byte(src), conf.Config{})
+
+	tkn := lexer.Lex()
+
+	actual := tkn.FreeFloating
+	for _, v := range actual {
+		v.Position = nil
+	}
+
+	assert.DeepEqual(t, expected, actual)
+}
+
+func TestAttribute(t *testing.T) {
+	src := `<?php #[ FooAttribute]`
+
+	expected := []*token.Token{
+		{
+			ID:    token.T_OPEN_TAG,
+			Value: []byte("<?php"),
+		},
+		{
+			ID:    token.T_WHITESPACE,
+			Value: []byte(" "),
+		},
+	}
+
+	lexer := php8.NewLexer([]byte(src), conf.Config{})
+
+	tkn := lexer.Lex()
+
+	actual := tkn.FreeFloating
+	for _, v := range actual {
+		v.Position = nil
+	}
+
+	assert.DeepEqual(t, expected, actual)
+
+	assert.DeepEqual(t, token.T_ATTRIBUTE, tkn.ID)
+	assert.DeepEqual(t, "#[", string(tkn.Value))
+}
+
+func TestNamespaceFullyQualifiedTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStructTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = `<?php use \Foo;`
+	suite.Expected = []*token.Token{
+		{
+			ID:    php8.T_USE,
+			Value: []byte("use"),
+		},
+		{
+			ID:    php8.T_NAME_FULLY_QUALIFIED,
+			Value: []byte(`\Foo`),
+		},
+		{
+			ID:    ';',
+			Value: []byte(";"),
+		},
+	}
+	suite.Run()
+}
+
+func TestNamespaceFullyQualifiedWithKeywordsTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStructTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = `<?php use \Foo\match\fn;`
+	suite.Expected = []*token.Token{
+		{
+			ID:    php8.T_USE,
+			Value: []byte("use"),
+		},
+		{
+			ID:    php8.T_NAME_FULLY_QUALIFIED,
+			Value: []byte(`\Foo\match\fn`),
+		},
+		{
+			ID:    ';',
+			Value: []byte(";"),
+		},
+	}
+	suite.Run()
+}
+
+func TestNamespaceQualifiedTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStructTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = `<?php namespace Boo\Foo;`
+	suite.Expected = []*token.Token{
+		{
+			ID:    php8.T_NAMESPACE,
+			Value: []byte("namespace"),
+		},
+		{
+			ID:    php8.T_NAME_QUALIFIED,
+			Value: []byte(`Boo\Foo`),
+		},
+		{
+			ID:    ';',
+			Value: []byte(";"),
+		},
+	}
+	suite.Run()
+}
+
+func TestNamespaceRelativeTokens(t *testing.T) {
+	suite := tester.NewLexerTokenStructTestSuite(t)
+	suite.UsePHP8()
+	suite.Code = `<?php namespace\match;`
+	suite.Expected = []*token.Token{
+		{
+			ID:    php8.T_NAME_RELATIVE,
+			Value: []byte(`namespace\match`),
+		},
+		{
+			ID:    ';',
+			Value: []byte(";"),
+		},
+	}
+	suite.Run()
+}
--- a/internal/php8/scanner_test.go
+++ b/internal/php8/scanner_test.go
--- a/internal/php8/test.php
+++ b/internal/php8/test.php
@@ -0,0 +1,356 @@
+<?
+foo($a, ...$b);
+$foo($a, ...$b);
+$foo->bar($a, ...$b);
+foo::bar($a, ...$b);
+$foo::bar($a, ...$b);
+new foo($a, ...$b);
+/** anonymous class */
+new class ($a, ...$b) {};
+new class {};
+new $foo;
+new $foo[1];
+new $foo{$bar};
+new $foo->bar;
+new $foo::$bar;
+new static::$bar;
+
+function foo(?bar $bar=null, baz &...$baz) {}
+class foo {public function foo(?bar $bar=null, baz &...$baz) {}}
+function(?bar $bar=null, baz &...$baz) {};
+static function(?bar $bar=null, baz &...$baz) {};
+
+1234567890123456789;
+12345678901234567890;
+0.;
+0b0111111111111111111111111111111111111111111111111111111111111111;
+0b1111111111111111111111111111111111111111111111111111111111111111;
+0x007111111111111111;
+0x8111111111111111;
+__CLASS__;
+__DIR__;
+__FILE__;
+__FUNCTION__;
+__LINE__;
+__NAMESPACE__;
+__METHOD__;
+__TRAIT__;
+
+"test $var";
+"test $var[1]";
+"test $var[-1]";
+"test $var[1234567890123456789012345678901234567890]";
+"test $var[-1234567890123456789012345678901234567890]";
+"test $var[bar]";
+"test $var[$bar]";
+"$foo $bar";
+"test $foo->bar()";
+"test ${foo}";
+"test ${foo[0]}";
+"test ${$foo}";
+"test {$foo->bar()}";
+
+if ($a) :
+endif;
+if ($a) :
+elseif ($b):
+endif;
+if ($a) :
+else:
+endif;
+if ($a) :
+elseif ($b):
+elseif ($c):
+else:
+endif;
+
+while (1) { break; }
+while (1) { break 2; }
+while (1) : break(3); endwhile;
+class foo{ public const FOO = 1, BAR = 2; }
+class foo{ const FOO = 1, BAR = 2; }
+class foo{ function bar() {} }
+class foo{ public static function &bar() {} }
+class foo{ public static function &bar(): void {} }
+abstract class foo{ }
+final class foo extends bar { }
+final class foo implements bar { }
+final class foo implements bar, baz { }
+new class() extends foo implements bar, baz { };
+
+const FOO = 1, BAR = 2;
+while (1) { continue; }
+while (1) { continue 2; }
+while (1) { continue(3); }
+declare(ticks=1);
+declare(ticks=1) {}
+declare(ticks=1): enddeclare;
+do {} while(1);
+echo $a, 1;
+echo($a);
+for($i = 0; $i < 10; $i++, $i++) {}
+for(; $i < 10; $i++, $i++) : endfor;
+foreach ($a as $v) {}
+foreach ($a as $v) : endforeach;
+foreach ($a as $k => $v) {}
+foreach ($a as $k => &$v) {}
+foreach ($a as $k => list($v)) {}
+foreach ($a as $k => [$v]) {}
+function foo() {}
+function foo() {return;}
+function &foo() {return 1;}
+function &foo(): void {}
+global $a, $b;
+a:
+goto a;
+if ($a) {}
+if ($a) {} elseif ($b) {}
+if ($a) {} else {}
+if ($a) {} elseif ($b) {} elseif ($c) {} else {}
+if ($a) {} elseif ($b) {} else if ($c) {} else {}
+?> <div></div> <?
+interface Foo {}
+interface Foo extends Bar {}
+interface Foo extends Bar, Baz {}
+namespace Foo;
+namespace Foo {}
+namespace {}
+class foo {var $a;}
+class foo {public static $a, $b = 1;}
+static $a, $b = 1;
+
+switch (1) :
+    case 1:
+    default:
+    case 2:
+endswitch;
+
+switch (1) :;
+    case 1;
+    case 2;
+endswitch;
+
+switch (1) {
+    case 1: break;
+    case 2: break;
+}
+
+switch (1) {;
+    case 1; break;
+    case 2; break;
+}
+
+throw $e;
+
+trait Foo {}
+class Foo { use Bar; }
+class Foo { use Bar, Baz {} }
+class Foo { use Bar, Baz { one as include; } }
+class Foo { use Bar, Baz { one as public; } }
+class Foo { use Bar, Baz { one as public two; } }
+class Foo { use Bar, Baz { Bar::one insteadof Baz, Quux; Baz::one as two; } }
+
+try {}
+try {} catch (Exception $e) {}
+try {} catch (Exception|RuntimeException $e) {}
+try {} catch (Exception $e) {} catch (RuntimeException $e) {}
+try {} catch (Exception $e) {} finally {}
+
+unset($a, $b,);
+
+use Foo;
+use \Foo;
+use \Foo as Bar;
+use Foo, Bar;
+use Foo, Bar as Baz;
+use function Foo, \Bar;
+use function Foo as foo, \Bar as bar;
+use const Foo, \Bar;
+use const Foo as foo, \Bar as bar;
+
+use \Foo\{Bar, Baz};
+use Foo\{Bar, Baz as quux};
+use function Foo\{Bar, Baz};
+use const \Foo\{Bar, Baz};
+use Foo\{const Bar, function Baz};
+
+$a[1];
+$a[1][2];
+array();
+array(1);
+array(1=>1, &$b,);
+~$a;
+!$a;
+
+Foo::Bar;
+$foo::Bar;
+clone($a);
+clone $a;
+function(){};
+function($a, $b) use ($c, &$d) {};
+function(): void {};
+foo;
+namespace\foo;
+\foo;
+
+empty($a);
+@$a;
+eval($a);
+exit;
+exit($a);
+die;
+die($a);
+foo();
+namespace\foo();
+\foo();
+$foo();
+
+$a--;
+$a++;
+--$a;
++$a;
+
+include $a;
+include_once $a;
+require $a;
+require_once $a;
+
+$a instanceof Foo;
+$a instanceof namespace\Foo;
+$a instanceof \Foo;
+
+isset($a, $b);
+list($a) = $b;
+list($a[]) = $b;
+list(list($a)) = $b;
+
+$a->foo();
+new Foo();
+new namespace\Foo();
+new \Foo();
+new class ($a, ...$b) {};
+print($a);
+$a->foo;
+`cmd $a`;
+`cmd`;
+``;
+[];
+[1];
+[1=>1, &$b,];
+
+[$a] = $b;
+[$a[]] = $b;
+[list($a)] = $b;
+Foo::bar();
+namespace\Foo::bar();
+\Foo::bar();
+Foo::$bar;
+$foo::$bar;
+namespace\Foo::$bar;
+\Foo::$bar;
+$a ? $b : $c;
+$a ? : $c;
+$a ? $b ? $c : $d : $e;
+$a ? $b : $c ? $d : $e;
+-$a;
+$a;
+$$a;
+yield;
+yield $a;
+yield $a => $b;
+yield from $a;
+
+(array)$a;
+(boolean)$a;
+(bool)$a;
+(double)$a;
+(float)$a;
+(integer)$a;
+(int)$a;
+(object)$a;
+(string)$a;
+(unset)$a;
+
+$a & $b;
+$a | $b;
+$a ^ $b;
+$a && $b;
+$a || $b;
+$a ?? $b;
+$a . $b;
+$a / $b;
+$a == $b;
+$a >= $b;
+$a > $b;
+$a === $b;
+$a and $b;
+$a or $b;
+$a xor $b;
+$a - $b;
+$a % $b;
+$a * $b;
+$a != $b;
+$a !== $b;
+$a + $b;
+$a ** $b;
+$a << $b;
+$a >> $b;
+$a <= $b;
+$a < $b;
+$a <=> $b;
+
+$a =& $b;
+$a = $b;
+$a &= $b;
+$a |= $b;
+$a ^= $b;
+$a .= $b;
+$a /= $b;
+$a -= $b;
+$a %= $b;
+$a *= $b;
+$a += $b;
+$a **= $b;
+$a <<= $b;
+$a >>= $b;
+
+class foo {public function class() {} }
+\foo\bar();
+
+function foo(&$a, ...$b) {
+
+    function bar() {}
+    class Baz {}
+    trait Quux{}
+    interface Quuux {}
+}
+
+function foo(&$a = 1, ...$b = 1, $c = 1) {}
+function foo(array $a, callable $b) {}
+abstract final class foo { abstract protected static function bar(); final private function baz() {} }
+
+(new Foo)->bar;
+(new Foo)();
+[$foo][0]();
+foo[1]();
+"foo"();
+[1]{$foo}();
+${foo()};
+
+Foo::$bar();
+Foo::{$bar[0]}();
+
+$foo->$bar;
+$foo->{$bar[0]};
+
+[1=>&$a, 2=>list($b)];
+
+__halt_compiler();
+
+(new Foo)?->bar;
+(new Foo)?->bar();
+$foo?->$bar;
+$foo?->{$bar[0]};
+"{$foo?->{$bar[0]}}";
+
+parsing process must be terminated
--- a/internal/tester/lexer_token_freefloating.go
+++ b/internal/tester/lexer_token_freefloating.go
@@ -0,0 +1,62 @@
+package tester
+
+import (
+	"testing"
+
+	"github.com/z7zmey/php-parser/internal/php8"
+	"github.com/z7zmey/php-parser/internal/scanner"
+	"github.com/z7zmey/php-parser/pkg/conf"
+	"github.com/z7zmey/php-parser/pkg/token"
+	"github.com/z7zmey/php-parser/pkg/version"
+	"gotest.tools/assert"
+)
+
+type Lexer interface {
+	Lex() *token.Token
+}
+
+type LexerTokenFreeFloatingTestSuite struct {
+	t *testing.T
+
+	Code     string
+	Expected [][]*token.Token
+
+	Version version.Version
+}
+
+func NewLexerTokenFreeFloatingTestSuite(t *testing.T) *LexerTokenFreeFloatingTestSuite {
+	return &LexerTokenFreeFloatingTestSuite{
+		t: t,
+		Version: version.Version{
+			Major: 7,
+			Minor: 4,
+		},
+	}
+}
+
+func (l *LexerTokenFreeFloatingTestSuite) UsePHP8() {
+	l.Version = version.Version{Major: 8, Minor: 0}
+}
+
+func (l *LexerTokenFreeFloatingTestSuite) Run() {
+	config := conf.Config{
+		Version: &l.Version,
+	}
+
+	var lexer Lexer
+
+	if l.Version.Less(&version.Version{Major: 8, Minor: 0}) {
+		lexer = scanner.NewLexer([]byte(l.Code), config)
+	} else {
+		lexer = php8.NewLexer([]byte(l.Code), config)
+	}
+
+	for _, expected := range l.Expected {
+		tkn := lexer.Lex()
+		actual := tkn.FreeFloating
+		for _, v := range actual {
+			v.Position = nil
+		}
+		assert.DeepEqual(l.t, expected, actual)
+	}
+}
--- a/internal/tester/lexer_token_string.go
+++ b/internal/tester/lexer_token_string.go
@@ -0,0 +1,54 @@
+package tester
+
+import (
+	"testing"
+
+	"github.com/z7zmey/php-parser/internal/php8"
+	"github.com/z7zmey/php-parser/internal/scanner"
+	"github.com/z7zmey/php-parser/pkg/conf"
+	"github.com/z7zmey/php-parser/pkg/version"
+	"gotest.tools/assert"
+)
+
+type LexerTokenStringTestSuite struct {
+	t *testing.T
+
+	Code     string
+	Expected []string
+
+	Version version.Version
+}
+
+func NewLexerTokenStringTestSuite(t *testing.T) *LexerTokenStringTestSuite {
+	return &LexerTokenStringTestSuite{
+		t: t,
+		Version: version.Version{
+			Major: 7,
+			Minor: 4,
+		},
+	}
+}
+
+func (l *LexerTokenStringTestSuite) UsePHP8() {
+	l.Version = version.Version{Major: 8, Minor: 0}
+}
+
+func (l *LexerTokenStringTestSuite) Run() {
+	config := conf.Config{
+		Version: &l.Version,
+	}
+
+	var lexer Lexer
+
+	if l.Version.Less(&version.Version{Major: 8, Minor: 0}) {
+		lexer = scanner.NewLexer([]byte(l.Code), config)
+	} else {
+		lexer = php8.NewLexer([]byte(l.Code), config)
+	}
+
+	for _, expected := range l.Expected {
+		tkn := lexer.Lex()
+		actual := string(tkn.Value)
+		assert.DeepEqual(l.t, expected, actual)
+	}
+}
--- a/internal/tester/lexer_token_struct.go
+++ b/internal/tester/lexer_token_struct.go
@@ -0,0 +1,56 @@
+package tester
+
+import (
+	"testing"
+
+	"github.com/z7zmey/php-parser/internal/php8"
+	"github.com/z7zmey/php-parser/internal/scanner"
+	"github.com/z7zmey/php-parser/pkg/conf"
+	"github.com/z7zmey/php-parser/pkg/token"
+	"github.com/z7zmey/php-parser/pkg/version"
+	"gotest.tools/assert"
+)
+
+type LexerTokenStructTestSuite struct {
+	t *testing.T
+
+	Code     string
+	Expected []*token.Token
+
+	Version version.Version
+}
+
+func NewLexerTokenStructTestSuite(t *testing.T) *LexerTokenStructTestSuite {
+	return &LexerTokenStructTestSuite{
+		t: t,
+		Version: version.Version{
+			Major: 7,
+			Minor: 4,
+		},
+	}
+}
+
+func (l *LexerTokenStructTestSuite) UsePHP8() {
+	l.Version = version.Version{Major: 8, Minor: 0}
+}
+
+func (l *LexerTokenStructTestSuite) Run() {
+	config := conf.Config{
+		Version: &l.Version,
+	}
+
+	var lexer Lexer
+
+	if l.Version.Less(&version.Version{Major: 8, Minor: 0}) {
+		lexer = scanner.NewLexer([]byte(l.Code), config)
+	} else {
+		lexer = php8.NewLexer([]byte(l.Code), config)
+	}
+
+	for _, expected := range l.Expected {
+		actual := lexer.Lex()
+		actual.Position = nil
+		actual.FreeFloating = nil
+		assert.DeepEqual(l.t, expected, actual)
+	}
+}
--- a/internal/tester/parse_print.go
+++ b/internal/tester/parse_print.go
@@ -0,0 +1,60 @@
+package tester
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/z7zmey/php-parser/pkg/ast"
+	"github.com/z7zmey/php-parser/pkg/conf"
+	"github.com/z7zmey/php-parser/pkg/parser"
+	"github.com/z7zmey/php-parser/pkg/version"
+	"github.com/z7zmey/php-parser/pkg/visitor/printer"
+	"gotest.tools/assert"
+)
+
+type ParserPrintTestSuite struct {
+	t       *testing.T
+	Version version.Version
+}
+
+func NewParserPrintTestSuite(t *testing.T) *ParserPrintTestSuite {
+	return &ParserPrintTestSuite{
+		t: t,
+		Version: version.Version{
+			Major: 7,
+			Minor: 4,
+		},
+	}
+}
+
+func (p *ParserPrintTestSuite) UsePHP8() *ParserPrintTestSuite {
+	p.Version = version.Version{Major: 8, Minor: 0}
+	return p
+}
+
+func (p *ParserPrintTestSuite) Run(code string) {
+	actual := p.print(p.parse(code))
+	assert.DeepEqual(p.t, code, actual)
+}
+
+func (p *ParserPrintTestSuite) parse(src string) ast.Vertex {
+	config := conf.Config{
+		Version: &p.Version,
+	}
+
+	root, err := parser.Parse([]byte(src), config)
+	if err != nil {
+		p.t.Fatal(err)
+	}
+
+	return root
+}
+
+func (p *ParserPrintTestSuite) print(n ast.Vertex) string {
+	o := bytes.NewBufferString("")
+
+	pr := printer.NewPrinter(o)
+	n.Accept(pr)
+
+	return o.String()
+}
--- a/internal/tester/parser.go
+++ b/internal/tester/parser.go
@@ -0,0 +1,46 @@
+package tester
+
+import (
+	"testing"
+
+	"github.com/z7zmey/php-parser/pkg/ast"
+	"github.com/z7zmey/php-parser/pkg/conf"
+	"github.com/z7zmey/php-parser/pkg/parser"
+	"github.com/z7zmey/php-parser/pkg/version"
+	"gotest.tools/assert"
+)
+
+type ParserTestSuite struct {
+	t *testing.T
+
+	Code     string
+	Expected ast.Vertex
+
+	Version version.Version
+}
+
+func NewParserTestSuite(t *testing.T) *ParserTestSuite {
+	return &ParserTestSuite{
+		t: t,
+		Version: version.Version{
+			Major: 7,
+			Minor: 4,
+		},
+	}
+}
+
+func (p *ParserTestSuite) UsePHP8() {
+	p.Version = version.Version{Major: 8, Minor: 0}
+}
+
+func (p *ParserTestSuite) Run() {
+	config := conf.Config{
+		Version: &p.Version,
+	}
+
+	actual, err := parser.Parse([]byte(p.Code), config)
+	if err != nil {
+		p.t.Fatalf("Error parse: %v", err)
+	}
+	assert.DeepEqual(p.t, p.Expected, actual)
+}
--- a/internal/tester/parser_dump.go
+++ b/internal/tester/parser_dump.go
@@ -0,0 +1,64 @@
+package tester
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/z7zmey/php-parser/pkg/conf"
+	"github.com/z7zmey/php-parser/pkg/parser"
+	"github.com/z7zmey/php-parser/pkg/version"
+	"github.com/z7zmey/php-parser/pkg/visitor/dumper"
+	"gotest.tools/assert"
+)
+
+type ParserDumpTestSuite struct {
+	t *testing.T
+
+	Code     string
+	Expected string
+
+	Version version.Version
+
+	actualDump *bytes.Buffer
+	dumper     *dumper.Dumper
+}
+
+func NewParserDumpTestSuite(t *testing.T) *ParserDumpTestSuite {
+	actualDump := bytes.NewBuffer(nil)
+	return &ParserDumpTestSuite{
+		t: t,
+		Version: version.Version{
+			Major: 7,
+			Minor: 4,
+		},
+		actualDump: actualDump,
+		dumper:     dumper.NewDumper(actualDump),
+	}
+}
+
+func (p *ParserDumpTestSuite) WithTokens() {
+	p.dumper = p.dumper.WithTokens()
+}
+
+func (p *ParserDumpTestSuite) WithPositions() {
+	p.dumper = p.dumper.WithPositions()
+}
+
+func (p *ParserDumpTestSuite) UsePHP8() {
+	p.Version = version.Version{Major: 8, Minor: 0}
+}
+
+func (p *ParserDumpTestSuite) Run() {
+	config := conf.Config{
+		Version: &p.Version,
+	}
+
+	actual, err := parser.Parse([]byte(p.Code), config)
+	if err != nil {
+		p.t.Fatalf("Error parse: %v", err)
+	}
+
+	p.dumper.Dump(actual)
+
+	assert.DeepEqual(p.t, p.Expected+"\n", p.actualDump.String())
+}
--- a/internal/tester/parser_error.go
+++ b/internal/tester/parser_error.go
@@ -0,0 +1,52 @@
+package tester
+
+import (
+	"testing"
+
+	"github.com/z7zmey/php-parser/pkg/conf"
+	"github.com/z7zmey/php-parser/pkg/errors"
+	"github.com/z7zmey/php-parser/pkg/parser"
+	"github.com/z7zmey/php-parser/pkg/version"
+	"gotest.tools/assert"
+)
+
+type ParserErrorTestSuite struct {
+	t *testing.T
+
+	Code     string
+	Expected []*errors.Error
+
+	Version version.Version
+}
+
+func NewParserErrorTestSuite(t *testing.T) *ParserErrorTestSuite {
+	return &ParserErrorTestSuite{
+		t: t,
+		Version: version.Version{
+			Major: 7,
+			Minor: 4,
+		},
+	}
+}
+
+func (p *ParserErrorTestSuite) UsePHP8() {
+	p.Version = version.Version{Major: 8, Minor: 0}
+}
+
+func (p *ParserErrorTestSuite) Run() {
+	config := conf.Config{
+		Version: &p.Version,
+	}
+
+	var errs []*errors.Error
+
+	config.ErrorHandlerFunc = func(e *errors.Error) {
+		errs = append(errs, e)
+	}
+
+	_, err := parser.Parse([]byte(p.Code), config)
+	if err != nil {
+		p.t.Fatalf("Error parse: %v", err)
+	}
+	assert.DeepEqual(p.t, p.Expected, errs)
+}